From 2bd81bb7ca653772b3be94f39ace4beb4ad00908 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 26 Aug 2026 17:13:39 +1000 Subject: [PATCH] Advance HEIF codec implementation --- HEIF_IMPLEMENTATION_PLAN.md | 12 +- src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 107 ++- .../Formats/Heif/Av1/Color/Av1YuvConverter.cs | 57 +- .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 2 +- .../Heif/Av1/OpenBitstreamUnit/ObuWriter.cs | 2 +- .../Heif/Av1/Pipeline/Av1FrameDecoder.cs | 2 +- .../Av1DeQuantizationContext.cs | 2 +- .../Av1InverseQuantizationLookup.cs | 2 +- .../Av1InverseQuantizer.cs | 6 +- .../Av1QuantizationLookup.cs | 2 +- .../Heif/Av1/Transform/Av1BlockDecoder.cs | 2 +- .../Av1ByteInverseTransformOutputOperator.cs | 18 + .../Av1/Transform/Av1ForwardTransformer.cs | 165 ++++ ...hBitDepthInverseTransformOutputOperator.cs | 14 + .../Av1/Transform/Av1IdentityTransform1d.cs | 27 + .../Av1/Transform/Av1Inverse2dTransformer.cs | 183 +++++ .../Heif/Av1/Transform/Av1Transform1dMath.cs | 72 ++ .../Av1/Transform/Av1Transform2dOperations.cs | 144 +++- .../Av1/Transform/Av1TransformWorkspace.cs | 9 +- .../Av1Adst16Forward1dOperator.Simd.cs | 179 ++++- .../Forward/Av1Adst4Forward1dOperator.Simd.cs | 33 + .../Forward/Av1Adst8Forward1dOperator.Simd.cs | 86 +- .../Forward/Av1Dct16Forward1dOperator.Simd.cs | 142 +++- .../Forward/Av1Dct32Forward1dOperator.Simd.cs | 322 +++++++- .../Forward/Av1Dct4Forward1dOperator.Simd.cs | 33 +- .../Forward/Av1Dct64Forward1dOperator.Simd.cs | 742 ++++++++++++++++- .../Forward/Av1Dct8Forward1dOperator.Simd.cs | 66 +- .../Av1IdentityForward1dOperators.Simd.cs | 56 ++ .../IAv1InverseTransformOutputOperator.cs | 9 + .../Av1/Transform/IAv1Transform1dOperator.cs | 15 + .../Av1Adst16Inverse1dOperator.Simd.cs | 183 ++++- .../Inverse/Av1Adst4Inverse1dOperator.Simd.cs | 33 + .../Inverse/Av1Adst8Inverse1dOperator.Simd.cs | 90 ++- .../Inverse/Av1Dct16Inverse1dOperator.Simd.cs | 152 +++- .../Inverse/Av1Dct32Inverse1dOperator.Simd.cs | 336 +++++++- .../Inverse/Av1Dct4Inverse1dOperator.Simd.cs | 31 +- .../Inverse/Av1Dct64Inverse1dOperator.Simd.cs | 751 +++++++++++++++++- .../Inverse/Av1Dct8Inverse1dOperator.Simd.cs | 71 +- .../Av1IdentityInverse1dOperators.Simd.cs | 56 ++ .../Formats/Heif/Av1HeifItemDecoder.cs | 81 +- .../Alpha/HeifPlanarAlphaCompositor.cs | 173 ++++ .../Alpha/HeifPlanarAlphaResizeWorker.cs | 323 ++++++++ .../Components/Alpha/IHeifAlphaItemDecoder.cs | 35 + .../HeifPlanarColorConverter.cs | 39 +- ...ter.Samples.cs => HeifSampleConversion.cs} | 282 +++---- .../HeifYuv420ToRgb8Converter.Simd.cs | 19 +- .../ColorConverters/IHeifSampleConverter.cs | 167 ++++ .../Formats/Heif/GridHeifItemDecoder.cs | 344 +++++--- .../Formats/Heif/HeifDecoderCore.cs | 255 +++--- .../Formats/Heif/HeifEncoderCore.cs | 2 +- .../Heif/Hevc/Color/HevcYuvConverter.cs | 42 +- .../Formats/Heif/Hevc/HevcCabacContext.cs | 56 +- .../Formats/Heif/HevcHeifItemDecoder.cs | 155 +++- .../JpegColorConverter.Operator.cs | 1 - .../PixelOperations/L16.PixelOperations.cs | 162 +++- .../Codecs/Heif/Av1TransformBenchmarks.cs | 30 + .../Heif/Av1/Av1ForwardTransformTests.cs | 130 ++- .../Heif/Av1/Av1InverseQuantizationTests.cs | 2 +- .../Heif/Av1/Av1InverseTransformTests.cs | 154 +++- .../Formats/Heif/Av1/Av1YuvConverterTests.cs | 258 ++++++ .../Formats/Heif/HeifSequenceParserTests.cs | 52 +- 61 files changed, 6387 insertions(+), 589 deletions(-) rename src/ImageSharp/Formats/Heif/Av1/Pipeline/{Quantification => Quantizers}/Av1DeQuantizationContext.cs (98%) rename src/ImageSharp/Formats/Heif/Av1/Pipeline/{Quantification => Quantizers}/Av1InverseQuantizationLookup.cs (99%) rename src/ImageSharp/Formats/Heif/Av1/Pipeline/{Quantification => Quantizers}/Av1InverseQuantizer.cs (97%) rename src/ImageSharp/Formats/Heif/Av1/Pipeline/{Quantification => Quantizers}/Av1QuantizationLookup.cs (99%) create mode 100644 src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaCompositor.cs create mode 100644 src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaResizeWorker.cs create mode 100644 src/ImageSharp/Formats/Heif/Components/Alpha/IHeifAlphaItemDecoder.cs rename src/ImageSharp/Formats/Heif/Components/ColorConverters/{HeifColorConverter.Samples.cs => HeifSampleConversion.cs} (83%) create mode 100644 src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index a486b0134..29def16cc 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -78,12 +78,14 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Remove the shared mutable `Av1ForwardTransformer.TemporaryCoefficientsBuffer` and give each concurrent transform operation exclusive workspace ownership. - [x] Replace the instance-transformer interface with stateless forward and inverse operator structs following the JPEG color-transform static-generic operator pattern. - [x] Select transform type, size, bit depth, and ISA once at the 2-D block boundary rather than dispatching through an interface for every row and column. - - [x] Port the applicable libaom bulk forward-transform kernels using the existing ImageSharp `Vector128` and `Vector256` infrastructure while retaining the scalar oracle; add a managed `Vector512` path only if complete-block measurements justify extending beyond the pinned upstream SIMD implementations. + - [x] Port the applicable libaom bulk forward-transform kernels using the existing ImageSharp `Vector128` and `Vector256` infrastructure while retaining the scalar oracle. - [x] Port the applicable libaom bulk inverse-transform kernels using the same tables, rounding, saturation, and clipping rules as the scalar oracle. + - [x] Extend every forward and inverse DCT, ADST, identity, two-dimensional traversal, and 8/10/12-bit reconstruction operator with a managed `Vector512` path. Dispatch it first for blocks with complete sixteen-lane tiles and retain the same staged arithmetic as the scalar, `Vector128`, and `Vector256` operators. - [x] Document scratch ownership, stage-buffer alternation, fixed-point rounding, lane layout, transposition, and scalar fallback decisions at their implementation points. - - [x] Add complete-block BenchmarkDotNet coverage for scalar, `Vector128`, `Vector256`, and runtime-dispatched 8x8 and 32x32 forward/inverse DCT paths, including managed-allocation reporting. - - [x] Verify every scalar and hardware path across all transform types, sizes, bit depths, edge blocks, and bounded overflow cases, then record representative complete-block timings and allocations. + - [x] Add complete-block BenchmarkDotNet coverage for scalar, `Vector128`, `Vector256`, and runtime-dispatched 8x8 and 32x32 forward/inverse DCT paths, plus the applicable `Vector512` 32x32 paths, including managed-allocation reporting. + - [x] Verify the scalar, `Vector128`, and `Vector256` paths across all transform types, sizes, bit depths, edge blocks, and bounded overflow cases, then record representative complete-block timings and allocations. - Verification covers all 159 normative size/type combinations at 8, 10, and 12 bits with padded input, prediction, and destination strides. Each of the 477 configurations compares scalar and `Vector128` output, and every configuration that contains a complete eight-lane tile also compares `Vector256` output. The 989-test focused suite passes with normal AVX2 dispatch, with AVX2 disabled, and with all hardware intrinsics disabled. + - [ ] Execute the new `FeatureTestRunner` matrix for every sixteen-lane transform configuration with AVX-512 enabled and disabled, run the Release build, and record complete-block `Vector512` versus `Vector256` timings before treating the new tier as verified. - [ ] **Queued:** restore bounded animated HEIC and AVIF image-sequence scope, including the required image-level and per-frame metadata contracts, without introducing unrelated ISO BMFF surfaces. - [x] Reconcile the top-level still-image-only scope with the required animated HEIC and AVIF completion boundary before sequence implementation begins. - [x] Define the ImageSharp image-level sequence metadata and per-frame metadata contracts, including observable timing, repetition, color, alpha, orientation, and profile behavior. @@ -233,7 +235,7 @@ This snapshot pins or classifies the available references and failures; it does | `Av1FilmGrainDecoder` and `Av1FilmGrainGaussianSequence` | AV1 section 7.18 film-grain synthesis | libaom `av1/decoder/grain_synthesis.c`, `av1/decoder/grain_synthesis.h`, and `aom_dsp/grain_params.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative 2,048-sample Gaussian sequence, linear-feedback shift register, luma/chroma autoregressive templates, scaling lookup interpolation, 32x32 block selection, boundary overlap, restricted-range clipping, monochrome and 4:2:0/4:2:2/4:4:4 layouts, and 8/10/12-bit arithmetic. Use allocator-owned scratch and runtime-optimized span copies. Apply grain only to the displayed still-image samples after all in-loop filters; reference-frame parameter inheritance remains sequence-playback state and is outside this codec scope. | | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | | `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. | -| `Av1ForwardTransformer`, `Av1Inverse2dTransformer`, `Av1Transform2dFlipConfiguration`, the forward/inverse 1-D operator structs, and `Av1Transform1dMath` | AV1 forward transform definitions and section 7.11.2 inverse transform and reconstruction | libaom `av1/encoder/av1_fwd_txfm1d.c`, `av1/encoder/av1_fwd_txfm2d.c`, `av1/common/av1_inv_txfm1d_cfg.h`, `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, the x86 AVX2/SSE4 implementations, and the corresponding Neon implementations at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative staged DCT, ADST, and identity arithmetic, direction-specific stage ranges and shifts, transposition, clipping, and high-bit-depth sample addition. Stateless static-generic operators follow ImageSharp's JPEG color-transform pattern. `Vector128` and `Vector256` kernels retain wrapping 32-bit lane arithmetic before the bounded rounding shift, matching the optimized upstream implementations, with the scalar operators as the behavioral oracle. The pinned upstream transform paths do not provide an AVX-512 implementation. | +| `Av1ForwardTransformer`, `Av1Inverse2dTransformer`, `Av1Transform2dFlipConfiguration`, the forward/inverse 1-D operator structs, and `Av1Transform1dMath` | AV1 forward transform definitions and section 7.11.2 inverse transform and reconstruction | libaom `av1/encoder/av1_fwd_txfm1d.c`, `av1/encoder/av1_fwd_txfm2d.c`, `av1/common/av1_inv_txfm1d_cfg.h`, `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, the x86 AVX2/SSE4 implementations, and the corresponding Neon implementations at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve the normative staged DCT, ADST, and identity arithmetic, direction-specific stage ranges and shifts, transposition, clipping, and high-bit-depth sample addition. Stateless static-generic operators follow ImageSharp's JPEG color-transform pattern. `Vector128`, `Vector256`, and managed `Vector512` kernels retain the same wrapping 32-bit lane arithmetic before the bounded rounding shift, with the scalar operators as the behavioral oracle. The pinned upstream transform paths do not provide a complete AVX-512 forward, inverse, and high-bit-depth family, so the managed sixteen-lane tier must prove exact parity against the scalar path rather than claim upstream AVX-512 equivalence. | | `HeifDecoderCore` box extension handling and `HeifDecoderCore`/`HeifEncoderCore` item-property associations | ISO/IEC 14496-12 box extensibility and section 8.11.14 item properties and `ipma` syntax | libavif `src/read.c` and `src/write.c` at `092276ce89098ead06db80975173191e5fee1826` | Skip unrecognized top-level and metadata child boxes, preserve the position of every property in `ipco`, reject an unrecognized property only when its item association marks it essential, associate properties by item ID, and read or write the essential bit plus one-based 7-bit or 15-bit property index according to the full-box flags. Independent HEIC, HIF, and AVIF fixtures provide the reader oracle; container-level identification of encoded output guards the writer independently of pixel roundtripping. | | `HeifCleanAperture`, `HeifItem` presentation state, and `HeifDecoderCore` transformative-property parsing and application | ISO/IEC 14496-12 section 12.1.4 clean aperture; HEIF image rotation and mirror properties; MIAF section 7.3.6.7 presentation order and section 7.3.9 essential transformative properties | libavif `src/avif.c` clean-aperture conversion, `src/read.c` property parsers and alpha-property validation, and `apps/shared/avifutil.c` transform application at `092276ce89098ead06db80975173191e5fee1826` | Resolve fractional clean-aperture dimensions and center offsets to exact bounded integer pixels, validate the registered rotation/mirror reserved bits, require essential associations, crop after auxiliary-alpha composition, map counter-clockwise HEIF quarter turns to ImageSharp's optimized clockwise rotate modes, then mirror around the signaled axis. Reuse ImageSharp's existing crop, rotation, and flip processors for every pixel type. Retain only the three image-item property values; do not add a generic transform-box or ISO BMFF model. | | `HeifConstants.IsSupportedFileType`, `HeifImageFormatDetector`, and `HeifDecoderCore.CheckFileTypeBox` | ISO/IEC 14496-12 `FileTypeBox` syntax and the MP4 Registration Authority HEIF/AVIF still-image and sequence brand registrations | libavif `src/read.c` functions `avifParseFileTypeBox`, `avifFileTypeHasBrand`, and `avifFileTypeIsCompatible` at `092276ce89098ead06db80975173191e5fee1826` | Apply one rule to the major and compatible brands, accept implemented still-image brands and the bounded `avis`, `hevc`, and `hevx` image-sequence brands, and distinguish item and sequence presentation before parsing their payloads. Layered HEVC and JPEG sequence brands remain unsupported. The decoder validates the complete `ftyp` payload; the fixed-size format detector inspects the available prefix. | @@ -313,7 +315,7 @@ This assessment is based on the current source after the upstream ImageSharp mer - Loop-restoration unit parsing records tile-local switchable/Wiener/self-guided filter selections and coefficients in frame-owned plane grids, including super-resolution-adjusted unit corners and the corrected conditional 64x64-superblock unit-size bit. The active restoration stage implements the normative unit geometry, striped deblocked boundaries, Wiener filtering, self-guided projection, and 8/10/12-bit clipping, while reusing the existing SIMD-backed super-resolution and adjacent multiply/add primitives. Independently encoded fixtures covering every parameter set, plane layout, bit depth, and frame-edge geometry are still required. Other normative independently decodable still-image syntax paths still contain `NotImplementedException` or equivalent unsupported branches. Tile-local palette CDF adaptation is present; the remaining still-image frame-context behavior requires a separate source audit without introducing sequence playback state. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Chroma-from-luma storage, subsampling, parameter derivation, U/V sharing, and 8/10/12-bit prediction are active; independently encoded high-bit-depth and chroma-from-luma AVIF conformance files are still required. - `Av1YuvConverter` now consumes the signaled full or limited range, every non-reserved AV1 H.273 matrix coefficient, transfer characteristics where the matrix definition requires them, subsampling, and chroma sample position for 8, 10, and 12-bit output. Its high-bit-depth decode and encode paths use allocator-backed `Rgb48` rows and the existing `PixelOperations` conversions, avoiding the former eight-bit intermediate. Encoder conversion covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with libavif-compatible box averaging. Identity, full/limited-range YCgCo, the fixed non-constant-luminance matrices, both fixed and chromaticity-derived constant/non-constant-luminance systems, SMPTE ST 2085, and PQ/HLG ICtCp are active in both directions. Independent vectors for every matrix, transfer, range, bit depth, sampling layout, and chroma position remain required before the complete color matrix is externally verified. -- Forward and inverse transforms now use operation-owned allocator workspace and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. The two-dimensional traversal selects `Vector256`, then `Vector128`, with scalar only when hardware vectorization is unavailable. Focused tests prove exact scalar/SIMD parity for all 159 normative size/type combinations at every supported bit depth, padded edge strides, bounded residual extrema, the complete inverse shift/range tables, and zero per-block managed allocations. The 989-test suite passes with normal AVX2 dispatch, AVX2 disabled, and all hardware intrinsics disabled. On the measured .NET 10 AVX2 host, runtime-dispatched 8x8 DCT blocks take 102.42 ns forward and 112.14 ns inverse versus 441.19 ns and 602.52 ns scalar; 32x32 blocks take 1.439 microseconds forward and 8.825 microseconds inverse versus 9.112 microseconds and 23.455 microseconds scalar. BenchmarkDotNet reports no managed allocation for any measured path. A managed `Vector512` path remains deliberately absent: pinned libaom has an optional Highway AVX-512 low-bit-depth forward path, but no matching complete inverse and high-bit-depth transform family, and the complete-block results do not justify maintaining a separate partial algorithm. +- Forward and inverse transforms use operation-owned allocator workspace and stateless static-generic operator structs for every valid DCT, ADST, and identity size. Named configuration factories keep the encoder's three shifts and variable cosine precision separate from the decoder's two shifts, fixed 12-bit cosine precision, and 8/10/12-bit clamp ranges. The two-dimensional traversal now selects `Vector512`, then `Vector256`, then `Vector128`, with scalar only when hardware vectorization is unavailable. The managed sixteen-lane tier covers every one-dimensional operator, both two-dimensional traversals, byte reconstruction, and high-bit-depth reconstruction without per-block allocation. Its `FeatureTestRunner` parity matrix and permanent 32x32 benchmark cases are implemented but have not yet been executed. The existing verified baseline still covers all 159 normative size/type combinations at every supported bit depth with scalar, `Vector128`, and applicable `Vector256` paths; the 989-test suite passes with normal AVX2 dispatch, AVX2 disabled, and all hardware intrinsics disabled. On the measured .NET 10 AVX2 host, runtime-dispatched 8x8 DCT blocks take 102.42 ns forward and 112.14 ns inverse versus 441.19 ns and 602.52 ns scalar; 32x32 blocks take 1.439 microseconds forward and 8.825 microseconds inverse versus 9.112 microseconds and 23.455 microseconds scalar. BenchmarkDotNet reports no managed allocation for those measured paths. - The production prediction and nonlinear self-guided paths remain predominantly scalar. Transform traversal is SIMD-first on supported hardware, while normative super-resolution and Wiener horizontal products reuse ImageSharp's cross-platform adjacent multiply/add SIMD helper with exact scalar fallbacks; further SIMD work must preserve these scalar reconstruction oracles. ### AV1 encoder diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index 61e36fb21..bf75d19cd 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -112,6 +112,94 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable Av1CodecConfiguration? codecConfiguration, out CicpProfile effectiveColorProfile) where TPixel : unmanaged, IPixel + { + using Av1FrameBuffer frameBuffer = this.DecodeFrameBuffer( + buffer, + containerColorProfile, + codecConfiguration, + out effectiveColorProfile); + + ImageFrame? resultFrame = null; + try + { + resultFrame = new ImageFrame( + this.configuration, + this.FrameHeader!.FrameSize.SuperResolutionUpscaledWidth, + this.FrameHeader.FrameSize.FrameHeight); + + Av1YuvConverter.ConvertToRgb(this.configuration, frameBuffer, resultFrame); + resultFrame.Metadata.CicpProfile = effectiveColorProfile.DeepClone(); + return resultFrame; + } + catch + { + resultFrame?.Dispose(); + throw; + } + } + + /// + /// Decodes an AV1 elementary-stream payload and composes its luma plane directly into a packed color frame. + /// + /// The destination color pixel type. + /// The complete AV1 elementary-stream payload. + /// + /// The container color description that supplies unspecified sequence-header color information. + /// + /// The AV1 codec configuration validated against the coded sequence header. + /// The required coded dimensions, or an empty size when the item extent may differ. + /// The packed color frame receiving alpha values. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the top-left portion of the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + public void DecodeAlpha( + Span buffer, + CicpProfile? containerColorProfile, + Av1CodecConfiguration? codecConfiguration, + Size expectedCodedSize, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel + { + using Av1FrameBuffer frameBuffer = this.DecodeFrameBuffer(buffer, containerColorProfile, codecConfiguration, out _); + if (expectedCodedSize != default && (frameBuffer.Width != expectedCodedSize.Width || frameBuffer.Height != expectedCodedSize.Height)) + { + throw new InvalidImageContentException("The decoded alpha sample dimensions do not match its visual sample entry."); + } + + if (frameBuffer.ColorFormat != Av1ColorFormat.Yuv400) + { + // AVIF auxiliary alpha is the luma plane of an AV1 monochrome image. Accepting chroma-bearing payloads + // would silently reinterpret a color image and contradict the Sequence Header mono_chrome requirement. + throw new InvalidImageContentException("An AV1 auxiliary alpha image must be encoded as monochrome."); + } + + Av1YuvConverter.ComposeAlpha( + this.configuration, + frameBuffer, + destination, + outputSize, + destinationRectangle, + premultiplied); + } + + /// + /// Parses and reconstructs one AV1 frame while retaining its native component planes for the caller. + /// + /// The complete AV1 elementary-stream payload. + /// + /// The container color description that supplies unspecified sequence-header color information. + /// + /// The AV1 codec configuration validated against the coded sequence header. + /// Receives the effective CICP description associated with the native planes. + /// The reconstructed native frame buffer. Ownership transfers to the caller. + private Av1FrameBuffer DecodeFrameBuffer( + Span buffer, + CicpProfile? containerColorProfile, + Av1CodecConfiguration? codecConfiguration, + out CicpProfile effectiveColorProfile) { Av1BitStreamReader reader = new(buffer); this.obuReader.ReadAll(ref reader, buffer.Length, () => this, false); @@ -166,24 +254,16 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable } this.FrameInfo = this.tileReader.FrameInfo; - using Av1FrameBuffer frameBuffer = new( + Av1FrameBuffer frameBuffer = new( this.configuration, this.SequenceHeader, this.SequenceHeader.ColorConfig.GetColorFormat(), false); - using Av1FrameDecoder frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, frameBuffer); - frameDecoder.DecodeFrame(); - - ImageFrame? resultFrame = null; try { - resultFrame = new ImageFrame( - this.configuration, - this.FrameHeader.FrameSize.SuperResolutionUpscaledWidth, - this.FrameHeader.FrameSize.FrameHeight); - - Av1YuvConverter.ConvertToRgb(this.configuration, frameBuffer, resultFrame); + using Av1FrameDecoder frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, frameBuffer); + frameDecoder.DecodeFrame(); // Preserve the effective CICP description used for conversion, including container values that legally // supplied unspecified bitstream fields. This also exposes bitstream-only color metadata to callers. @@ -194,12 +274,11 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable (byte)effectiveColorConfig.MatrixCoefficients, effectiveColorConfig.ColorRange); - resultFrame.Metadata.CicpProfile = effectiveColorProfile.DeepClone(); - return resultFrame; + return frameBuffer; } catch { - resultFrame?.Dispose(); + frameBuffer.Dispose(); throw; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs index 02bc725bf..67ef7ec76 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs @@ -3,9 +3,9 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Components; +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; -using static SixLabors.ImageSharp.Formats.Heif.Components.HeifColorConverterBase; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Color; @@ -28,7 +28,7 @@ internal static class Av1YuvConverter if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { Av1PlanarSampleBuffer buffer = new(frameBuffer); - HeifPlanarColorConverter.ConvertToRgb, byte, HeifByteSampleLoader>( + HeifPlanarColorConverter.ConvertToRgb, byte, HeifByteSampleConverter>( configuration, buffer, image, @@ -47,6 +47,55 @@ internal static class Av1YuvConverter mode); } + /// + /// Composes the reconstructed luma plane into a packed color frame as auxiliary alpha. + /// + /// The destination color pixel type. + /// The configuration used for allocation and pixel conversion. + /// The reconstructed AV1 frame containing the alpha luma plane. + /// The packed color frame receiving alpha values. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the top-left portion of the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + public static void ComposeAlpha( + Configuration configuration, + Av1FrameBuffer frameBuffer, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel + { + HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer, out _); + Rectangle sourceRectangle = new(0, 0, frameBuffer.Width, frameBuffer.Height); + if (frameBuffer.BitDepth == Av1BitDepth.EightBit) + { + Av1PlanarSampleBuffer buffer = new(frameBuffer); + HeifPlanarAlphaCompositor.Compose, byte, HeifByteSampleConverter>( + configuration, + buffer, + destination, + in parameters, + sourceRectangle, + outputSize, + destinationRectangle, + premultiplied); + + return; + } + + Av1PlanarSampleBuffer highBitDepthBuffer = new(frameBuffer); + HeifPlanarAlphaCompositor.Compose, ushort, HeifUShortSampleConverter>( + configuration, + highBitDepthBuffer, + destination, + in parameters, + sourceRectangle, + outputSize, + destinationRectangle, + premultiplied); + } + /// /// Converts packed pixels to the configured monochrome or component planes used by the AV1 encoder. /// @@ -61,7 +110,7 @@ internal static class Av1YuvConverter if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { Av1PlanarSampleBuffer buffer = new(frameBuffer); - HeifPlanarColorConverter.ConvertFromRgb, byte, HeifByteSampleStorer>( + HeifPlanarColorConverter.ConvertFromRgb, byte, HeifByteSampleConverter>( configuration, image, buffer, @@ -72,7 +121,7 @@ internal static class Av1YuvConverter } Av1PlanarSampleBuffer highBitDepthBuffer = new(frameBuffer); - HeifPlanarColorConverter.ConvertFromRgb, ushort, HeifUShortSampleStorer>( + HeifPlanarColorConverter.ConvertFromRgb, ushort, HeifUShortSampleConverter>( configuration, image, highBitDepthBuffer, diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index c218174dd..56411f41e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -1,7 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs index 0a27fad92..5d943d8e9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs @@ -1,7 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs index 30d59fe02..69d366064 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs @@ -6,7 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration; -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.SuperResolution; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1DeQuantizationContext.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1DeQuantizationContext.cs similarity index 98% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1DeQuantizationContext.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1DeQuantizationContext.cs index d5c91dc37..408e66ef9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1DeQuantizationContext.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1DeQuantizationContext.cs @@ -3,7 +3,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; /// /// Stores the AV1 DC and AC dequantization values for every segment and color plane in a frame. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizationLookup.cs similarity index 99% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizationLookup.cs index b48f9ca12..f91692f24 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizationLookup.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizationLookup.cs @@ -4,7 +4,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; /// /// Provides the normative AV1 inverse quantization matrices for each matrix level, plane class, and transform size. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizer.cs similarity index 97% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizer.cs index a318f5389..d7a6957a2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1InverseQuantizer.cs @@ -5,7 +5,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; /// /// Reconstructs AV1 transform coefficients from quantized coefficient levels. @@ -101,8 +101,8 @@ internal class Av1InverseQuantizer ? Av1ScanOrderConstants.QuantizationMatrixLevelCount - 1 : this.frameHeader.SegmentationParameters.QMLevel[(int)plane][mode.SegmentId]; - ReadOnlySpan iqMatrix = (transformType.ToClass() == Av1TransformClass.Class2D) ? - Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, transformSize) + ReadOnlySpan iqMatrix = (transformType.ToClass() == Av1TransformClass.Class2D) + ? Av1InverseQuantizationLookup.GetQuantizationMatrix(qmLevel, plane, transformSize) : Av1InverseQuantizationLookup.GetQuantizationMatrix(Av1Constants.QuantificationMatrixLevelCount - 1, Av1Plane.Y, transformSize); int shift = transformSize.GetScale(); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1QuantizationLookup.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1QuantizationLookup.cs similarity index 99% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1QuantizationLookup.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1QuantizationLookup.cs index e5bf28aaf..7d5170324 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1QuantizationLookup.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantizers/Av1QuantizationLookup.cs @@ -3,7 +3,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; /// /// Provides the normative AV1 DC and AC dequantization values for each quantizer index and supported bit depth. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index fc4450ca7..f5603acbd 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -5,7 +5,7 @@ using System.Buffers; using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs index 2a51b57c2..611fa24b0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs @@ -46,4 +46,22 @@ internal readonly struct Av1ByteInverseTransformOutputOperator : IAv1InverseTran Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt64().ToScalar()); _ = bitDepth; } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref byte prediction, ref byte destination, Vector512 residual, int bitDepth) + { + // Residuals occupy signed 32-bit lanes, so widen the sixteen packed predictions before adding. The clamp + // then guarantees that both narrowing steps preserve the reconstructed byte values exactly. + Vector128 packed = Vector128.LoadUnsafe(ref prediction); + (Vector128 predicted16Lower, Vector128 predicted16Upper) = Vector128.Widen(packed); + Vector256 predicted32Lower = Vector256.Create(Vector128.WidenLower(predicted16Lower), Vector128.WidenUpper(predicted16Lower)).AsInt32(); + Vector256 predicted32Upper = Vector256.Create(Vector128.WidenLower(predicted16Upper), Vector128.WidenUpper(predicted16Upper)).AsInt32(); + Vector512 predicted32 = Vector512.Create(predicted32Lower, predicted32Upper); + Vector512 reconstructed = Vector512.Clamp(predicted32 + residual, Vector512.Zero, Vector512.Create((int)byte.MaxValue)); + Vector256 reconstructed16 = Vector256.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32()); + Vector128 reconstructed8 = Vector128.Narrow(reconstructed16.GetLower(), reconstructed16.GetUpper()); + reconstructed8.StoreUnsafe(ref destination); + _ = bitDepth; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs index cb2019f3b..fcdca7c90 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs @@ -159,6 +159,12 @@ internal static class Av1ForwardTransformer int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); + if (Vector512.IsHardwareAccelerated && width >= Vector512.Count && height >= Vector512.Count) + { + Transform2dVector512(input, coefficients, stride, ref config, workspace); + return; + } + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count && height >= Vector256.Count) { Transform2dVector256(input, coefficients, stride, ref config, workspace); @@ -174,6 +180,153 @@ internal static class Av1ForwardTransformer Transform2dScalar(input, coefficients, stride, ref config, workspace); } + /// + /// Applies both transform axes with sixteen samples packed into each SIMD vector. + /// + /// The one-dimensional operator applied down each column. + /// The one-dimensional operator applied across each row. + /// The spatial residual samples. + /// The destination transform coefficients. + /// The number of input samples between rows. + /// The transform dimensions, operators, flips, and fixed-point settings. + /// The reusable storage for SIMD vectors and transposed coefficients. + public static void Transform2dVector512( + Span input, + Span output, + uint inputStride, + ref Av1Transform2dFlipConfiguration config, + Span workspace) + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator + { + const int laneCount = 16; + const int vectorLength = Av1Constants.MaxTransformSize * laneCount; + + int width = config.TransformSize.GetWidth(); + int height = config.TransformSize.GetHeight(); + int shift0 = config.Shift0; + int shift1 = config.Shift1; + int shift2 = config.Shift2; + bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; + + ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); + ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); + ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); + ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); + Span buffer = workspace.Slice(Av1TransformWorkspace.Vector512StorageLength, width * height); + ref short inputBase = ref MemoryMarshal.GetReference(input); + ref int bufferBase = ref MemoryMarshal.GetReference(buffer); + + // Each lane carries one complete column through every stage of the first transform axis. + for (int column = 0; column < width; column += laneCount) + { + for (int row = 0; row < height; row++) + { + int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; + ref short source = ref Unsafe.Add(ref inputBase, (sourceRow * (int)inputStride) + column); + tempIn[row] = Av1Transform2dOperations.RoundShift(Av1Transform2dOperations.Load16Int16(ref source), -shift0); + } + + TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); + int destinationColumn = config.FlipLeftToRight ? width - column - laneCount : column; + + for (int row = 0; row < height; row++) + { + Vector512 value = Av1Transform2dOperations.RoundShift(tempOut[row], -shift1); + value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + value.StoreUnsafe(ref bufferBase, (nuint)((row * width) + destinationColumn)); + } + } + + ref int outputBase = ref MemoryMarshal.GetReference(output); + + // Tile transposition changes the lane meaning from columns to rows without scalar gathers. + for (int row = 0; row < height; row += laneCount) + { + for (int column = 0; column < width; column += laneCount) + { + Vector512 row0 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); + Vector512 row1 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); + Vector512 row2 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); + Vector512 row3 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); + Vector512 row4 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 4) * width) + column)); + Vector512 row5 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 5) * width) + column)); + Vector512 row6 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 6) * width) + column)); + Vector512 row7 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 7) * width) + column)); + Vector512 row8 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 8) * width) + column)); + Vector512 row9 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 9) * width) + column)); + Vector512 row10 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 10) * width) + column)); + Vector512 row11 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 11) * width) + column)); + Vector512 row12 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 12) * width) + column)); + Vector512 row13 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 13) * width) + column)); + Vector512 row14 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 14) * width) + column)); + Vector512 row15 = Vector512.LoadUnsafe(ref bufferBase, (nuint)(((row + 15) * width) + column)); + Av1Transform2dOperations.Transpose( + ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, + ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); + + tempIn[column + 0] = row0; + tempIn[column + 1] = row1; + tempIn[column + 2] = row2; + tempIn[column + 3] = row3; + tempIn[column + 4] = row4; + tempIn[column + 5] = row5; + tempIn[column + 6] = row6; + tempIn[column + 7] = row7; + tempIn[column + 8] = row8; + tempIn[column + 9] = row9; + tempIn[column + 10] = row10; + tempIn[column + 11] = row11; + tempIn[column + 12] = row12; + tempIn[column + 13] = row13; + tempIn[column + 14] = row14; + tempIn[column + 15] = row15; + } + + TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); + + for (int column = 0; column < width; column += laneCount) + { + Vector512 row0 = FinishForward(tempOut[column + 0], -shift2, normalizeRectangle); + Vector512 row1 = FinishForward(tempOut[column + 1], -shift2, normalizeRectangle); + Vector512 row2 = FinishForward(tempOut[column + 2], -shift2, normalizeRectangle); + Vector512 row3 = FinishForward(tempOut[column + 3], -shift2, normalizeRectangle); + Vector512 row4 = FinishForward(tempOut[column + 4], -shift2, normalizeRectangle); + Vector512 row5 = FinishForward(tempOut[column + 5], -shift2, normalizeRectangle); + Vector512 row6 = FinishForward(tempOut[column + 6], -shift2, normalizeRectangle); + Vector512 row7 = FinishForward(tempOut[column + 7], -shift2, normalizeRectangle); + Vector512 row8 = FinishForward(tempOut[column + 8], -shift2, normalizeRectangle); + Vector512 row9 = FinishForward(tempOut[column + 9], -shift2, normalizeRectangle); + Vector512 row10 = FinishForward(tempOut[column + 10], -shift2, normalizeRectangle); + Vector512 row11 = FinishForward(tempOut[column + 11], -shift2, normalizeRectangle); + Vector512 row12 = FinishForward(tempOut[column + 12], -shift2, normalizeRectangle); + Vector512 row13 = FinishForward(tempOut[column + 13], -shift2, normalizeRectangle); + Vector512 row14 = FinishForward(tempOut[column + 14], -shift2, normalizeRectangle); + Vector512 row15 = FinishForward(tempOut[column + 15], -shift2, normalizeRectangle); + Av1Transform2dOperations.Transpose( + ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, + ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); + + row0.StoreUnsafe(ref outputBase, (nuint)(((row + 0) * width) + column)); + row1.StoreUnsafe(ref outputBase, (nuint)(((row + 1) * width) + column)); + row2.StoreUnsafe(ref outputBase, (nuint)(((row + 2) * width) + column)); + row3.StoreUnsafe(ref outputBase, (nuint)(((row + 3) * width) + column)); + row4.StoreUnsafe(ref outputBase, (nuint)(((row + 4) * width) + column)); + row5.StoreUnsafe(ref outputBase, (nuint)(((row + 5) * width) + column)); + row6.StoreUnsafe(ref outputBase, (nuint)(((row + 6) * width) + column)); + row7.StoreUnsafe(ref outputBase, (nuint)(((row + 7) * width) + column)); + row8.StoreUnsafe(ref outputBase, (nuint)(((row + 8) * width) + column)); + row9.StoreUnsafe(ref outputBase, (nuint)(((row + 9) * width) + column)); + row10.StoreUnsafe(ref outputBase, (nuint)(((row + 10) * width) + column)); + row11.StoreUnsafe(ref outputBase, (nuint)(((row + 11) * width) + column)); + row12.StoreUnsafe(ref outputBase, (nuint)(((row + 12) * width) + column)); + row13.StoreUnsafe(ref outputBase, (nuint)(((row + 13) * width) + column)); + row14.StoreUnsafe(ref outputBase, (nuint)(((row + 14) * width) + column)); + row15.StoreUnsafe(ref outputBase, (nuint)(((row + 15) * width) + column)); + } + } + } + /// /// Applies both transform axes with eight samples packed into each SIMD vector. /// @@ -467,4 +620,16 @@ internal static class Av1ForwardTransformer ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) : value; } + + /// + /// Applies the terminal shift and optional rectangular normalization to sixteen coefficients. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 FinishForward(Vector512 value, int shift, bool normalizeRectangle) + { + value = Av1Transform2dOperations.RoundShift(value, shift); + return normalizeRectangle + ? Av1Transform1dMath.MultiplyRound(value, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits) + : value; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs index d17113392..016bc717a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs @@ -37,4 +37,18 @@ internal readonly struct Av1HighBitDepthInverseTransformOutputOperator : IAv1Inv Vector128 narrowed = Vector128.Narrow(reconstructed.GetLower(), reconstructed.GetUpper()); narrowed.StoreUnsafe(ref destination); } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Add(ref short prediction, ref short destination, Vector512 residual, int bitDepth) + { + // AV1 high-bit-depth samples are nonnegative Int16 values. Widening before the residual add preserves signed + // arithmetic, and the bit-depth clamp makes the final narrowing exact for both 10-bit and 12-bit output. + Vector256 packed = Vector256.LoadUnsafe(ref prediction); + (Vector256 predictedLower, Vector256 predictedUpper) = Vector256.Widen(packed); + Vector512 predicted = Vector512.Create(predictedLower, predictedUpper); + Vector512 reconstructed = Vector512.Clamp(predicted + residual, Vector512.Zero, Vector512.Create((1 << bitDepth) - 1)); + Vector256 narrowed = Vector256.Narrow(reconstructed.GetLower(), reconstructed.GetUpper()); + narrowed.StoreUnsafe(ref destination); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs index e4880757c..aad957b91 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs @@ -64,4 +64,31 @@ internal static class Av1IdentityTransform1d output[i] = Av1Transform1dMath.MultiplyRound(input[i], multiplier, fractionalBits); } } + + /// + /// Scales sixteen independent identity-transform axes in parallel. + /// + /// The source values for sixteen transform axes. + /// The destination values for sixteen transform axes. + /// The number of values in each axis. + /// The fixed-point identity scale. + /// The number of fractional bits in . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Transform(ref Av1TransformVector> input, ref Av1TransformVector> output, int length, int multiplier, int fractionalBits) + { + if (fractionalBits == 0) + { + for (int i = 0; i < length; i++) + { + output[i] = input[i] * multiplier; + } + + return; + } + + for (int i = 0; i < length; i++) + { + output[i] = Av1Transform1dMath.MultiplyRound(input[i], multiplier, fractionalBits); + } + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs index 08e4fdb53..ad17cf556 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs @@ -257,6 +257,14 @@ internal static class Av1Inverse2dTransformer int width = config.TransformSize.GetWidth(); int height = config.TransformSize.GetHeight(); + if (Vector512.IsHardwareAccelerated && width >= Vector512.Count && height >= Vector512.Count) + { + Transform2dVector512( + input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); + + return; + } + if (Vector256.IsHardwareAccelerated && width >= Vector256.Count && height >= Vector256.Count) { Transform2dVector256( @@ -277,6 +285,168 @@ internal static class Av1Inverse2dTransformer input, outputForRead, strideForRead, outputForWrite, strideForWrite, ref config, workspace, bitDepth); } + /// + /// Applies both inverse-transform axes with sixteen samples packed into each SIMD vector. + /// + /// The reconstructed sample storage type. + /// The operator that adds and clips inverse residuals. + /// The one-dimensional operator applied down each column. + /// The one-dimensional operator applied across each row. + /// The dequantized transform coefficients. + /// The prediction samples. + /// The number of prediction samples between rows. + /// The destination reconstruction samples. + /// The number of destination samples between rows. + /// The transform dimensions, operators, flips, and fixed-point settings. + /// The reusable storage for SIMD vectors and transposed coefficients. + /// The coded sample bit depth used to clamp reconstructed values. + public static void Transform2dVector512( + Span input, + Span outputForRead, + int strideForRead, + Span outputForWrite, + int strideForWrite, + ref Av1Transform2dFlipConfiguration config, + Span workspace, + int bitDepth) + where TSample : unmanaged + where TOutputOperator : struct, IAv1InverseTransformOutputOperator + where TColumnOperator : struct, IAv1Transform1dOperator + where TRowOperator : struct, IAv1Transform1dOperator + { + const int laneCount = 16; + const int vectorLength = Av1Constants.MaxTransformSize * laneCount; + + int width = config.TransformSize.GetWidth(); + int height = config.TransformSize.GetHeight(); + int shift0 = config.Shift0; + int shift1 = config.Shift1; + bool normalizeRectangle = Math.Abs(config.TransformSize.GetRectangleLogRatio()) == 1; + byte rowClampBits = (byte)(bitDepth + 8); + byte columnClampBits = (byte)Math.Max(bitDepth + 6, 16); + + ref int workspaceBase = ref MemoryMarshal.GetReference(workspace); + ref Av1TransformVector> tempIn = ref Unsafe.As>>(ref workspaceBase); + ref Av1TransformVector> tempOut = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, vectorLength)); + ref Av1TransformVector> step = ref Unsafe.As>>(ref Unsafe.Add(ref workspaceBase, 2 * vectorLength)); + Span buffer = workspace.Slice(Av1TransformWorkspace.Vector512StorageLength, width * height); + ref int inputBase = ref MemoryMarshal.GetReference(input); + ref int bufferBase = ref MemoryMarshal.GetReference(buffer); + + // Rows are transposed into lanes so the complete 1-D operator runs once for sixteen rows. + for (int row = 0; row < height; row += laneCount) + { + for (int column = 0; column < width; column += laneCount) + { + Vector512 row0 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 0) * width) + column)); + Vector512 row1 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 1) * width) + column)); + Vector512 row2 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 2) * width) + column)); + Vector512 row3 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 3) * width) + column)); + Vector512 row4 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 4) * width) + column)); + Vector512 row5 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 5) * width) + column)); + Vector512 row6 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 6) * width) + column)); + Vector512 row7 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 7) * width) + column)); + Vector512 row8 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 8) * width) + column)); + Vector512 row9 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 9) * width) + column)); + Vector512 row10 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 10) * width) + column)); + Vector512 row11 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 11) * width) + column)); + Vector512 row12 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 12) * width) + column)); + Vector512 row13 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 13) * width) + column)); + Vector512 row14 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 14) * width) + column)); + Vector512 row15 = Vector512.LoadUnsafe(ref inputBase, (nuint)(((row + 15) * width) + column)); + Av1Transform2dOperations.Transpose( + ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, + ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); + + tempIn[column + 0] = PrepareInverseRow(row0, normalizeRectangle, rowClampBits); + tempIn[column + 1] = PrepareInverseRow(row1, normalizeRectangle, rowClampBits); + tempIn[column + 2] = PrepareInverseRow(row2, normalizeRectangle, rowClampBits); + tempIn[column + 3] = PrepareInverseRow(row3, normalizeRectangle, rowClampBits); + tempIn[column + 4] = PrepareInverseRow(row4, normalizeRectangle, rowClampBits); + tempIn[column + 5] = PrepareInverseRow(row5, normalizeRectangle, rowClampBits); + tempIn[column + 6] = PrepareInverseRow(row6, normalizeRectangle, rowClampBits); + tempIn[column + 7] = PrepareInverseRow(row7, normalizeRectangle, rowClampBits); + tempIn[column + 8] = PrepareInverseRow(row8, normalizeRectangle, rowClampBits); + tempIn[column + 9] = PrepareInverseRow(row9, normalizeRectangle, rowClampBits); + tempIn[column + 10] = PrepareInverseRow(row10, normalizeRectangle, rowClampBits); + tempIn[column + 11] = PrepareInverseRow(row11, normalizeRectangle, rowClampBits); + tempIn[column + 12] = PrepareInverseRow(row12, normalizeRectangle, rowClampBits); + tempIn[column + 13] = PrepareInverseRow(row13, normalizeRectangle, rowClampBits); + tempIn[column + 14] = PrepareInverseRow(row14, normalizeRectangle, rowClampBits); + tempIn[column + 15] = PrepareInverseRow(row15, normalizeRectangle, rowClampBits); + } + + TRowOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitRow, config.StageRangeRow); + + for (int column = 0; column < width; column += laneCount) + { + Vector512 row0 = Av1Transform2dOperations.RoundShift(tempOut[column + 0], -shift0); + Vector512 row1 = Av1Transform2dOperations.RoundShift(tempOut[column + 1], -shift0); + Vector512 row2 = Av1Transform2dOperations.RoundShift(tempOut[column + 2], -shift0); + Vector512 row3 = Av1Transform2dOperations.RoundShift(tempOut[column + 3], -shift0); + Vector512 row4 = Av1Transform2dOperations.RoundShift(tempOut[column + 4], -shift0); + Vector512 row5 = Av1Transform2dOperations.RoundShift(tempOut[column + 5], -shift0); + Vector512 row6 = Av1Transform2dOperations.RoundShift(tempOut[column + 6], -shift0); + Vector512 row7 = Av1Transform2dOperations.RoundShift(tempOut[column + 7], -shift0); + Vector512 row8 = Av1Transform2dOperations.RoundShift(tempOut[column + 8], -shift0); + Vector512 row9 = Av1Transform2dOperations.RoundShift(tempOut[column + 9], -shift0); + Vector512 row10 = Av1Transform2dOperations.RoundShift(tempOut[column + 10], -shift0); + Vector512 row11 = Av1Transform2dOperations.RoundShift(tempOut[column + 11], -shift0); + Vector512 row12 = Av1Transform2dOperations.RoundShift(tempOut[column + 12], -shift0); + Vector512 row13 = Av1Transform2dOperations.RoundShift(tempOut[column + 13], -shift0); + Vector512 row14 = Av1Transform2dOperations.RoundShift(tempOut[column + 14], -shift0); + Vector512 row15 = Av1Transform2dOperations.RoundShift(tempOut[column + 15], -shift0); + Av1Transform2dOperations.Transpose( + ref row0, ref row1, ref row2, ref row3, ref row4, ref row5, ref row6, ref row7, + ref row8, ref row9, ref row10, ref row11, ref row12, ref row13, ref row14, ref row15); + + row0.StoreUnsafe(ref bufferBase, (nuint)(((row + 0) * width) + column)); + row1.StoreUnsafe(ref bufferBase, (nuint)(((row + 1) * width) + column)); + row2.StoreUnsafe(ref bufferBase, (nuint)(((row + 2) * width) + column)); + row3.StoreUnsafe(ref bufferBase, (nuint)(((row + 3) * width) + column)); + row4.StoreUnsafe(ref bufferBase, (nuint)(((row + 4) * width) + column)); + row5.StoreUnsafe(ref bufferBase, (nuint)(((row + 5) * width) + column)); + row6.StoreUnsafe(ref bufferBase, (nuint)(((row + 6) * width) + column)); + row7.StoreUnsafe(ref bufferBase, (nuint)(((row + 7) * width) + column)); + row8.StoreUnsafe(ref bufferBase, (nuint)(((row + 8) * width) + column)); + row9.StoreUnsafe(ref bufferBase, (nuint)(((row + 9) * width) + column)); + row10.StoreUnsafe(ref bufferBase, (nuint)(((row + 10) * width) + column)); + row11.StoreUnsafe(ref bufferBase, (nuint)(((row + 11) * width) + column)); + row12.StoreUnsafe(ref bufferBase, (nuint)(((row + 12) * width) + column)); + row13.StoreUnsafe(ref bufferBase, (nuint)(((row + 13) * width) + column)); + row14.StoreUnsafe(ref bufferBase, (nuint)(((row + 14) * width) + column)); + row15.StoreUnsafe(ref bufferBase, (nuint)(((row + 15) * width) + column)); + } + } + + ref TSample readBase = ref MemoryMarshal.GetReference(outputForRead); + ref TSample writeBase = ref MemoryMarshal.GetReference(outputForWrite); + + // The intermediate rows already contain contiguous column groups, avoiding a second transpose. + for (int column = 0; column < width; column += laneCount) + { + int sourceColumn = config.FlipLeftToRight ? width - column - laneCount : column; + + for (int row = 0; row < height; row++) + { + Vector512 value = Vector512.LoadUnsafe(ref bufferBase, (nuint)((row * width) + sourceColumn)); + value = config.FlipLeftToRight ? Av1Transform2dOperations.Reverse(value) : value; + tempIn[row] = Av1Transform1dMath.Clamp(value, columnClampBits); + } + + TColumnOperator.Transform(ref tempIn, ref tempOut, ref step, config.CosBitColumn, config.StageRangeColumn); + + for (int row = 0; row < height; row++) + { + int sourceRow = config.FlipUpsideDown ? height - row - 1 : row; + Vector512 residual = Av1Transform2dOperations.RoundShift(tempOut[sourceRow], -shift1); + ref TSample prediction = ref Unsafe.Add(ref readBase, (row * strideForRead) + column); + ref TSample destination = ref Unsafe.Add(ref writeBase, (row * strideForWrite) + column); + TOutputOperator.Add(ref prediction, ref destination, residual, bitDepth); + } + } + } + /// /// Applies both inverse-transform axes with eight samples packed into each SIMD vector. /// @@ -614,4 +784,17 @@ internal static class Av1Inverse2dTransformer return Av1Transform1dMath.Clamp(value, clampBits); } + + /// + /// Applies rectangular normalization and the row-input clamp to sixteen coefficient lanes. + /// + private static Vector512 PrepareInverseRow(Vector512 value, bool normalizeRectangle, byte clampBits) + { + if (normalizeRectangle) + { + value = Av1Transform1dMath.MultiplyRound(value, Av1InverseTransformMath.NewInverseSqrt2, Av1InverseTransformMath.NewSqrt2BitCount); + } + + return Av1Transform1dMath.Clamp(value, clampBits); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs index a79e8fb36..d1ea5f9db 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs @@ -89,6 +89,24 @@ internal static class Av1Transform1dMath return (weightedSum + Vector256.Create(1 << (cosBit - 1))) >> cosBit; } + /// + /// Calculates sixteen outputs of a rounded, weighted two-input butterfly in parallel. + /// + /// The first fixed-point weight. + /// The first sixteen input values. + /// The second fixed-point weight. + /// The second sixteen input values. + /// The number of fractional bits in each weight. + /// The sixteen rounded fixed-point results. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 HalfButterfly(int weight0, Vector512 input0, int weight1, Vector512 input1, int cosBit) + { + // AV1 stage ranges keep the products and sum inside the normative wrapping Int32 domain. Preserving that lane + // width lets 512-bit SIMD evaluate sixteen independent transform axes without widened intermediate vectors. + Vector512 weightedSum = (input0 * weight0) + (input1 * weight1); + return (weightedSum + Vector512.Create(1 << (cosBit - 1))) >> cosBit; + } + /// /// Clamps four transform-stage values to the signed range represented by a bit count. /// @@ -117,6 +135,20 @@ internal static class Av1Transform1dMath return Vector256.Clamp(value, Vector256.Create(minimum), Vector256.Create(maximum)); } + /// + /// Clamps sixteen transform-stage values to the signed range represented by a bit count. + /// + /// The sixteen transform-stage values. + /// The width of the signed range. + /// The values clamped to the permitted stage range. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Clamp(Vector512 value, byte bitCount) + { + int maximum = (1 << (bitCount - 1)) - 1; + int minimum = -(1 << (bitCount - 1)); + return Vector512.Clamp(value, Vector512.Create(minimum), Vector512.Create(maximum)); + } + /// /// Multiplies and rounds four fixed-point values in parallel. /// @@ -139,6 +171,17 @@ internal static class Av1Transform1dMath public static Vector256 MultiplyRound(Vector256 value, int multiplier, int fractionalBits) => HalfButterfly(multiplier, value, 0, Vector256.Zero, fractionalBits); + /// + /// Multiplies and rounds sixteen fixed-point values in parallel. + /// + /// The sixteen values to scale. + /// The fixed-point multiplier. + /// The number of fractional bits in the multiplier. + /// The sixteen rounded results. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 MultiplyRound(Vector512 value, int multiplier, int fractionalBits) + => HalfButterfly(multiplier, value, 0, Vector512.Zero, fractionalBits); + /// /// Multiplies four scalar inputs by fixed-point weights and rounds their sum. /// @@ -218,4 +261,33 @@ internal static class Av1Transform1dMath Vector256 weightedSum = (input0 * weight0) + (input1 * weight1) + (input2 * weight2) + (input3 * weight3); return (weightedSum + Vector256.Create(1 << (fractionalBits - 1))) >> fractionalBits; } + + /// + /// Multiplies four sets of sixteen inputs by fixed-point weights and rounds their sums. + /// + /// The first fixed-point weight. + /// The first sixteen input values. + /// The second fixed-point weight. + /// The second sixteen input values. + /// The third fixed-point weight. + /// The third sixteen input values. + /// The fourth fixed-point weight. + /// The fourth sixteen input values. + /// The number of fractional bits in each weight. + /// The sixteen rounded fixed-point sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 MultiplyAdd4( + int weight0, + Vector512 input0, + int weight1, + Vector512 input1, + int weight2, + Vector512 input2, + int weight3, + Vector512 input3, + int fractionalBits) + { + Vector512 weightedSum = (input0 * weight0) + (input1 * weight1) + (input2 * weight2) + (input3 * weight3); + return (weightedSum + Vector512.Create(1 << (fractionalBits - 1))) >> fractionalBits; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs index 09577c9d5..d2de5398b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs @@ -33,6 +33,19 @@ internal static class Av1Transform2dOperations public static Vector256 Load8Int16(ref short source) => Vector256_.Widen(Vector128.LoadUnsafe(ref source)); + /// + /// Loads sixteen signed sixteen-bit values and widens them to sixteen signed thirty-two-bit lanes. + /// + /// The first source value. + /// The sixteen widened values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Load16Int16(ref short source) + { + (Vector256 lower, Vector256 upper) = Vector256.Widen(Vector256.LoadUnsafe(ref source)); + + return Vector512.Create(lower, upper); + } + /// /// Applies a signed AV1 pipeline shift to four values in parallel. /// @@ -67,6 +80,23 @@ internal static class Av1Transform2dOperations return bit < 0 ? value << -bit : value; } + /// + /// Applies a signed AV1 pipeline shift to sixteen values in parallel. + /// + /// The values to shift. + /// A positive rounded-right shift or a negative exact-left shift. + /// The shifted values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 RoundShift(Vector512 value, int bit) + { + if (bit > 0) + { + return (value + Vector512.Create(1 << (bit - 1))) >> bit; + } + + return bit < 0 ? value << -bit : value; + } + /// /// Reverses four signed thirty-two-bit lanes. /// @@ -74,7 +104,7 @@ internal static class Av1Transform2dOperations /// The values in reverse lane order. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector128 Reverse(Vector128 value) - => Vector128.Shuffle(value, Vector128.Create(3, 2, 1, 0)); + => Vector128.ShuffleNative(value, Vector128.Create(3, 2, 1, 0)); /// /// Reverses eight signed thirty-two-bit lanes. @@ -83,7 +113,16 @@ internal static class Av1Transform2dOperations /// The values in reverse lane order. [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector256 Reverse(Vector256 value) - => Vector256.Shuffle(value, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); + => Vector256.ShuffleNative(value, Vector256.Create(7, 6, 5, 4, 3, 2, 1, 0)); + + /// + /// Reverses sixteen signed thirty-two-bit lanes. + /// + /// The values to reverse. + /// The values in reverse lane order. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Reverse(Vector512 value) + => Vector512.ShuffleNative(value, Vector512.Create(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); /// /// Transposes a four-by-four matrix of signed thirty-two-bit lanes. @@ -161,4 +200,105 @@ internal static class Av1Transform2dOperations row6 = Vector256.Create(column6Lower, column6Upper); row7 = Vector256.Create(column7Lower, column7Upper); } + + /// + /// Transposes a sixteen-by-sixteen matrix of signed thirty-two-bit lanes. + /// + /// The first input row, replaced by the first output row. + /// The second input row, replaced by the second output row. + /// The third input row, replaced by the third output row. + /// The fourth input row, replaced by the fourth output row. + /// The fifth input row, replaced by the fifth output row. + /// The sixth input row, replaced by the sixth output row. + /// The seventh input row, replaced by the seventh output row. + /// The eighth input row, replaced by the eighth output row. + /// The ninth input row, replaced by the ninth output row. + /// The tenth input row, replaced by the tenth output row. + /// The eleventh input row, replaced by the eleventh output row. + /// The twelfth input row, replaced by the twelfth output row. + /// The thirteenth input row, replaced by the thirteenth output row. + /// The fourteenth input row, replaced by the fourteenth output row. + /// The fifteenth input row, replaced by the fifteenth output row. + /// The sixteenth input row, replaced by the sixteenth output row. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Transpose( + ref Vector512 row0, + ref Vector512 row1, + ref Vector512 row2, + ref Vector512 row3, + ref Vector512 row4, + ref Vector512 row5, + ref Vector512 row6, + ref Vector512 row7, + ref Vector512 row8, + ref Vector512 row9, + ref Vector512 row10, + ref Vector512 row11, + ref Vector512 row12, + ref Vector512 row13, + ref Vector512 row14, + ref Vector512 row15) + { + // A 16x16 transpose consists of four independent 8x8 quadrants. Reusing the established 256-bit transpose + // keeps the portable layout path branch-free while the transform arithmetic itself remains in 512-bit lanes. + // Preserve the bottom-left quadrant before row8-row15 become upper-column output storage. Emitting those upper + // columns first avoids keeping all four quadrants live across the complete operation. + Vector256 lowerBottom0 = row8.GetLower(); + Vector256 lowerBottom1 = row9.GetLower(); + Vector256 lowerBottom2 = row10.GetLower(); + Vector256 lowerBottom3 = row11.GetLower(); + Vector256 lowerBottom4 = row12.GetLower(); + Vector256 lowerBottom5 = row13.GetLower(); + Vector256 lowerBottom6 = row14.GetLower(); + Vector256 lowerBottom7 = row15.GetLower(); + Vector256 upperTop0 = row0.GetUpper(); + Vector256 upperTop1 = row1.GetUpper(); + Vector256 upperTop2 = row2.GetUpper(); + Vector256 upperTop3 = row3.GetUpper(); + Vector256 upperTop4 = row4.GetUpper(); + Vector256 upperTop5 = row5.GetUpper(); + Vector256 upperTop6 = row6.GetUpper(); + Vector256 upperTop7 = row7.GetUpper(); + Vector256 upperBottom0 = row8.GetUpper(); + Vector256 upperBottom1 = row9.GetUpper(); + Vector256 upperBottom2 = row10.GetUpper(); + Vector256 upperBottom3 = row11.GetUpper(); + Vector256 upperBottom4 = row12.GetUpper(); + Vector256 upperBottom5 = row13.GetUpper(); + Vector256 upperBottom6 = row14.GetUpper(); + Vector256 upperBottom7 = row15.GetUpper(); + + Transpose(ref upperTop0, ref upperTop1, ref upperTop2, ref upperTop3, ref upperTop4, ref upperTop5, ref upperTop6, ref upperTop7); + Transpose(ref upperBottom0, ref upperBottom1, ref upperBottom2, ref upperBottom3, ref upperBottom4, ref upperBottom5, ref upperBottom6, ref upperBottom7); + + row8 = Vector512.Create(upperTop0, upperBottom0); + row9 = Vector512.Create(upperTop1, upperBottom1); + row10 = Vector512.Create(upperTop2, upperBottom2); + row11 = Vector512.Create(upperTop3, upperBottom3); + row12 = Vector512.Create(upperTop4, upperBottom4); + row13 = Vector512.Create(upperTop5, upperBottom5); + row14 = Vector512.Create(upperTop6, upperBottom6); + row15 = Vector512.Create(upperTop7, upperBottom7); + + Vector256 lowerTop0 = row0.GetLower(); + Vector256 lowerTop1 = row1.GetLower(); + Vector256 lowerTop2 = row2.GetLower(); + Vector256 lowerTop3 = row3.GetLower(); + Vector256 lowerTop4 = row4.GetLower(); + Vector256 lowerTop5 = row5.GetLower(); + Vector256 lowerTop6 = row6.GetLower(); + Vector256 lowerTop7 = row7.GetLower(); + + Transpose(ref lowerTop0, ref lowerTop1, ref lowerTop2, ref lowerTop3, ref lowerTop4, ref lowerTop5, ref lowerTop6, ref lowerTop7); + Transpose(ref lowerBottom0, ref lowerBottom1, ref lowerBottom2, ref lowerBottom3, ref lowerBottom4, ref lowerBottom5, ref lowerBottom6, ref lowerBottom7); + + row0 = Vector512.Create(lowerTop0, lowerBottom0); + row1 = Vector512.Create(lowerTop1, lowerBottom1); + row2 = Vector512.Create(lowerTop2, lowerBottom2); + row3 = Vector512.Create(lowerTop3, lowerBottom3); + row4 = Vector512.Create(lowerTop4, lowerBottom4); + row5 = Vector512.Create(lowerTop5, lowerBottom5); + row6 = Vector512.Create(lowerTop6, lowerBottom6); + row7 = Vector512.Create(lowerTop7, lowerBottom7); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs index 5651c152a..c497ff529 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs @@ -8,6 +8,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// internal static class Av1TransformWorkspace { + /// + /// The number of integer elements occupied by the three 512-bit transform vectors. + /// + public const int Vector512StorageLength = 3 * Av1Constants.MaxTransformSize * 16; + /// /// The number of integer elements occupied by the three 256-bit transform vectors. /// @@ -21,7 +26,7 @@ internal static class Av1TransformWorkspace /// /// The number of integers required for the largest supported transform block. /// - public const int MaximumLength = (Av1Constants.MaxTransformSize * Av1Constants.MaxTransformSize) + Vector256StorageLength; + public const int MaximumLength = (Av1Constants.MaxTransformSize * Av1Constants.MaxTransformSize) + Vector512StorageLength; /// /// Gets the number of integers required for a transform size. @@ -30,6 +35,6 @@ internal static class Av1TransformWorkspace /// The required workspace length. public static int GetRequiredLength(Av1TransformSize transformSize) { - return (transformSize.GetWidth() * transformSize.GetHeight()) + Vector256StorageLength; + return (transformSize.GetWidth() * transformSize.GetHeight()) + Vector512StorageLength; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs index 62ce98e8d..34f211304 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst16Forward1dOperator.Simd.cs @@ -11,13 +11,190 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Adst16Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // The range table is consumed by coefficient-range-checking builds of libaom. ImageSharp preserves the same + // staged arithmetic, while its production path relies on the bit-depth and shift invariants established by + // the two-dimensional transform configuration. + _ = stageRange; + + // Reordering and alternating signs express the ADST as progressively wider symmetric butterflies. + output[0] = input[0]; + output[1] = -input[15]; + output[2] = -input[7]; + output[3] = input[8]; + output[4] = -input[3]; + output[5] = input[12]; + output[6] = input[4]; + output[7] = -input[11]; + output[8] = -input[1]; + output[9] = input[14]; + output[10] = input[6]; + output[11] = -input[9]; + output[12] = input[2]; + output[13] = -input[13]; + output[14] = -input[5]; + output[15] = input[10]; + + // Rotate four independent pairs by pi/4 so the following butterflies can double their span. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); + step[12] = output[12]; + step[13] = output[13]; + step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); + + // Combine adjacent rotated pairs into four-sample butterflies. + output[0] = step[0] + step[2]; + output[1] = step[1] + step[3]; + output[2] = step[0] - step[2]; + output[3] = step[1] - step[3]; + output[4] = step[4] + step[6]; + output[5] = step[5] + step[7]; + output[6] = step[4] - step[6]; + output[7] = step[5] - step[7]; + output[8] = step[8] + step[10]; + output[9] = step[9] + step[11]; + output[10] = step[8] - step[10]; + output[11] = step[9] - step[11]; + output[12] = step[12] + step[14]; + output[13] = step[13] + step[15]; + output[14] = step[12] - step[14]; + output[15] = step[13] - step[15]; + + // Rotate the upper half of each eight-sample group by pi/8 and 3pi/8. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); + + // Merge the four-sample groups into two eight-sample butterflies. + output[0] = step[0] + step[4]; + output[1] = step[1] + step[5]; + output[2] = step[2] + step[6]; + output[3] = step[3] + step[7]; + output[4] = step[0] - step[4]; + output[5] = step[1] - step[5]; + output[6] = step[2] - step[6]; + output[7] = step[3] - step[7]; + output[8] = step[8] + step[12]; + output[9] = step[9] + step[13]; + output[10] = step[10] + step[14]; + output[11] = step[11] + step[15]; + output[12] = step[8] - step[12]; + output[13] = step[9] - step[13]; + output[14] = step[10] - step[14]; + output[15] = step[11] - step[15]; + + // Rotate the upper eight coefficients with the pi/16 odd-angle pairs. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); + + // Merge both eight-sample halves into the complete sixteen-sample butterfly. + output[0] = step[0] + step[8]; + output[1] = step[1] + step[9]; + output[2] = step[2] + step[10]; + output[3] = step[3] + step[11]; + output[4] = step[4] + step[12]; + output[5] = step[5] + step[13]; + output[6] = step[6] + step[14]; + output[7] = step[7] + step[15]; + output[8] = step[0] - step[8]; + output[9] = step[1] - step[9]; + output[10] = step[2] - step[10]; + output[11] = step[3] - step[11]; + output[12] = step[4] - step[12]; + output[13] = step[5] - step[13]; + output[14] = step[6] - step[14]; + output[15] = step[7] - step[15]; + + // Apply the terminal odd-frequency rotations that define the ADST basis vectors. + step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); + step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); + + // Permute the rotated values into AV1 coefficient order. + output[0] = step[1]; + output[1] = step[14]; + output[2] = step[3]; + output[3] = step[12]; + output[4] = step[5]; + output[5] = step[10]; + output[6] = step[7]; + output[7] = step[8]; + output[8] = step[9]; + output[9] = step[6]; + output[10] = step[11]; + output[11] = step[4]; + output[12] = step[13]; + output[13] = step[2]; + output[14] = step[15]; + output[15] = step[0]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs index 011a99d21..25420fc2d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst4Forward1dOperator.Simd.cs @@ -36,6 +36,19 @@ internal readonly partial struct Av1Adst4Forward1dOperator _ = stageRange; } + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + /// /// Applies the four-point matrix to four independent axes. /// @@ -77,4 +90,24 @@ internal readonly partial struct Av1Adst4Forward1dOperator output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[4], x0, -sinpi[1], x1, -sinpi[3], x2, sinpi[2], x3, cosBit); output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[4] - sinpi[1], x0, -sinpi[1] - sinpi[2], x1, sinpi[3], x2, sinpi[2] - sinpi[4], x3, cosBit); } + + /// + /// Applies the four-point matrix to sixteen independent axes. + /// + /// The source values for sixteen transform axes. + /// The destination values for sixteen transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector512 x0 = input[0]; + Vector512 x1 = input[1]; + Vector512 x2 = input[2]; + Vector512 x3 = input[3]; + + output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[2], x1, sinpi[3], x2, sinpi[4], x3, cosBit); + output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, sinpi[3], x1, 0, x2, -sinpi[3], x3, cosBit); + output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[4], x0, -sinpi[1], x1, -sinpi[3], x2, sinpi[2], x3, cosBit); + output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[4] - sinpi[1], x0, -sinpi[1] - sinpi[2], x1, sinpi[3], x2, sinpi[2] - sinpi[4], x3, cosBit); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs index 0d66e81f6..8083f9f6f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Adst8Forward1dOperator.Simd.cs @@ -11,13 +11,97 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Adst8Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on + // the ranges already established from the coded bit depth and the normative two-dimensional shifts. + _ = stageRange; + + // Stage 1 reorders and signs the inputs so the ADST can be expressed as symmetric butterflies. + output[0] = input[0]; + output[1] = -input[7]; + output[2] = -input[3]; + output[3] = input[4]; + output[4] = -input[1]; + output[5] = input[6]; + output[6] = input[2]; + output[7] = -input[5]; + + // Stage 2 rotates the middle pairs by pi/4. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + + // Stage 3 combines adjacent rotated pairs into four-sample butterflies. + output[0] = step[0] + step[2]; + output[1] = step[1] + step[3]; + output[2] = step[0] - step[2]; + output[3] = step[1] - step[3]; + output[4] = step[4] + step[6]; + output[5] = step[5] + step[7]; + output[6] = step[4] - step[6]; + output[7] = step[5] - step[7]; + + // Stage 4 rotates the upper half by pi/8 and 3pi/8. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + + // Stage 5 merges both four-sample halves into the complete eight-sample butterfly. + output[0] = step[0] + step[4]; + output[1] = step[1] + step[5]; + output[2] = step[2] + step[6]; + output[3] = step[3] + step[7]; + output[4] = step[0] - step[4]; + output[5] = step[1] - step[5]; + output[6] = step[2] - step[6]; + output[7] = step[3] - step[7]; + + // Stage 6 applies the terminal odd-frequency rotations that define the ADST basis vectors. + step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); + + // Stage 7 permutes the rotated values into AV1 coefficient order. + output[0] = step[1]; + output[1] = step[6]; + output[2] = step[3]; + output[3] = step[4]; + output[4] = step[5]; + output[5] = step[2]; + output[6] = step[7]; + output[7] = step[0]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs index fe7c6c8fb..e27bbef0d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct16Forward1dOperator.Simd.cs @@ -11,13 +11,153 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Dct16Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on + // the ranges already established from the coded bit depth and the normative two-dimensional shifts. + _ = stageRange; + + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + output[0] = input[0] + input[15]; + output[1] = input[1] + input[14]; + output[2] = input[2] + input[13]; + output[3] = input[3] + input[12]; + output[4] = input[4] + input[11]; + output[5] = input[5] + input[10]; + output[6] = input[6] + input[9]; + output[7] = input[7] + input[8]; + output[8] = -input[8] + input[7]; + output[9] = -input[9] + input[6]; + output[10] = -input[10] + input[5]; + output[11] = -input[11] + input[4]; + output[12] = -input[12] + input[3]; + output[13] = -input[13] + input[2]; + output[14] = -input[14] + input[1]; + output[15] = -input[15] + input[0]; + + // Stage 2 factorizes the even half and rotates the central odd pairs by pi/4. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0] + output[7]; + step[1] = output[1] + output[6]; + step[2] = output[2] + output[5]; + step[3] = output[3] + output[4]; + step[4] = -output[4] + output[3]; + step[5] = -output[5] + output[2]; + step[6] = -output[6] + output[1]; + step[7] = -output[7] + output[0]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 3 recursively factorizes both eight-sample groups into four-sample butterflies. + output[0] = step[0] + step[3]; + output[1] = step[1] + step[2]; + output[2] = -step[2] + step[1]; + output[3] = -step[3] + step[0]; + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); + output[7] = step[7]; + output[8] = step[8] + step[11]; + output[9] = step[9] + step[10]; + output[10] = -step[10] + step[9]; + output[11] = -step[11] + step[8]; + output[12] = -step[12] + step[15]; + output[13] = -step[13] + step[14]; + output[14] = step[14] + step[13]; + output[15] = step[15] + step[12]; + + // Stage 4 completes the low-frequency four-point DCT and rotates the first odd-frequency pairs. + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); + step[4] = output[4] + output[5]; + step[5] = -output[5] + output[4]; + step[6] = -output[6] + output[7]; + step[7] = output[7] + output[6]; + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); + step[15] = output[15]; + + // Stage 5 combines the remaining odd terms into the sign pattern required by the next rotations. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); + output[8] = step[8] + step[9]; + output[9] = -step[9] + step[8]; + output[10] = -step[10] + step[11]; + output[11] = step[11] + step[10]; + output[12] = step[12] + step[13]; + output[13] = -step[13] + step[12]; + output[14] = -step[14] + step[15]; + output[15] = step[15] + step[14]; + + // Stage 6 applies the final pi/32 odd-frequency rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); + + // Stage 7 permutes the staged values into ascending AV1 coefficient order. + output[0] = step[0]; + output[1] = step[8]; + output[2] = step[4]; + output[3] = step[12]; + output[4] = step[2]; + output[5] = step[10]; + output[6] = step[6]; + output[7] = step[14]; + output[8] = step[1]; + output[9] = step[9]; + output[10] = step[5]; + output[11] = step[13]; + output[12] = step[3]; + output[13] = step[11]; + output[14] = step[7]; + output[15] = step[15]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs index 121334d51..9742c53f6 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct32Forward1dOperator.Simd.cs @@ -11,13 +11,333 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Dct32Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on + // the ranges already established from the coded bit depth and the normative two-dimensional shifts. + _ = stageRange; + + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + output[0] = input[0] + input[31]; + output[1] = input[1] + input[30]; + output[2] = input[2] + input[29]; + output[3] = input[3] + input[28]; + output[4] = input[4] + input[27]; + output[5] = input[5] + input[26]; + output[6] = input[6] + input[25]; + output[7] = input[7] + input[24]; + output[8] = input[8] + input[23]; + output[9] = input[9] + input[22]; + output[10] = input[10] + input[21]; + output[11] = input[11] + input[20]; + output[12] = input[12] + input[19]; + output[13] = input[13] + input[18]; + output[14] = input[14] + input[17]; + output[15] = input[15] + input[16]; + output[16] = -input[16] + input[15]; + output[17] = -input[17] + input[14]; + output[18] = -input[18] + input[13]; + output[19] = -input[19] + input[12]; + output[20] = -input[20] + input[11]; + output[21] = -input[21] + input[10]; + output[22] = -input[22] + input[9]; + output[23] = -input[23] + input[8]; + output[24] = -input[24] + input[7]; + output[25] = -input[25] + input[6]; + output[26] = -input[26] + input[5]; + output[27] = -input[27] + input[4]; + output[28] = -input[28] + input[3]; + output[29] = -input[29] + input[2]; + output[30] = -input[30] + input[1]; + output[31] = -input[31] + input[0]; + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0] + output[15]; + step[1] = output[1] + output[14]; + step[2] = output[2] + output[13]; + step[3] = output[3] + output[12]; + step[4] = output[4] + output[11]; + step[5] = output[5] + output[10]; + step[6] = output[6] + output[9]; + step[7] = output[7] + output[8]; + step[8] = -output[8] + output[7]; + step[9] = -output[9] + output[6]; + step[10] = -output[10] + output[5]; + step[11] = -output[11] + output[4]; + step[12] = -output[12] + output[3]; + step[13] = -output[13] + output[2]; + step[14] = -output[14] + output[1]; + step[15] = -output[15] + output[0]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[24], cospi[32], output[23], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[25], cospi[32], output[22], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[26], cospi[32], output[21], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[27], cospi[32], output[20], cosBit); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. + output[0] = step[0] + step[7]; + output[1] = step[1] + step[6]; + output[2] = step[2] + step[5]; + output[3] = step[3] + step[4]; + output[4] = -step[4] + step[3]; + output[5] = -step[5] + step[2]; + output[6] = -step[6] + step[1]; + output[7] = -step[7] + step[0]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[12], cospi[32], step[11], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[13], cospi[32], step[10], cosBit); + output[14] = step[14]; + output[15] = step[15]; + output[16] = step[16] + step[23]; + output[17] = step[17] + step[22]; + output[18] = step[18] + step[21]; + output[19] = step[19] + step[20]; + output[20] = -step[20] + step[19]; + output[21] = -step[21] + step[18]; + output[22] = -step[22] + step[17]; + output[23] = -step[23] + step[16]; + output[24] = -step[24] + step[31]; + output[25] = -step[25] + step[30]; + output[26] = -step[26] + step[29]; + output[27] = -step[27] + step[28]; + output[28] = step[28] + step[27]; + output[29] = step[29] + step[26]; + output[30] = step[30] + step[25]; + output[31] = step[31] + step[24]; + + // Stage 4 continues the factorization as independent eight-sample groups. + step[0] = output[0] + output[3]; + step[1] = output[1] + output[2]; + step[2] = -output[2] + output[1]; + step[3] = -output[3] + output[0]; + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); + step[7] = output[7]; + step[8] = output[8] + output[11]; + step[9] = output[9] + output[10]; + step[10] = -output[10] + output[9]; + step[11] = -output[11] + output[8]; + step[12] = -output[12] + output[15]; + step[13] = -output[13] + output[14]; + step[14] = output[14] + output[13]; + step[15] = output[15] + output[12]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = Av1Transform1dMath.HalfButterfly(cospi[48], output[26], -cospi[16], output[21], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[48], output[27], -cospi[16], output[20], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[16], output[28], cospi[48], output[19], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[16], output[29], cospi[48], output[18], cosBit); + step[30] = output[30]; + step[31] = output[31]; + + // Stage 5 completes the low-frequency DCT and rotates the first separated odd groups. + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); + output[4] = step[4] + step[5]; + output[5] = -step[5] + step[4]; + output[6] = -step[6] + step[7]; + output[7] = step[7] + step[6]; + output[8] = step[8]; + output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); + output[11] = step[11]; + output[12] = step[12]; + output[13] = Av1Transform1dMath.HalfButterfly(cospi[48], step[13], -cospi[16], step[10], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[16], step[14], cospi[48], step[9], cosBit); + output[15] = step[15]; + output[16] = step[16] + step[19]; + output[17] = step[17] + step[18]; + output[18] = -step[18] + step[17]; + output[19] = -step[19] + step[16]; + output[20] = -step[20] + step[23]; + output[21] = -step[21] + step[22]; + output[22] = step[22] + step[21]; + output[23] = step[23] + step[20]; + output[24] = step[24] + step[27]; + output[25] = step[25] + step[26]; + output[26] = -step[26] + step[25]; + output[27] = -step[27] + step[24]; + output[28] = -step[28] + step[31]; + output[29] = -step[29] + step[30]; + output[30] = step[30] + step[29]; + output[31] = step[31] + step[28]; + + // Stage 6 merges adjacent odd-frequency terms with the required AV1 sign pattern. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); + step[8] = output[8] + output[9]; + step[9] = -output[9] + output[8]; + step[10] = -output[10] + output[11]; + step[11] = output[11] + output[10]; + step[12] = output[12] + output[13]; + step[13] = -output[13] + output[12]; + step[14] = -output[14] + output[15]; + step[15] = output[15] + output[14]; + step[16] = output[16]; + step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); + step[19] = output[19]; + step[20] = output[20]; + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); + step[23] = output[23]; + step[24] = output[24]; + step[25] = Av1Transform1dMath.HalfButterfly(cospi[24], output[25], -cospi[40], output[22], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[40], output[26], cospi[24], output[21], cosBit); + step[27] = output[27]; + step[28] = output[28]; + step[29] = Av1Transform1dMath.HalfButterfly(cospi[56], output[29], -cospi[8], output[18], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[8], output[30], cospi[56], output[17], cosBit); + step[31] = output[31]; + + // Stage 7 applies the pi/32 rotations to the next odd-frequency level. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], cospi[4], step[15], cosBit); + output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], cospi[36], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], cospi[20], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], cospi[52], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[12], step[12], -cospi[52], step[11], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[44], step[13], -cospi[20], step[10], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[28], step[14], -cospi[36], step[9], cosBit); + output[15] = Av1Transform1dMath.HalfButterfly(cospi[60], step[15], -cospi[4], step[8], cosBit); + output[16] = step[16] + step[17]; + output[17] = -step[17] + step[16]; + output[18] = -step[18] + step[19]; + output[19] = step[19] + step[18]; + output[20] = step[20] + step[21]; + output[21] = -step[21] + step[20]; + output[22] = -step[22] + step[23]; + output[23] = step[23] + step[22]; + output[24] = step[24] + step[25]; + output[25] = -step[25] + step[24]; + output[26] = -step[26] + step[27]; + output[27] = step[27] + step[26]; + output[28] = step[28] + step[29]; + output[29] = -step[29] + step[28]; + output[30] = -step[30] + step[31]; + output[31] = step[31] + step[30]; + + // Stage 8 merges the final odd-frequency pairs before their terminal rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], cospi[2], output[31], cosBit); + step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], cospi[34], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], cospi[18], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], cospi[50], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], cospi[10], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], cospi[42], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], cospi[26], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], cospi[58], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[6], output[24], -cospi[58], output[23], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[38], output[25], -cospi[26], output[22], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[22], output[26], -cospi[42], output[21], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[54], output[27], -cospi[10], output[20], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[14], output[28], -cospi[50], output[19], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[46], output[29], -cospi[18], output[18], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[30], output[30], -cospi[34], output[17], cosBit); + step[31] = Av1Transform1dMath.HalfButterfly(cospi[62], output[31], -cospi[2], output[16], cosBit); + + // Stage 9 applies the terminal pi/64 rotations and produces the staged coefficient values. + output[0] = step[0]; + output[1] = step[16]; + output[2] = step[8]; + output[3] = step[24]; + output[4] = step[4]; + output[5] = step[20]; + output[6] = step[12]; + output[7] = step[28]; + output[8] = step[2]; + output[9] = step[18]; + output[10] = step[10]; + output[11] = step[26]; + output[12] = step[6]; + output[13] = step[22]; + output[14] = step[14]; + output[15] = step[30]; + output[16] = step[1]; + output[17] = step[17]; + output[18] = step[9]; + output[19] = step[25]; + output[20] = step[5]; + output[21] = step[21]; + output[22] = step[13]; + output[23] = step[29]; + output[24] = step[3]; + output[25] = step[19]; + output[26] = step[11]; + output[27] = step[27]; + output[28] = step[7]; + output[29] = step[23]; + output[30] = step[15]; + output[31] = step[31]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs index f5b0c3a2d..ef74fc886 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct4Forward1dOperator.Simd.cs @@ -11,13 +11,44 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Dct4Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + _ = stageRange; + + // Mirror butterflies separate the even and odd spatial symmetries used by the four DCT basis vectors. + output[0] = input[0] + input[3]; + output[1] = input[1] + input[2]; + output[2] = input[1] - input[2]; + output[3] = input[0] - input[3]; + + // Each half-butterfly keeps the optimized kernels' wrapping 32-bit arithmetic before applying the + // normative fixed-point rounding shift. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); + + // The staged order groups butterfly partners; AV1 coefficient order interleaves their frequency indices. + output[0] = step[0]; + output[1] = step[2]; + output[2] = step[1]; + output[3] = step[3]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs index fc5ab70f6..32d5118dc 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct64Forward1dOperator.Simd.cs @@ -11,13 +11,753 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Dct64Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on + // the ranges already established from the coded bit depth and the normative two-dimensional shifts. + _ = stageRange; + + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + output[0] = input[0] + input[63]; + output[1] = input[1] + input[62]; + output[2] = input[2] + input[61]; + output[3] = input[3] + input[60]; + output[4] = input[4] + input[59]; + output[5] = input[5] + input[58]; + output[6] = input[6] + input[57]; + output[7] = input[7] + input[56]; + output[8] = input[8] + input[55]; + output[9] = input[9] + input[54]; + output[10] = input[10] + input[53]; + output[11] = input[11] + input[52]; + output[12] = input[12] + input[51]; + output[13] = input[13] + input[50]; + output[14] = input[14] + input[49]; + output[15] = input[15] + input[48]; + output[16] = input[16] + input[47]; + output[17] = input[17] + input[46]; + output[18] = input[18] + input[45]; + output[19] = input[19] + input[44]; + output[20] = input[20] + input[43]; + output[21] = input[21] + input[42]; + output[22] = input[22] + input[41]; + output[23] = input[23] + input[40]; + output[24] = input[24] + input[39]; + output[25] = input[25] + input[38]; + output[26] = input[26] + input[37]; + output[27] = input[27] + input[36]; + output[28] = input[28] + input[35]; + output[29] = input[29] + input[34]; + output[30] = input[30] + input[33]; + output[31] = input[31] + input[32]; + output[32] = -input[32] + input[31]; + output[33] = -input[33] + input[30]; + output[34] = -input[34] + input[29]; + output[35] = -input[35] + input[28]; + output[36] = -input[36] + input[27]; + output[37] = -input[37] + input[26]; + output[38] = -input[38] + input[25]; + output[39] = -input[39] + input[24]; + output[40] = -input[40] + input[23]; + output[41] = -input[41] + input[22]; + output[42] = -input[42] + input[21]; + output[43] = -input[43] + input[20]; + output[44] = -input[44] + input[19]; + output[45] = -input[45] + input[18]; + output[46] = -input[46] + input[17]; + output[47] = -input[47] + input[16]; + output[48] = -input[48] + input[15]; + output[49] = -input[49] + input[14]; + output[50] = -input[50] + input[13]; + output[51] = -input[51] + input[12]; + output[52] = -input[52] + input[11]; + output[53] = -input[53] + input[10]; + output[54] = -input[54] + input[9]; + output[55] = -input[55] + input[8]; + output[56] = -input[56] + input[7]; + output[57] = -input[57] + input[6]; + output[58] = -input[58] + input[5]; + output[59] = -input[59] + input[4]; + output[60] = -input[60] + input[3]; + output[61] = -input[61] + input[2]; + output[62] = -input[62] + input[1]; + output[63] = -input[63] + input[0]; + + // Stage 2 begins the recursive radix-2 factorization and rotates the central odd pairs. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0] + output[31]; + step[1] = output[1] + output[30]; + step[2] = output[2] + output[29]; + step[3] = output[3] + output[28]; + step[4] = output[4] + output[27]; + step[5] = output[5] + output[26]; + step[6] = output[6] + output[25]; + step[7] = output[7] + output[24]; + step[8] = output[8] + output[23]; + step[9] = output[9] + output[22]; + step[10] = output[10] + output[21]; + step[11] = output[11] + output[20]; + step[12] = output[12] + output[19]; + step[13] = output[13] + output[18]; + step[14] = output[14] + output[17]; + step[15] = output[15] + output[16]; + step[16] = -output[16] + output[15]; + step[17] = -output[17] + output[14]; + step[18] = -output[18] + output[13]; + step[19] = -output[19] + output[12]; + step[20] = -output[20] + output[11]; + step[21] = -output[21] + output[10]; + step[22] = -output[22] + output[9]; + step[23] = -output[23] + output[8]; + step[24] = -output[24] + output[7]; + step[25] = -output[25] + output[6]; + step[26] = -output[26] + output[5]; + step[27] = -output[27] + output[4]; + step[28] = -output[28] + output[3]; + step[29] = -output[29] + output[2]; + step[30] = -output[30] + output[1]; + step[31] = -output[31] + output[0]; + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[48], cospi[32], output[47], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[49], cospi[32], output[46], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[50], cospi[32], output[45], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[51], cospi[32], output[44], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[52], cospi[32], output[43], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[53], cospi[32], output[42], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[54], cospi[32], output[41], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[55], cospi[32], output[40], cosBit); + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 3 reduces the even half and folds the next odd-frequency groups into butterflies. + output[0] = step[0] + step[15]; + output[1] = step[1] + step[14]; + output[2] = step[2] + step[13]; + output[3] = step[3] + step[12]; + output[4] = step[4] + step[11]; + output[5] = step[5] + step[10]; + output[6] = step[6] + step[9]; + output[7] = step[7] + step[8]; + output[8] = -step[8] + step[7]; + output[9] = -step[9] + step[6]; + output[10] = -step[10] + step[5]; + output[11] = -step[11] + step[4]; + output[12] = -step[12] + step[3]; + output[13] = -step[13] + step[2]; + output[14] = -step[14] + step[1]; + output[15] = -step[15] + step[0]; + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[24], cospi[32], step[23], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[25], cospi[32], step[22], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[26], cospi[32], step[21], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[27], cospi[32], step[20], cosBit); + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + output[32] = step[32] + step[47]; + output[33] = step[33] + step[46]; + output[34] = step[34] + step[45]; + output[35] = step[35] + step[44]; + output[36] = step[36] + step[43]; + output[37] = step[37] + step[42]; + output[38] = step[38] + step[41]; + output[39] = step[39] + step[40]; + output[40] = -step[40] + step[39]; + output[41] = -step[41] + step[38]; + output[42] = -step[42] + step[37]; + output[43] = -step[43] + step[36]; + output[44] = -step[44] + step[35]; + output[45] = -step[45] + step[34]; + output[46] = -step[46] + step[33]; + output[47] = -step[47] + step[32]; + output[48] = -step[48] + step[63]; + output[49] = -step[49] + step[62]; + output[50] = -step[50] + step[61]; + output[51] = -step[51] + step[60]; + output[52] = -step[52] + step[59]; + output[53] = -step[53] + step[58]; + output[54] = -step[54] + step[57]; + output[55] = -step[55] + step[56]; + output[56] = step[56] + step[55]; + output[57] = step[57] + step[54]; + output[58] = step[58] + step[53]; + output[59] = step[59] + step[52]; + output[60] = step[60] + step[51]; + output[61] = step[61] + step[50]; + output[62] = step[62] + step[49]; + output[63] = step[63] + step[48]; + + // Stage 4 continues the factorization as independent sixteen-sample groups. + step[0] = output[0] + output[7]; + step[1] = output[1] + output[6]; + step[2] = output[2] + output[5]; + step[3] = output[3] + output[4]; + step[4] = -output[4] + output[3]; + step[5] = -output[5] + output[2]; + step[6] = -output[6] + output[1]; + step[7] = -output[7] + output[0]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[12], cospi[32], output[11], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[13], cospi[32], output[10], cosBit); + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16] + output[23]; + step[17] = output[17] + output[22]; + step[18] = output[18] + output[21]; + step[19] = output[19] + output[20]; + step[20] = -output[20] + output[19]; + step[21] = -output[21] + output[18]; + step[22] = -output[22] + output[17]; + step[23] = -output[23] + output[16]; + step[24] = -output[24] + output[31]; + step[25] = -output[25] + output[30]; + step[26] = -output[26] + output[29]; + step[27] = -output[27] + output[28]; + step[28] = output[28] + output[27]; + step[29] = output[29] + output[26]; + step[30] = output[30] + output[25]; + step[31] = output[31] + output[24]; + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[52] = Av1Transform1dMath.HalfButterfly(cospi[48], output[52], -cospi[16], output[43], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[48], output[53], -cospi[16], output[42], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[48], output[54], -cospi[16], output[41], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[48], output[55], -cospi[16], output[40], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[16], output[56], cospi[48], output[39], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[16], output[57], cospi[48], output[38], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[16], output[58], cospi[48], output[37], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[16], output[59], cospi[48], output[36], cosBit); + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 5 reduces those groups into the eight-sample DCT and ADST building blocks. + output[0] = step[0] + step[3]; + output[1] = step[1] + step[2]; + output[2] = -step[2] + step[1]; + output[3] = -step[3] + step[0]; + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[6], cospi[32], step[5], cosBit); + output[7] = step[7]; + output[8] = step[8] + step[11]; + output[9] = step[9] + step[10]; + output[10] = -step[10] + step[9]; + output[11] = -step[11] + step[8]; + output[12] = -step[12] + step[15]; + output[13] = -step[13] + step[14]; + output[14] = step[14] + step[13]; + output[15] = step[15] + step[12]; + output[16] = step[16]; + output[17] = step[17]; + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = Av1Transform1dMath.HalfButterfly(cospi[48], step[26], -cospi[16], step[21], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[48], step[27], -cospi[16], step[20], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[16], step[28], cospi[48], step[19], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[16], step[29], cospi[48], step[18], cosBit); + output[30] = step[30]; + output[31] = step[31]; + output[32] = step[32] + step[39]; + output[33] = step[33] + step[38]; + output[34] = step[34] + step[37]; + output[35] = step[35] + step[36]; + output[36] = -step[36] + step[35]; + output[37] = -step[37] + step[34]; + output[38] = -step[38] + step[33]; + output[39] = -step[39] + step[32]; + output[40] = -step[40] + step[47]; + output[41] = -step[41] + step[46]; + output[42] = -step[42] + step[45]; + output[43] = -step[43] + step[44]; + output[44] = step[44] + step[43]; + output[45] = step[45] + step[42]; + output[46] = step[46] + step[41]; + output[47] = step[47] + step[40]; + output[48] = step[48] + step[55]; + output[49] = step[49] + step[54]; + output[50] = step[50] + step[53]; + output[51] = step[51] + step[52]; + output[52] = -step[52] + step[51]; + output[53] = -step[53] + step[50]; + output[54] = -step[54] + step[49]; + output[55] = -step[55] + step[48]; + output[56] = -step[56] + step[63]; + output[57] = -step[57] + step[62]; + output[58] = -step[58] + step[61]; + output[59] = -step[59] + step[60]; + output[60] = step[60] + step[59]; + output[61] = step[61] + step[58]; + output[62] = step[62] + step[57]; + output[63] = step[63] + step[56]; + + // Stage 6 completes the low-frequency DCT and rotates the first separated odd groups. + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[1], cospi[32], output[0], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[48], output[3], -cospi[16], output[2], cosBit); + step[4] = output[4] + output[5]; + step[5] = -output[5] + output[4]; + step[6] = -output[6] + output[7]; + step[7] = output[7] + output[6]; + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[13], -cospi[16], output[10], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[9], cosBit); + step[15] = output[15]; + step[16] = output[16] + output[19]; + step[17] = output[17] + output[18]; + step[18] = -output[18] + output[17]; + step[19] = -output[19] + output[16]; + step[20] = -output[20] + output[23]; + step[21] = -output[21] + output[22]; + step[22] = output[22] + output[21]; + step[23] = output[23] + output[20]; + step[24] = output[24] + output[27]; + step[25] = output[25] + output[26]; + step[26] = -output[26] + output[25]; + step[27] = -output[27] + output[24]; + step[28] = -output[28] + output[31]; + step[29] = -output[29] + output[30]; + step[30] = output[30] + output[29]; + step[31] = output[31] + output[28]; + step[32] = output[32]; + step[33] = output[33]; + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = Av1Transform1dMath.HalfButterfly(cospi[24], output[50], -cospi[40], output[45], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[24], output[51], -cospi[40], output[44], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[40], output[52], cospi[24], output[43], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[40], output[53], cospi[24], output[42], cosBit); + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[58] = Av1Transform1dMath.HalfButterfly(cospi[56], output[58], -cospi[8], output[37], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[56], output[59], -cospi[8], output[36], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[8], output[60], cospi[56], output[35], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[8], output[61], cospi[56], output[34], cosBit); + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 merges adjacent odd-frequency terms with the required AV1 sign pattern. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[24], step[6], -cospi[40], step[5], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[56], step[7], -cospi[8], step[4], cosBit); + output[8] = step[8] + step[9]; + output[9] = -step[9] + step[8]; + output[10] = -step[10] + step[11]; + output[11] = step[11] + step[10]; + output[12] = step[12] + step[13]; + output[13] = -step[13] + step[12]; + output[14] = -step[14] + step[15]; + output[15] = step[15] + step[14]; + output[16] = step[16]; + output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); + output[19] = step[19]; + output[20] = step[20]; + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); + output[23] = step[23]; + output[24] = step[24]; + output[25] = Av1Transform1dMath.HalfButterfly(cospi[24], step[25], -cospi[40], step[22], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[40], step[26], cospi[24], step[21], cosBit); + output[27] = step[27]; + output[28] = step[28]; + output[29] = Av1Transform1dMath.HalfButterfly(cospi[56], step[29], -cospi[8], step[18], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[8], step[30], cospi[56], step[17], cosBit); + output[31] = step[31]; + output[32] = step[32] + step[35]; + output[33] = step[33] + step[34]; + output[34] = -step[34] + step[33]; + output[35] = -step[35] + step[32]; + output[36] = -step[36] + step[39]; + output[37] = -step[37] + step[38]; + output[38] = step[38] + step[37]; + output[39] = step[39] + step[36]; + output[40] = step[40] + step[43]; + output[41] = step[41] + step[42]; + output[42] = -step[42] + step[41]; + output[43] = -step[43] + step[40]; + output[44] = -step[44] + step[47]; + output[45] = -step[45] + step[46]; + output[46] = step[46] + step[45]; + output[47] = step[47] + step[44]; + output[48] = step[48] + step[51]; + output[49] = step[49] + step[50]; + output[50] = -step[50] + step[49]; + output[51] = -step[51] + step[48]; + output[52] = -step[52] + step[55]; + output[53] = -step[53] + step[54]; + output[54] = step[54] + step[53]; + output[55] = step[55] + step[52]; + output[56] = step[56] + step[59]; + output[57] = step[57] + step[58]; + output[58] = -step[58] + step[57]; + output[59] = -step[59] + step[56]; + output[60] = -step[60] + step[63]; + output[61] = -step[61] + step[62]; + output[62] = step[62] + step[61]; + output[63] = step[63] + step[60]; + + // Stage 8 applies the next level of odd-frequency rotations. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[12], output[12], -cospi[52], output[11], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[44], output[13], -cospi[20], output[10], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[28], output[14], -cospi[36], output[9], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[60], output[15], -cospi[4], output[8], cosBit); + step[16] = output[16] + output[17]; + step[17] = -output[17] + output[16]; + step[18] = -output[18] + output[19]; + step[19] = output[19] + output[18]; + step[20] = output[20] + output[21]; + step[21] = -output[21] + output[20]; + step[22] = -output[22] + output[23]; + step[23] = output[23] + output[22]; + step[24] = output[24] + output[25]; + step[25] = -output[25] + output[24]; + step[26] = -output[26] + output[27]; + step[27] = output[27] + output[26]; + step[28] = output[28] + output[29]; + step[29] = -output[29] + output[28]; + step[30] = -output[30] + output[31]; + step[31] = output[31] + output[30]; + step[32] = output[32]; + step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); + step[35] = output[35]; + step[36] = output[36]; + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); + step[39] = output[39]; + step[40] = output[40]; + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); + step[43] = output[43]; + step[44] = output[44]; + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); + step[47] = output[47]; + step[48] = output[48]; + step[49] = Av1Transform1dMath.HalfButterfly(cospi[12], output[49], -cospi[52], output[46], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[52], output[50], cospi[12], output[45], cosBit); + step[51] = output[51]; + step[52] = output[52]; + step[53] = Av1Transform1dMath.HalfButterfly(cospi[44], output[53], -cospi[20], output[42], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[20], output[54], cospi[44], output[41], cosBit); + step[55] = output[55]; + step[56] = output[56]; + step[57] = Av1Transform1dMath.HalfButterfly(cospi[28], output[57], -cospi[36], output[38], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[36], output[58], cospi[28], output[37], cosBit); + step[59] = output[59]; + step[60] = output[60]; + step[61] = Av1Transform1dMath.HalfButterfly(cospi[60], output[61], -cospi[4], output[34], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[4], output[62], cospi[60], output[33], cosBit); + step[63] = output[63]; + + // Stage 9 merges the remaining odd-frequency pairs before their terminal rotations. + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], cospi[2], step[31], cosBit); + output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], cospi[34], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], cospi[18], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], cospi[50], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], cospi[10], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], cospi[42], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], cospi[26], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], cospi[58], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[6], step[24], -cospi[58], step[23], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[38], step[25], -cospi[26], step[22], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[22], step[26], -cospi[42], step[21], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[54], step[27], -cospi[10], step[20], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[14], step[28], -cospi[50], step[19], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[46], step[29], -cospi[18], step[18], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[30], step[30], -cospi[34], step[17], cosBit); + output[31] = Av1Transform1dMath.HalfButterfly(cospi[62], step[31], -cospi[2], step[16], cosBit); + output[32] = step[32] + step[33]; + output[33] = -step[33] + step[32]; + output[34] = -step[34] + step[35]; + output[35] = step[35] + step[34]; + output[36] = step[36] + step[37]; + output[37] = -step[37] + step[36]; + output[38] = -step[38] + step[39]; + output[39] = step[39] + step[38]; + output[40] = step[40] + step[41]; + output[41] = -step[41] + step[40]; + output[42] = -step[42] + step[43]; + output[43] = step[43] + step[42]; + output[44] = step[44] + step[45]; + output[45] = -step[45] + step[44]; + output[46] = -step[46] + step[47]; + output[47] = step[47] + step[46]; + output[48] = step[48] + step[49]; + output[49] = -step[49] + step[48]; + output[50] = -step[50] + step[51]; + output[51] = step[51] + step[50]; + output[52] = step[52] + step[53]; + output[53] = -step[53] + step[52]; + output[54] = -step[54] + step[55]; + output[55] = step[55] + step[54]; + output[56] = step[56] + step[57]; + output[57] = -step[57] + step[56]; + output[58] = -step[58] + step[59]; + output[59] = step[59] + step[58]; + output[60] = step[60] + step[61]; + output[61] = -step[61] + step[60]; + output[62] = -step[62] + step[63]; + output[63] = step[63] + step[62]; + + // Stage 10 applies the pi/64 rotations to the penultimate odd-frequency level. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], cospi[1], output[63], cosBit); + step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], cospi[33], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], cospi[17], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], cospi[49], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], cospi[9], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], cospi[41], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], cospi[25], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], cospi[57], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], cospi[5], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], cospi[37], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], cospi[21], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], cospi[53], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], cospi[13], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], cospi[45], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], cospi[29], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], cospi[61], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[3], output[48], -cospi[61], output[47], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[35], output[49], -cospi[29], output[46], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[19], output[50], -cospi[45], output[45], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[51], output[51], -cospi[13], output[44], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[11], output[52], -cospi[53], output[43], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[43], output[53], -cospi[21], output[42], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[27], output[54], -cospi[37], output[41], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[59], output[55], -cospi[5], output[40], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[7], output[56], -cospi[57], output[39], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[39], output[57], -cospi[25], output[38], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[23], output[58], -cospi[41], output[37], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[55], output[59], -cospi[9], output[36], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[15], output[60], -cospi[49], output[35], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[47], output[61], -cospi[17], output[34], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[31], output[62], -cospi[33], output[33], cosBit); + step[63] = Av1Transform1dMath.HalfButterfly(cospi[63], output[63], -cospi[1], output[32], cosBit); + + // Stage 11 applies the terminal pi/128 rotations and produces the staged coefficient values. + output[0] = step[0]; + output[1] = step[32]; + output[2] = step[16]; + output[3] = step[48]; + output[4] = step[8]; + output[5] = step[40]; + output[6] = step[24]; + output[7] = step[56]; + output[8] = step[4]; + output[9] = step[36]; + output[10] = step[20]; + output[11] = step[52]; + output[12] = step[12]; + output[13] = step[44]; + output[14] = step[28]; + output[15] = step[60]; + output[16] = step[2]; + output[17] = step[34]; + output[18] = step[18]; + output[19] = step[50]; + output[20] = step[10]; + output[21] = step[42]; + output[22] = step[26]; + output[23] = step[58]; + output[24] = step[6]; + output[25] = step[38]; + output[26] = step[22]; + output[27] = step[54]; + output[28] = step[14]; + output[29] = step[46]; + output[30] = step[30]; + output[31] = step[62]; + output[32] = step[1]; + output[33] = step[33]; + output[34] = step[17]; + output[35] = step[49]; + output[36] = step[9]; + output[37] = step[41]; + output[38] = step[25]; + output[39] = step[57]; + output[40] = step[5]; + output[41] = step[37]; + output[42] = step[21]; + output[43] = step[53]; + output[44] = step[13]; + output[45] = step[45]; + output[46] = step[29]; + output[47] = step[61]; + output[48] = step[3]; + output[49] = step[35]; + output[50] = step[19]; + output[51] = step[51]; + output[52] = step[11]; + output[53] = step[43]; + output[54] = step[27]; + output[55] = step[59]; + output[56] = step[7]; + output[57] = step[39]; + output[58] = step[23]; + output[59] = step[55]; + output[60] = step[15]; + output[61] = step[47]; + output[62] = step[31]; + output[63] = step[63]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs index de48e731a..a5ebfca18 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1Dct8Forward1dOperator.Simd.cs @@ -11,13 +11,77 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; internal readonly partial struct Av1Dct8Forward1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // libaom uses this table only when coefficient-range checking is enabled. The production transform relies on + // the ranges already established from the coded bit depth and the normative two-dimensional shifts. + _ = stageRange; + + // Stage 1 forms mirror-symmetric sums and differences, separating the even and odd DCT terms. + output[0] = input[0] + input[7]; + output[1] = input[1] + input[6]; + output[2] = input[2] + input[5]; + output[3] = input[3] + input[4]; + output[4] = -input[4] + input[3]; + output[5] = -input[5] + input[2]; + output[6] = -input[6] + input[1]; + output[7] = -input[7] + input[0]; + + // Stage 2 applies a four-point DCT to the even half and a pi/4 rotation to the middle odd pair. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = output[0] + output[3]; + step[1] = output[1] + output[2]; + step[2] = -output[2] + output[1]; + step[3] = -output[3] + output[0]; + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[5], cosBit); + step[7] = output[7]; + + // Stage 3 completes the even transform and combines the odd terms into sum and difference pairs. + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[1], cospi[32], step[0], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[48], step[3], -cospi[16], step[2], cosBit); + output[4] = step[4] + step[5]; + output[5] = -step[5] + step[4]; + output[6] = -step[6] + step[7]; + output[7] = step[7] + step[6]; + + // Stage 4 rotates the odd-frequency pairs by the remaining pi/16 angles. + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[24], output[6], -cospi[40], output[5], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[56], output[7], -cospi[8], output[4], cosBit); + + // Stage 5 permutes the staged values into ascending AV1 coefficient order. + output[0] = step[0]; + output[1] = step[4]; + output[2] = step[2]; + output[3] = step[6]; + output[4] = step[1]; + output[5] = step[5]; + output[6] = step[3]; + output[7] = step[7]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs index 3fa1e4d0c..a8b362d03 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1IdentityForward1dOperators.Simd.cs @@ -37,6 +37,20 @@ internal readonly partial struct Av1Identity4Forward1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -71,6 +85,20 @@ internal readonly partial struct Av1Identity8Forward1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -105,6 +133,20 @@ internal readonly partial struct Av1Identity16Forward1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -139,4 +181,18 @@ internal readonly partial struct Av1Identity32Forward1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs index f9238be0c..b88fff214 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs @@ -38,4 +38,13 @@ internal interface IAv1InverseTransformOutputOperator /// The eight inverse-transform residuals. /// The coded sample bit depth. public static abstract void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth); + + /// + /// Adds sixteen residuals to sixteen predicted samples and stores the clipped results. + /// + /// The first predicted sample. + /// The first destination sample. + /// The sixteen inverse-transform residuals. + /// The coded sample bit depth. + public static abstract void Add(ref TSample prediction, ref TSample destination, Vector512 residual, int bitDepth); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs index 81f39ccd0..e67a58d8b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs @@ -53,4 +53,19 @@ internal interface IAv1Transform1dOperator ref Av1TransformVector> step, int cosBit, Av1TransformStageRange stageRange); + + /// + /// Transforms sixteen independent axes in parallel. + /// + /// The source values for sixteen transform axes. + /// The destination values for sixteen transform axes. + /// The fixed stage storage for sixteen transform axes. + /// The fixed-point precision of the cosine constants. + /// The signed-bit range assigned to each transform stage. + public static abstract void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs index 8c98547f3..e472fdbde 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.Simd.cs @@ -11,13 +11,194 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Adst16Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[15]; + output[1] = input[0]; + output[2] = input[13]; + output[3] = input[2]; + output[4] = input[11]; + output[5] = input[4]; + output[6] = input[9]; + output[7] = input[6]; + output[8] = input[7]; + output[9] = input[8]; + output[10] = input[5]; + output[11] = input[10]; + output[12] = input[3]; + output[13] = input[12]; + output[14] = input[1]; + output[15] = input[14]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[2], output[0], cospi[62], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[62], output[0], -cospi[2], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[10], output[2], cospi[54], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[54], output[2], -cospi[10], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[18], output[4], cospi[46], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[46], output[4], -cospi[18], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[26], output[6], cospi[38], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[38], output[6], -cospi[26], output[7], cosBit); + step[8] = Av1Transform1dMath.HalfButterfly(cospi[34], output[8], cospi[30], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[30], output[8], -cospi[34], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[42], output[10], cospi[22], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[22], output[10], -cospi[42], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[50], output[12], cospi[14], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[14], output[12], -cospi[50], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[58], output[14], cospi[6], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[6], output[14], -cospi[58], output[15], cosBit); + + // Stage 3 separates the complete butterfly into two eight-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[8], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[9], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[10], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[11], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[12], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[13], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[14], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[15], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[0] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[1] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[2] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[3] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[4] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[5] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[6] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[7] - step[15], stageRange[stage]); + + // Stage 4 reverses the pi/16 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[8], output[8], cospi[56], output[9], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[56], output[8], -cospi[8], output[9], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[40], output[10], cospi[24], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[24], output[10], -cospi[40], output[11], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[12], cospi[8], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[8], output[12], cospi[56], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[14], cospi[40], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[40], output[14], cospi[24], output[15], cosBit); + + // Stage 5 separates each eight-sample half into four-sample groups and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[12], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[13], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[14], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[15], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[8] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[9] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[10] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[11] - step[15], stageRange[stage]); + + // Stage 6 reverses the pi/8 and 3pi/8 rotations. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = Av1Transform1dMath.HalfButterfly(cospi[16], output[12], cospi[48], output[13], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[48], output[12], -cospi[16], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[14], cospi[16], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[16], output[14], cospi[48], output[15], cosBit); + + // Stage 7 separates the four-sample groups into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[10], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[11], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[8] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[9] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[14], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[15], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[12] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[13] - step[15], stageRange[stage]); + + // Stage 8 reverses the pi/4 rotations for the middle pairs. + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[11], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], -cospi[32], output[11], cosBit); + step[12] = output[12]; + step[13] = output[13]; + step[14] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], cospi[32], output[15], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[32], output[14], -cospi[32], output[15], cosBit); + + // Stage 9 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[8]; + output[2] = step[12]; + output[3] = -step[4]; + output[4] = step[6]; + output[5] = -step[14]; + output[6] = step[10]; + output[7] = -step[2]; + output[8] = step[3]; + output[9] = -step[11]; + output[10] = step[15]; + output[11] = -step[7]; + output[12] = step[5]; + output[13] = -step[13]; + output[14] = step[9]; + output[15] = -step[1]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs index 90fdbf2f9..83a8a104f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.Simd.cs @@ -36,6 +36,19 @@ internal readonly partial struct Av1Adst4Inverse1dOperator _ = stageRange; } + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + TransformCore(ref input, ref output, cosBit); + _ = step; + _ = stageRange; + } + /// /// Applies the inverse four-point matrix to four independent axes. /// @@ -77,4 +90,24 @@ internal readonly partial struct Av1Adst4Inverse1dOperator output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); } + + /// + /// Applies the inverse four-point matrix to sixteen independent axes. + /// + /// The source values for sixteen transform axes. + /// The destination values for sixteen transform axes. + /// The fixed-point precision of the sine constants. + private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + { + ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); + Vector512 x0 = input[0]; + Vector512 x1 = input[1]; + Vector512 x2 = input[2]; + Vector512 x3 = input[3]; + + output[0] = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output[1] = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output[2] = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output[3] = Av1Transform1dMath.MultiplyAdd4(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs index 83458e6a0..a9b6df2fa 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.Simd.cs @@ -11,13 +11,101 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Adst8Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes the coefficients into the signed order used by the ADST factorization. + stage++; + output[0] = input[7]; + output[1] = input[0]; + output[2] = input[5]; + output[3] = input[2]; + output[4] = input[3]; + output[5] = input[4]; + output[6] = input[1]; + output[7] = input[6]; + + // Stage 2 applies the terminal odd-angle rotations in reverse. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[4], output[0], cospi[60], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[60], output[0], -cospi[4], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[20], output[2], cospi[44], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[44], output[2], -cospi[20], output[3], cosBit); + step[4] = Av1Transform1dMath.HalfButterfly(cospi[36], output[4], cospi[28], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[28], output[4], -cospi[36], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[52], output[6], cospi[12], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[12], output[6], -cospi[52], output[7], cosBit); + + // Stage 3 separates the complete butterfly into two four-sample halves and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[4], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[5], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[6], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[7], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[0] - step[4], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[1] - step[5], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[2] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[3] - step[7], stageRange[stage]); + + // Stage 4 reverses the pi/8 and 3pi/8 rotations in the upper half. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[16], output[4], cospi[48], output[5], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[48], output[4], -cospi[16], output[5], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[6], cospi[16], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[16], output[6], cospi[48], output[7], cosBit); + + // Stage 5 separates the four-sample halves into adjacent coefficient pairs and clamps each lane. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[2], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[3], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[0] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[1] - step[3], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[6], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[7], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[4] - step[6], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[5] - step[7], stageRange[stage]); + + // Stage 6 reverses the pi/4 rotations for the middle pairs. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], cospi[32], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[32], output[2], -cospi[32], output[3], cosBit); + step[4] = output[4]; + step[5] = output[5]; + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], cospi[32], output[7], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[32], output[6], -cospi[32], output[7], cosBit); + + // Stage 7 applies the AV1 signs and permutation that restore spatial sample order. + output[0] = step[0]; + output[1] = -step[4]; + output[2] = step[6]; + output[3] = -step[2]; + output[4] = step[3]; + output[5] = -step[7]; + output[6] = step[5]; + output[7] = -step[1]; + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs index 6a517d2b4..1d6cdda62 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.Simd.cs @@ -11,13 +11,163 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Dct16Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[8]; + output[2] = input[4]; + output[3] = input[12]; + output[4] = input[2]; + output[5] = input[10]; + output[6] = input[6]; + output[7] = input[14]; + output[8] = input[1]; + output[9] = input[9]; + output[10] = input[5]; + output[11] = input[13]; + output[12] = input[3]; + output[13] = input[11]; + output[14] = input[7]; + output[15] = input[15]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/32 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + + // Stage 3 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], range); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[11] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], range); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[15] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], range); + + // Stage 4 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], range); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[7] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], range); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + + // Stage 5 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], range); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[15] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[14] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], range); + + // Stage 6 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], range); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], range); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], range); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], range); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], range); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + + // Stage 7 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], range); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], range); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], range); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], range); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], range); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], range); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], range); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], range); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], range); + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs index 39b689d0a..3dc27a6ed 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.Simd.cs @@ -11,13 +11,347 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Dct32Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[16]; + output[2] = input[8]; + output[3] = input[24]; + output[4] = input[4]; + output[5] = input[20]; + output[6] = input[12]; + output[7] = input[28]; + output[8] = input[2]; + output[9] = input[18]; + output[10] = input[10]; + output[11] = input[26]; + output[12] = input[6]; + output[13] = input[22]; + output[14] = input[14]; + output[15] = input[30]; + output[16] = input[1]; + output[17] = input[17]; + output[18] = input[9]; + output[19] = input[25]; + output[20] = input[5]; + output[21] = input[21]; + output[22] = input[13]; + output[23] = input[29]; + output[24] = input[3]; + output[25] = input[19]; + output[26] = input[11]; + output[27] = input[27]; + output[28] = input[7]; + output[29] = input[23]; + output[30] = input[15]; + output[31] = input[31]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/64 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.HalfButterfly(cospi[62], output[16], -cospi[2], output[31], cosBit); + step[17] = Av1Transform1dMath.HalfButterfly(cospi[30], output[17], -cospi[34], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(cospi[46], output[18], -cospi[18], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(cospi[14], output[19], -cospi[50], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(cospi[54], output[20], -cospi[10], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(cospi[22], output[21], -cospi[42], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(cospi[38], output[22], -cospi[26], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(cospi[6], output[23], -cospi[58], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[58], output[23], cospi[6], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[26], output[22], cospi[38], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[42], output[21], cospi[22], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[10], output[20], cospi[54], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[50], output[19], cospi[14], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[18], output[18], cospi[46], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[34], output[17], cospi[30], output[30], cosBit); + step[31] = Av1Transform1dMath.HalfButterfly(cospi[2], output[16], cospi[62], output[31], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = Av1Transform1dMath.HalfButterfly(cospi[60], step[8], -cospi[4], step[15], cosBit); + output[9] = Av1Transform1dMath.HalfButterfly(cospi[28], step[9], -cospi[36], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(cospi[44], step[10], -cospi[20], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(cospi[12], step[11], -cospi[52], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[52], step[11], cospi[12], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[20], step[10], cospi[44], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[36], step[9], cospi[28], step[14], cosBit); + output[15] = Av1Transform1dMath.HalfButterfly(cospi[4], step[8], cospi[60], step[15], cosBit); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[17], range); + output[17] = Av1Transform1dMath.Clamp(step[16] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(-step[18] + step[19], range); + output[19] = Av1Transform1dMath.Clamp(step[18] + step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[21], range); + output[21] = Av1Transform1dMath.Clamp(step[20] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(-step[22] + step[23], range); + output[23] = Av1Transform1dMath.Clamp(step[22] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[25], range); + output[25] = Av1Transform1dMath.Clamp(step[24] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[27], range); + output[27] = Av1Transform1dMath.Clamp(step[26] + step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[29], range); + output[29] = Av1Transform1dMath.Clamp(step[28] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(-step[30] + step[31], range); + output[31] = Av1Transform1dMath.Clamp(step[30] + step[31], range); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + range = stageRange[stage]; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], step[7], cosBit); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[9], range); + step[9] = Av1Transform1dMath.Clamp(output[8] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(-output[10] + output[11], range); + step[11] = Av1Transform1dMath.Clamp(output[10] + output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[13], range); + step[13] = Av1Transform1dMath.Clamp(output[12] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(-output[14] + output[15], range); + step[15] = Av1Transform1dMath.Clamp(output[14] + output[15], range); + step[16] = output[16]; + step[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[17], cospi[56], output[30], cosBit); + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[18], -cospi[8], output[29], cosBit); + step[19] = output[19]; + step[20] = output[20]; + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[21], cospi[24], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[22], -cospi[40], output[25], cosBit); + step[23] = output[23]; + step[24] = output[24]; + step[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[22], cospi[24], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[24], output[21], cospi[40], output[26], cosBit); + step[27] = output[27]; + step[28] = output[28]; + step[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[18], cospi[56], output[29], cosBit); + step[30] = Av1Transform1dMath.HalfButterfly(cospi[56], output[17], cospi[8], output[30], cosBit); + step[31] = output[31]; + + // Stage 5 reconstructs the embedded eight-point groups and combines adjacent odd terms. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(-step[6] + step[7], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + output[8] = step[8]; + output[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[9], cospi[48], step[14], cosBit); + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[10], -cospi[16], step[13], cosBit); + output[11] = step[11]; + output[12] = step[12]; + output[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[10], cospi[48], step[13], cosBit); + output[14] = Av1Transform1dMath.HalfButterfly(cospi[48], step[9], cospi[16], step[14], cosBit); + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[19], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[18], range); + output[18] = Av1Transform1dMath.Clamp(step[17] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[16] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(-step[20] + step[23], range); + output[21] = Av1Transform1dMath.Clamp(-step[21] + step[22], range); + output[22] = Av1Transform1dMath.Clamp(step[21] + step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[20] + step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[27], range); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[26], range); + output[26] = Av1Transform1dMath.Clamp(step[25] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[24] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(-step[28] + step[31], range); + output[29] = Av1Transform1dMath.Clamp(-step[29] + step[30], range); + output[30] = Av1Transform1dMath.Clamp(step[29] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[28] + step[31], range); + + // Stage 6 completes the low-frequency four-point DCT and rotates the next odd-frequency pairs. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + step[8] = Av1Transform1dMath.Clamp(output[8] + output[11], range); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[10], range); + step[10] = Av1Transform1dMath.Clamp(output[9] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[8] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(-output[12] + output[15], range); + step[13] = Av1Transform1dMath.Clamp(-output[13] + output[14], range); + step[14] = Av1Transform1dMath.Clamp(output[13] + output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[12] + output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[18], cospi[48], output[29], cosBit); + step[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[19], cospi[48], output[28], cosBit); + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[20], -cospi[16], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[21], -cospi[16], output[26], cosBit); + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[21], cospi[48], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[20], cospi[48], output[27], cosBit); + step[28] = Av1Transform1dMath.HalfButterfly(cospi[48], output[19], cospi[16], output[28], cosBit); + step[29] = Av1Transform1dMath.HalfButterfly(cospi[48], output[18], cospi[16], output[29], cosBit); + step[30] = output[30]; + step[31] = output[31]; + + // Stage 7 widens the reconstructed groups through their next butterfly level. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + output[8] = step[8]; + output[9] = step[9]; + output[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[10], cospi[32], step[13], cosBit); + output[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[11], cospi[32], step[12], cosBit); + output[12] = Av1Transform1dMath.HalfButterfly(cospi[32], step[11], cospi[32], step[12], cosBit); + output[13] = Av1Transform1dMath.HalfButterfly(cospi[32], step[10], cospi[32], step[13], cosBit); + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.Clamp(step[16] + step[23], range); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[22], range); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[21], range); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[20], range); + output[20] = Av1Transform1dMath.Clamp(step[19] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[18] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[17] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[16] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(-step[24] + step[31], range); + output[25] = Av1Transform1dMath.Clamp(-step[25] + step[30], range); + output[26] = Av1Transform1dMath.Clamp(-step[26] + step[29], range); + output[27] = Av1Transform1dMath.Clamp(-step[27] + step[28], range); + output[28] = Av1Transform1dMath.Clamp(step[27] + step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[26] + step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[25] + step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[24] + step[31], range); + + // Stage 8 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + range = stageRange[stage]; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[15], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[14], range); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[13], range); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[12], range); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[11], range); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[10], range); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[9], range); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[8], range); + step[8] = Av1Transform1dMath.Clamp(output[7] - output[8], range); + step[9] = Av1Transform1dMath.Clamp(output[6] - output[9], range); + step[10] = Av1Transform1dMath.Clamp(output[5] - output[10], range); + step[11] = Av1Transform1dMath.Clamp(output[4] - output[11], range); + step[12] = Av1Transform1dMath.Clamp(output[3] - output[12], range); + step[13] = Av1Transform1dMath.Clamp(output[2] - output[13], range); + step[14] = Av1Transform1dMath.Clamp(output[1] - output[14], range); + step[15] = Av1Transform1dMath.Clamp(output[0] - output[15], range); + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[20], cospi[32], output[27], cosBit); + step[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[21], cospi[32], output[26], cosBit); + step[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[22], cospi[32], output[25], cosBit); + step[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[23], cospi[32], output[24], cosBit); + step[24] = Av1Transform1dMath.HalfButterfly(cospi[32], output[23], cospi[32], output[24], cosBit); + step[25] = Av1Transform1dMath.HalfButterfly(cospi[32], output[22], cospi[32], output[25], cosBit); + step[26] = Av1Transform1dMath.HalfButterfly(cospi[32], output[21], cospi[32], output[26], cosBit); + step[27] = Av1Transform1dMath.HalfButterfly(cospi[32], output[20], cospi[32], output[27], cosBit); + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + + // Stage 9 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[31], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[30], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[29], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[28], range); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[27], range); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[26], range); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[25], range); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[24], range); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[23], range); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[22], range); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[21], range); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[20], range); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[19], range); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[18], range); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[17], range); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[16], range); + output[16] = Av1Transform1dMath.Clamp(step[15] - step[16], range); + output[17] = Av1Transform1dMath.Clamp(step[14] - step[17], range); + output[18] = Av1Transform1dMath.Clamp(step[13] - step[18], range); + output[19] = Av1Transform1dMath.Clamp(step[12] - step[19], range); + output[20] = Av1Transform1dMath.Clamp(step[11] - step[20], range); + output[21] = Av1Transform1dMath.Clamp(step[10] - step[21], range); + output[22] = Av1Transform1dMath.Clamp(step[9] - step[22], range); + output[23] = Av1Transform1dMath.Clamp(step[8] - step[23], range); + output[24] = Av1Transform1dMath.Clamp(step[7] - step[24], range); + output[25] = Av1Transform1dMath.Clamp(step[6] - step[25], range); + output[26] = Av1Transform1dMath.Clamp(step[5] - step[26], range); + output[27] = Av1Transform1dMath.Clamp(step[4] - step[27], range); + output[28] = Av1Transform1dMath.Clamp(step[3] - step[28], range); + output[29] = Av1Transform1dMath.Clamp(step[2] - step[29], range); + output[30] = Av1Transform1dMath.Clamp(step[1] - step[30], range); + output[31] = Av1Transform1dMath.Clamp(step[0] - step[31], range); + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs index c4ec7d437..b56402464 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.Simd.cs @@ -11,13 +11,42 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Dct4Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + // AV1 stores coefficients in frequency order; this permutation restores the order expected by the staged DCT. + output[0] = input[0]; + output[1] = input[2]; + output[2] = input[1]; + output[3] = input[3]; + + // Rotate the even and odd coefficient pairs using the same fixed-point basis as the forward transform. + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + + // The terminal butterflies reconstruct spatial order and clamp every result to the normative stage range. + byte range = stageRange[3]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], range); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], range); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], range); + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs index dede6caee..f83a6c811 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.Simd.cs @@ -11,13 +11,762 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Dct64Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[32]; + output[2] = input[16]; + output[3] = input[48]; + output[4] = input[8]; + output[5] = input[40]; + output[6] = input[24]; + output[7] = input[56]; + output[8] = input[4]; + output[9] = input[36]; + output[10] = input[20]; + output[11] = input[52]; + output[12] = input[12]; + output[13] = input[44]; + output[14] = input[28]; + output[15] = input[60]; + output[16] = input[2]; + output[17] = input[34]; + output[18] = input[18]; + output[19] = input[50]; + output[20] = input[10]; + output[21] = input[42]; + output[22] = input[26]; + output[23] = input[58]; + output[24] = input[6]; + output[25] = input[38]; + output[26] = input[22]; + output[27] = input[54]; + output[28] = input[14]; + output[29] = input[46]; + output[30] = input[30]; + output[31] = input[62]; + output[32] = input[1]; + output[33] = input[33]; + output[34] = input[17]; + output[35] = input[49]; + output[36] = input[9]; + output[37] = input[41]; + output[38] = input[25]; + output[39] = input[57]; + output[40] = input[5]; + output[41] = input[37]; + output[42] = input[21]; + output[43] = input[53]; + output[44] = input[13]; + output[45] = input[45]; + output[46] = input[29]; + output[47] = input[61]; + output[48] = input[3]; + output[49] = input[35]; + output[50] = input[19]; + output[51] = input[51]; + output[52] = input[11]; + output[53] = input[43]; + output[54] = input[27]; + output[55] = input[59]; + output[56] = input[7]; + output[57] = input[39]; + output[58] = input[23]; + output[59] = input[55]; + output[60] = input[15]; + output[61] = input[47]; + output[62] = input[31]; + output[63] = input[63]; + + // Stage 2 rotates the highest odd-frequency coefficient pairs by their pi/128 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = output[8]; + step[9] = output[9]; + step[10] = output[10]; + step[11] = output[11]; + step[12] = output[12]; + step[13] = output[13]; + step[14] = output[14]; + step[15] = output[15]; + step[16] = output[16]; + step[17] = output[17]; + step[18] = output[18]; + step[19] = output[19]; + step[20] = output[20]; + step[21] = output[21]; + step[22] = output[22]; + step[23] = output[23]; + step[24] = output[24]; + step[25] = output[25]; + step[26] = output[26]; + step[27] = output[27]; + step[28] = output[28]; + step[29] = output[29]; + step[30] = output[30]; + step[31] = output[31]; + step[32] = Av1Transform1dMath.HalfButterfly(cospi[63], output[32], -cospi[1], output[63], cosBit); + step[33] = Av1Transform1dMath.HalfButterfly(cospi[31], output[33], -cospi[33], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(cospi[47], output[34], -cospi[17], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(cospi[15], output[35], -cospi[49], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(cospi[55], output[36], -cospi[9], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(cospi[23], output[37], -cospi[41], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(cospi[39], output[38], -cospi[25], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(cospi[7], output[39], -cospi[57], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(cospi[59], output[40], -cospi[5], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(cospi[27], output[41], -cospi[37], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(cospi[43], output[42], -cospi[21], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(cospi[11], output[43], -cospi[53], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(cospi[51], output[44], -cospi[13], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(cospi[19], output[45], -cospi[45], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(cospi[35], output[46], -cospi[29], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(cospi[3], output[47], -cospi[61], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[61], output[47], cospi[3], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[29], output[46], cospi[35], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[45], output[45], cospi[19], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[13], output[44], cospi[51], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[53], output[43], cospi[11], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[21], output[42], cospi[43], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[37], output[41], cospi[27], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[5], output[40], cospi[59], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[57], output[39], cospi[7], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[25], output[38], cospi[39], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[41], output[37], cospi[23], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[9], output[36], cospi[55], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[49], output[35], cospi[15], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[17], output[34], cospi[47], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[33], output[33], cospi[31], output[62], cosBit); + step[63] = Av1Transform1dMath.HalfButterfly(cospi[1], output[32], cospi[63], output[63], cosBit); + + // Stage 3 reconstructs the first nested groups and combines their adjacent odd terms. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = step[4]; + output[5] = step[5]; + output[6] = step[6]; + output[7] = step[7]; + output[8] = step[8]; + output[9] = step[9]; + output[10] = step[10]; + output[11] = step[11]; + output[12] = step[12]; + output[13] = step[13]; + output[14] = step[14]; + output[15] = step[15]; + output[16] = Av1Transform1dMath.HalfButterfly(cospi[62], step[16], -cospi[2], step[31], cosBit); + output[17] = Av1Transform1dMath.HalfButterfly(cospi[30], step[17], -cospi[34], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(cospi[46], step[18], -cospi[18], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(cospi[14], step[19], -cospi[50], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(cospi[54], step[20], -cospi[10], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(cospi[22], step[21], -cospi[42], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(cospi[38], step[22], -cospi[26], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(cospi[6], step[23], -cospi[58], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[58], step[23], cospi[6], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[26], step[22], cospi[38], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[42], step[21], cospi[22], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[10], step[20], cospi[54], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[50], step[19], cospi[14], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[18], step[18], cospi[46], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[34], step[17], cospi[30], step[30], cosBit); + output[31] = Av1Transform1dMath.HalfButterfly(cospi[2], step[16], cospi[62], step[31], cosBit); + output[32] = Av1Transform1dMath.Clamp(step[32] + step[33], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[32] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(-step[34] + step[35], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[34] + step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[37], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[36] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(-step[38] + step[39], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[38] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[41], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[40] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[43], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[42] + step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[44] + step[45], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[44] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(-step[46] + step[47], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[46] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[49], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[48] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[51], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[50] + step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[52] + step[53], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[52] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[55], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[54] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[57], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[56] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[59], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[58] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[60] + step[61], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[60] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(-step[62] + step[63], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[62] + step[63], stageRange[stage]); + + // Stage 4 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = output[4]; + step[5] = output[5]; + step[6] = output[6]; + step[7] = output[7]; + step[8] = Av1Transform1dMath.HalfButterfly(cospi[60], output[8], -cospi[4], output[15], cosBit); + step[9] = Av1Transform1dMath.HalfButterfly(cospi[28], output[9], -cospi[36], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(cospi[44], output[10], -cospi[20], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(cospi[12], output[11], -cospi[52], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[52], output[11], cospi[12], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[20], output[10], cospi[44], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[36], output[9], cospi[28], output[14], cosBit); + step[15] = Av1Transform1dMath.HalfButterfly(cospi[4], output[8], cospi[60], output[15], cosBit); + step[16] = Av1Transform1dMath.Clamp(output[16] + output[17], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[16] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(-output[18] + output[19], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[18] + output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[20] + output[21], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[20] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(-output[22] + output[23], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[22] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[25], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[24] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[27], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[26] + output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[28] + output[29], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[28] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(-output[30] + output[31], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[30] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[33], cospi[60], output[62], cosBit); + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[60], output[34], -cospi[4], output[61], cosBit); + step[35] = output[35]; + step[36] = output[36]; + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[37], cospi[28], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[28], output[38], -cospi[36], output[57], cosBit); + step[39] = output[39]; + step[40] = output[40]; + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[41], cospi[44], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[44], output[42], -cospi[20], output[53], cosBit); + step[43] = output[43]; + step[44] = output[44]; + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[45], cospi[12], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[12], output[46], -cospi[52], output[49], cosBit); + step[47] = output[47]; + step[48] = output[48]; + step[49] = Av1Transform1dMath.HalfButterfly(-cospi[52], output[46], cospi[12], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[12], output[45], cospi[52], output[50], cosBit); + step[51] = output[51]; + step[52] = output[52]; + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[20], output[42], cospi[44], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[44], output[41], cospi[20], output[54], cosBit); + step[55] = output[55]; + step[56] = output[56]; + step[57] = Av1Transform1dMath.HalfButterfly(-cospi[36], output[38], cospi[28], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[28], output[37], cospi[36], output[58], cosBit); + step[59] = output[59]; + step[60] = output[60]; + step[61] = Av1Transform1dMath.HalfButterfly(-cospi[4], output[34], cospi[60], output[61], cosBit); + step[62] = Av1Transform1dMath.HalfButterfly(cospi[60], output[33], cospi[4], output[62], cosBit); + step[63] = output[63]; + + // Stage 5 widens the nested groups through the next butterfly level. + stage++; + output[0] = step[0]; + output[1] = step[1]; + output[2] = step[2]; + output[3] = step[3]; + output[4] = Av1Transform1dMath.HalfButterfly(cospi[56], step[4], -cospi[8], step[7], cosBit); + output[5] = Av1Transform1dMath.HalfButterfly(cospi[24], step[5], -cospi[40], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[40], step[5], cospi[24], step[6], cosBit); + output[7] = Av1Transform1dMath.HalfButterfly(cospi[8], step[4], cospi[56], step[7], cosBit); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[9], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[8] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(-step[10] + step[11], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[10] + step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[13], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[12] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(-step[14] + step[15], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[14] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[17], cospi[56], step[30], cosBit); + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[56], step[18], -cospi[8], step[29], cosBit); + output[19] = step[19]; + output[20] = step[20]; + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[21], cospi[24], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[24], step[22], -cospi[40], step[25], cosBit); + output[23] = step[23]; + output[24] = step[24]; + output[25] = Av1Transform1dMath.HalfButterfly(-cospi[40], step[22], cospi[24], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[24], step[21], cospi[40], step[26], cosBit); + output[27] = step[27]; + output[28] = step[28]; + output[29] = Av1Transform1dMath.HalfButterfly(-cospi[8], step[18], cospi[56], step[29], cosBit); + output[30] = Av1Transform1dMath.HalfButterfly(cospi[56], step[17], cospi[8], step[30], cosBit); + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[35], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[34], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[33] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[32] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(-step[36] + step[39], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(-step[37] + step[38], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[37] + step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[36] + step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[40] + step[43], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[41] + step[42], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[41] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[40] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(-step[44] + step[47], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(-step[45] + step[46], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[45] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[44] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[51], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[50], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[49] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[48] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[55], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[54], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[53] + step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[52] + step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[56] + step[59], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[57] + step[58], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[57] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[56] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(-step[60] + step[63], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(-step[61] + step[62], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[61] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[60] + step[63], stageRange[stage]); + + // Stage 6 rotates the next odd-frequency level while preserving completed low-frequency lanes. + stage++; + step[0] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], cospi[32], output[1], cosBit); + step[1] = Av1Transform1dMath.HalfButterfly(cospi[32], output[0], -cospi[32], output[1], cosBit); + step[2] = Av1Transform1dMath.HalfButterfly(cospi[48], output[2], -cospi[16], output[3], cosBit); + step[3] = Av1Transform1dMath.HalfButterfly(cospi[16], output[2], cospi[48], output[3], cosBit); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[5], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[4] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(-output[6] + output[7], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[6] + output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[9], cospi[48], output[14], cosBit); + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[10], -cospi[16], output[13], cosBit); + step[11] = output[11]; + step[12] = output[12]; + step[13] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[10], cospi[48], output[13], cosBit); + step[14] = Av1Transform1dMath.HalfButterfly(cospi[48], output[9], cospi[16], output[14], cosBit); + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[19], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[18], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[17] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[16] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(-output[20] + output[23], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(-output[21] + output[22], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[21] + output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[20] + output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[24] + output[27], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[25] + output[26], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[25] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[24] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(-output[28] + output[31], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(-output[29] + output[30], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[29] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[28] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[34], cospi[56], output[61], cosBit); + step[35] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[35], cospi[56], output[60], cosBit); + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[36], -cospi[8], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[56], output[37], -cospi[8], output[58], cosBit); + step[38] = output[38]; + step[39] = output[39]; + step[40] = output[40]; + step[41] = output[41]; + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[42], cospi[24], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[43], cospi[24], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[44], -cospi[40], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[24], output[45], -cospi[40], output[50], cosBit); + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[45], cospi[24], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(-cospi[40], output[44], cospi[24], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[24], output[43], cospi[40], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[24], output[42], cospi[40], output[53], cosBit); + step[54] = output[54]; + step[55] = output[55]; + step[56] = output[56]; + step[57] = output[57]; + step[58] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[37], cospi[56], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(-cospi[8], output[36], cospi[56], output[59], cosBit); + step[60] = Av1Transform1dMath.HalfButterfly(cospi[56], output[35], cospi[8], output[60], cosBit); + step[61] = Av1Transform1dMath.HalfButterfly(cospi[56], output[34], cospi[8], output[61], cosBit); + step[62] = output[62]; + step[63] = output[63]; + + // Stage 7 reconstructs the embedded sixteen-point groups and combines adjacent odd terms. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[3], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[2], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[1] - step[2], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[0] - step[3], stageRange[stage]); + output[4] = step[4]; + output[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[5], cospi[32], step[6], cosBit); + output[6] = Av1Transform1dMath.HalfButterfly(cospi[32], step[5], cospi[32], step[6], cosBit); + output[7] = step[7]; + output[8] = Av1Transform1dMath.Clamp(step[8] + step[11], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[10], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[9] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[8] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(-step[12] + step[15], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(-step[13] + step[14], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[13] + step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[12] + step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[18], cospi[48], step[29], cosBit); + output[19] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[19], cospi[48], step[28], cosBit); + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[20], -cospi[16], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[48], step[21], -cospi[16], step[26], cosBit); + output[22] = step[22]; + output[23] = step[23]; + output[24] = step[24]; + output[25] = step[25]; + output[26] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[21], cospi[48], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(-cospi[16], step[20], cospi[48], step[27], cosBit); + output[28] = Av1Transform1dMath.HalfButterfly(cospi[48], step[19], cospi[16], step[28], cosBit); + output[29] = Av1Transform1dMath.HalfButterfly(cospi[48], step[18], cospi[16], step[29], cosBit); + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[39], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[38], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[37], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[36], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[35] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[34] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[33] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[32] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(-step[40] + step[47], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(-step[41] + step[46], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(-step[42] + step[45], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(-step[43] + step[44], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[43] + step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[42] + step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[41] + step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[40] + step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[48] + step[55], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[49] + step[54], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[50] + step[53], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[51] + step[52], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[51] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[50] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[49] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[48] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(-step[56] + step[63], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(-step[57] + step[62], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(-step[58] + step[61], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(-step[59] + step[60], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[59] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[58] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[57] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[56] + step[63], stageRange[stage]); + + // Stage 8 completes the embedded eight-point groups and rotates their odd-frequency pairs. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[7], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[6], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[5], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[4], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[3] - output[4], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[2] - output[5], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[1] - output[6], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[0] - output[7], stageRange[stage]); + step[8] = output[8]; + step[9] = output[9]; + step[10] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[10], cospi[32], output[13], cosBit); + step[11] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[11], cospi[32], output[12], cosBit); + step[12] = Av1Transform1dMath.HalfButterfly(cospi[32], output[11], cospi[32], output[12], cosBit); + step[13] = Av1Transform1dMath.HalfButterfly(cospi[32], output[10], cospi[32], output[13], cosBit); + step[14] = output[14]; + step[15] = output[15]; + step[16] = Av1Transform1dMath.Clamp(output[16] + output[23], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[17] + output[22], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[18] + output[21], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[19] + output[20], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[19] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[18] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[17] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[16] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(-output[24] + output[31], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(-output[25] + output[30], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(-output[26] + output[29], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(-output[27] + output[28], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[27] + output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[26] + output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[25] + output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[24] + output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[36], cospi[48], output[59], cosBit); + step[37] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[37], cospi[48], output[58], cosBit); + step[38] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[38], cospi[48], output[57], cosBit); + step[39] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[39], cospi[48], output[56], cosBit); + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[40], -cospi[16], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[41], -cospi[16], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[42], -cospi[16], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[48], output[43], -cospi[16], output[52], cosBit); + step[44] = output[44]; + step[45] = output[45]; + step[46] = output[46]; + step[47] = output[47]; + step[48] = output[48]; + step[49] = output[49]; + step[50] = output[50]; + step[51] = output[51]; + step[52] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[43], cospi[48], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[42], cospi[48], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[41], cospi[48], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(-cospi[16], output[40], cospi[48], output[55], cosBit); + step[56] = Av1Transform1dMath.HalfButterfly(cospi[48], output[39], cospi[16], output[56], cosBit); + step[57] = Av1Transform1dMath.HalfButterfly(cospi[48], output[38], cospi[16], output[57], cosBit); + step[58] = Av1Transform1dMath.HalfButterfly(cospi[48], output[37], cospi[16], output[58], cosBit); + step[59] = Av1Transform1dMath.HalfButterfly(cospi[48], output[36], cospi[16], output[59], cosBit); + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 9 widens the reconstructed groups through their next butterfly level. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[15], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[14], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[13], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[12], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[11], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[10], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[9], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[8], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[7] - step[8], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[6] - step[9], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[5] - step[10], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[4] - step[11], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[3] - step[12], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[2] - step[13], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[1] - step[14], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[0] - step[15], stageRange[stage]); + output[16] = step[16]; + output[17] = step[17]; + output[18] = step[18]; + output[19] = step[19]; + output[20] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[20], cospi[32], step[27], cosBit); + output[21] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[21], cospi[32], step[26], cosBit); + output[22] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[22], cospi[32], step[25], cosBit); + output[23] = Av1Transform1dMath.HalfButterfly(-cospi[32], step[23], cospi[32], step[24], cosBit); + output[24] = Av1Transform1dMath.HalfButterfly(cospi[32], step[23], cospi[32], step[24], cosBit); + output[25] = Av1Transform1dMath.HalfButterfly(cospi[32], step[22], cospi[32], step[25], cosBit); + output[26] = Av1Transform1dMath.HalfButterfly(cospi[32], step[21], cospi[32], step[26], cosBit); + output[27] = Av1Transform1dMath.HalfButterfly(cospi[32], step[20], cospi[32], step[27], cosBit); + output[28] = step[28]; + output[29] = step[29]; + output[30] = step[30]; + output[31] = step[31]; + output[32] = Av1Transform1dMath.Clamp(step[32] + step[47], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[33] + step[46], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[34] + step[45], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[35] + step[44], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[36] + step[43], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[37] + step[42], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[38] + step[41], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[39] + step[40], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[39] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[38] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[37] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[36] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[35] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[34] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[33] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[32] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(-step[48] + step[63], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(-step[49] + step[62], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(-step[50] + step[61], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(-step[51] + step[60], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(-step[52] + step[59], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(-step[53] + step[58], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(-step[54] + step[57], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(-step[55] + step[56], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[55] + step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[54] + step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[53] + step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[52] + step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[51] + step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[50] + step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[49] + step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[48] + step[63], stageRange[stage]); + + // Stage 10 applies the remaining pi/4 rotations before the terminal spatial merge. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[31], stageRange[stage]); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[30], stageRange[stage]); + step[2] = Av1Transform1dMath.Clamp(output[2] + output[29], stageRange[stage]); + step[3] = Av1Transform1dMath.Clamp(output[3] + output[28], stageRange[stage]); + step[4] = Av1Transform1dMath.Clamp(output[4] + output[27], stageRange[stage]); + step[5] = Av1Transform1dMath.Clamp(output[5] + output[26], stageRange[stage]); + step[6] = Av1Transform1dMath.Clamp(output[6] + output[25], stageRange[stage]); + step[7] = Av1Transform1dMath.Clamp(output[7] + output[24], stageRange[stage]); + step[8] = Av1Transform1dMath.Clamp(output[8] + output[23], stageRange[stage]); + step[9] = Av1Transform1dMath.Clamp(output[9] + output[22], stageRange[stage]); + step[10] = Av1Transform1dMath.Clamp(output[10] + output[21], stageRange[stage]); + step[11] = Av1Transform1dMath.Clamp(output[11] + output[20], stageRange[stage]); + step[12] = Av1Transform1dMath.Clamp(output[12] + output[19], stageRange[stage]); + step[13] = Av1Transform1dMath.Clamp(output[13] + output[18], stageRange[stage]); + step[14] = Av1Transform1dMath.Clamp(output[14] + output[17], stageRange[stage]); + step[15] = Av1Transform1dMath.Clamp(output[15] + output[16], stageRange[stage]); + step[16] = Av1Transform1dMath.Clamp(output[15] - output[16], stageRange[stage]); + step[17] = Av1Transform1dMath.Clamp(output[14] - output[17], stageRange[stage]); + step[18] = Av1Transform1dMath.Clamp(output[13] - output[18], stageRange[stage]); + step[19] = Av1Transform1dMath.Clamp(output[12] - output[19], stageRange[stage]); + step[20] = Av1Transform1dMath.Clamp(output[11] - output[20], stageRange[stage]); + step[21] = Av1Transform1dMath.Clamp(output[10] - output[21], stageRange[stage]); + step[22] = Av1Transform1dMath.Clamp(output[9] - output[22], stageRange[stage]); + step[23] = Av1Transform1dMath.Clamp(output[8] - output[23], stageRange[stage]); + step[24] = Av1Transform1dMath.Clamp(output[7] - output[24], stageRange[stage]); + step[25] = Av1Transform1dMath.Clamp(output[6] - output[25], stageRange[stage]); + step[26] = Av1Transform1dMath.Clamp(output[5] - output[26], stageRange[stage]); + step[27] = Av1Transform1dMath.Clamp(output[4] - output[27], stageRange[stage]); + step[28] = Av1Transform1dMath.Clamp(output[3] - output[28], stageRange[stage]); + step[29] = Av1Transform1dMath.Clamp(output[2] - output[29], stageRange[stage]); + step[30] = Av1Transform1dMath.Clamp(output[1] - output[30], stageRange[stage]); + step[31] = Av1Transform1dMath.Clamp(output[0] - output[31], stageRange[stage]); + step[32] = output[32]; + step[33] = output[33]; + step[34] = output[34]; + step[35] = output[35]; + step[36] = output[36]; + step[37] = output[37]; + step[38] = output[38]; + step[39] = output[39]; + step[40] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[40], cospi[32], output[55], cosBit); + step[41] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[41], cospi[32], output[54], cosBit); + step[42] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[42], cospi[32], output[53], cosBit); + step[43] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[43], cospi[32], output[52], cosBit); + step[44] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[44], cospi[32], output[51], cosBit); + step[45] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[45], cospi[32], output[50], cosBit); + step[46] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[46], cospi[32], output[49], cosBit); + step[47] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[47], cospi[32], output[48], cosBit); + step[48] = Av1Transform1dMath.HalfButterfly(cospi[32], output[47], cospi[32], output[48], cosBit); + step[49] = Av1Transform1dMath.HalfButterfly(cospi[32], output[46], cospi[32], output[49], cosBit); + step[50] = Av1Transform1dMath.HalfButterfly(cospi[32], output[45], cospi[32], output[50], cosBit); + step[51] = Av1Transform1dMath.HalfButterfly(cospi[32], output[44], cospi[32], output[51], cosBit); + step[52] = Av1Transform1dMath.HalfButterfly(cospi[32], output[43], cospi[32], output[52], cosBit); + step[53] = Av1Transform1dMath.HalfButterfly(cospi[32], output[42], cospi[32], output[53], cosBit); + step[54] = Av1Transform1dMath.HalfButterfly(cospi[32], output[41], cospi[32], output[54], cosBit); + step[55] = Av1Transform1dMath.HalfButterfly(cospi[32], output[40], cospi[32], output[55], cosBit); + step[56] = output[56]; + step[57] = output[57]; + step[58] = output[58]; + step[59] = output[59]; + step[60] = output[60]; + step[61] = output[61]; + step[62] = output[62]; + step[63] = output[63]; + + // Stage 11 merges the even and odd halves into spatial order and clamps every result. + stage++; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[63], stageRange[stage]); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[62], stageRange[stage]); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[61], stageRange[stage]); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[60], stageRange[stage]); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[59], stageRange[stage]); + output[5] = Av1Transform1dMath.Clamp(step[5] + step[58], stageRange[stage]); + output[6] = Av1Transform1dMath.Clamp(step[6] + step[57], stageRange[stage]); + output[7] = Av1Transform1dMath.Clamp(step[7] + step[56], stageRange[stage]); + output[8] = Av1Transform1dMath.Clamp(step[8] + step[55], stageRange[stage]); + output[9] = Av1Transform1dMath.Clamp(step[9] + step[54], stageRange[stage]); + output[10] = Av1Transform1dMath.Clamp(step[10] + step[53], stageRange[stage]); + output[11] = Av1Transform1dMath.Clamp(step[11] + step[52], stageRange[stage]); + output[12] = Av1Transform1dMath.Clamp(step[12] + step[51], stageRange[stage]); + output[13] = Av1Transform1dMath.Clamp(step[13] + step[50], stageRange[stage]); + output[14] = Av1Transform1dMath.Clamp(step[14] + step[49], stageRange[stage]); + output[15] = Av1Transform1dMath.Clamp(step[15] + step[48], stageRange[stage]); + output[16] = Av1Transform1dMath.Clamp(step[16] + step[47], stageRange[stage]); + output[17] = Av1Transform1dMath.Clamp(step[17] + step[46], stageRange[stage]); + output[18] = Av1Transform1dMath.Clamp(step[18] + step[45], stageRange[stage]); + output[19] = Av1Transform1dMath.Clamp(step[19] + step[44], stageRange[stage]); + output[20] = Av1Transform1dMath.Clamp(step[20] + step[43], stageRange[stage]); + output[21] = Av1Transform1dMath.Clamp(step[21] + step[42], stageRange[stage]); + output[22] = Av1Transform1dMath.Clamp(step[22] + step[41], stageRange[stage]); + output[23] = Av1Transform1dMath.Clamp(step[23] + step[40], stageRange[stage]); + output[24] = Av1Transform1dMath.Clamp(step[24] + step[39], stageRange[stage]); + output[25] = Av1Transform1dMath.Clamp(step[25] + step[38], stageRange[stage]); + output[26] = Av1Transform1dMath.Clamp(step[26] + step[37], stageRange[stage]); + output[27] = Av1Transform1dMath.Clamp(step[27] + step[36], stageRange[stage]); + output[28] = Av1Transform1dMath.Clamp(step[28] + step[35], stageRange[stage]); + output[29] = Av1Transform1dMath.Clamp(step[29] + step[34], stageRange[stage]); + output[30] = Av1Transform1dMath.Clamp(step[30] + step[33], stageRange[stage]); + output[31] = Av1Transform1dMath.Clamp(step[31] + step[32], stageRange[stage]); + output[32] = Av1Transform1dMath.Clamp(step[31] - step[32], stageRange[stage]); + output[33] = Av1Transform1dMath.Clamp(step[30] - step[33], stageRange[stage]); + output[34] = Av1Transform1dMath.Clamp(step[29] - step[34], stageRange[stage]); + output[35] = Av1Transform1dMath.Clamp(step[28] - step[35], stageRange[stage]); + output[36] = Av1Transform1dMath.Clamp(step[27] - step[36], stageRange[stage]); + output[37] = Av1Transform1dMath.Clamp(step[26] - step[37], stageRange[stage]); + output[38] = Av1Transform1dMath.Clamp(step[25] - step[38], stageRange[stage]); + output[39] = Av1Transform1dMath.Clamp(step[24] - step[39], stageRange[stage]); + output[40] = Av1Transform1dMath.Clamp(step[23] - step[40], stageRange[stage]); + output[41] = Av1Transform1dMath.Clamp(step[22] - step[41], stageRange[stage]); + output[42] = Av1Transform1dMath.Clamp(step[21] - step[42], stageRange[stage]); + output[43] = Av1Transform1dMath.Clamp(step[20] - step[43], stageRange[stage]); + output[44] = Av1Transform1dMath.Clamp(step[19] - step[44], stageRange[stage]); + output[45] = Av1Transform1dMath.Clamp(step[18] - step[45], stageRange[stage]); + output[46] = Av1Transform1dMath.Clamp(step[17] - step[46], stageRange[stage]); + output[47] = Av1Transform1dMath.Clamp(step[16] - step[47], stageRange[stage]); + output[48] = Av1Transform1dMath.Clamp(step[15] - step[48], stageRange[stage]); + output[49] = Av1Transform1dMath.Clamp(step[14] - step[49], stageRange[stage]); + output[50] = Av1Transform1dMath.Clamp(step[13] - step[50], stageRange[stage]); + output[51] = Av1Transform1dMath.Clamp(step[12] - step[51], stageRange[stage]); + output[52] = Av1Transform1dMath.Clamp(step[11] - step[52], stageRange[stage]); + output[53] = Av1Transform1dMath.Clamp(step[10] - step[53], stageRange[stage]); + output[54] = Av1Transform1dMath.Clamp(step[9] - step[54], stageRange[stage]); + output[55] = Av1Transform1dMath.Clamp(step[8] - step[55], stageRange[stage]); + output[56] = Av1Transform1dMath.Clamp(step[7] - step[56], stageRange[stage]); + output[57] = Av1Transform1dMath.Clamp(step[6] - step[57], stageRange[stage]); + output[58] = Av1Transform1dMath.Clamp(step[5] - step[58], stageRange[stage]); + output[59] = Av1Transform1dMath.Clamp(step[4] - step[59], stageRange[stage]); + output[60] = Av1Transform1dMath.Clamp(step[3] - step[60], stageRange[stage]); + output[61] = Av1Transform1dMath.Clamp(step[2] - step[61], stageRange[stage]); + output[62] = Av1Transform1dMath.Clamp(step[1] - step[62], stageRange[stage]); + output[63] = Av1Transform1dMath.Clamp(step[0] - step[63], stageRange[stage]); + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs index 3718d317b..2531b3069 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.Simd.cs @@ -11,13 +11,82 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; internal readonly partial struct Av1Dct8Inverse1dOperator { /// - /// Applies the transform to eight independent axes in parallel. + /// Applies the transform to sixteen independent axes in parallel. /// /// The source values for the parallel transform axes. /// The destination values for the parallel transform axes. /// The fixed stage storage for the parallel transform axes. /// The fixed-point precision of the cosine constants. /// The signed-bit range assigned to each transform stage. + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + ReadOnlySpan cospi = Av1SinusConstants.CosinusPi(cosBit); + int stage = 0; + + // Stage 1 permutes frequency-ordered coefficients into the recursive DCT factorization order. + stage++; + output[0] = input[0]; + output[1] = input[4]; + output[2] = input[2]; + output[3] = input[6]; + output[4] = input[1]; + output[5] = input[5]; + output[6] = input[3]; + output[7] = input[7]; + + // Stage 2 rotates the odd-frequency coefficient pairs by their pi/16 angles. + stage++; + step[0] = output[0]; + step[1] = output[1]; + step[2] = output[2]; + step[3] = output[3]; + step[4] = Av1Transform1dMath.HalfButterfly(cospi[56], output[4], -cospi[8], output[7], cosBit); + step[5] = Av1Transform1dMath.HalfButterfly(cospi[24], output[5], -cospi[40], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[40], output[5], cospi[24], output[6], cosBit); + step[7] = Av1Transform1dMath.HalfButterfly(cospi[8], output[4], cospi[56], output[7], cosBit); + + // Stage 3 reconstructs the even four-point DCT and combines adjacent odd terms. + stage++; + byte range = stageRange[stage]; + output[0] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], cospi[32], step[1], cosBit); + output[1] = Av1Transform1dMath.HalfButterfly(cospi[32], step[0], -cospi[32], step[1], cosBit); + output[2] = Av1Transform1dMath.HalfButterfly(cospi[48], step[2], -cospi[16], step[3], cosBit); + output[3] = Av1Transform1dMath.HalfButterfly(cospi[16], step[2], cospi[48], step[3], cosBit); + output[4] = Av1Transform1dMath.Clamp(step[4] + step[5], range); + output[5] = Av1Transform1dMath.Clamp(step[4] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[7] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[6] + step[7], range); + + // Stage 4 completes the even butterflies and applies the remaining pi/4 odd rotation. + stage++; + step[0] = Av1Transform1dMath.Clamp(output[0] + output[3], range); + step[1] = Av1Transform1dMath.Clamp(output[1] + output[2], range); + step[2] = Av1Transform1dMath.Clamp(output[1] - output[2], range); + step[3] = Av1Transform1dMath.Clamp(output[0] - output[3], range); + step[4] = output[4]; + step[5] = Av1Transform1dMath.HalfButterfly(-cospi[32], output[5], cospi[32], output[6], cosBit); + step[6] = Av1Transform1dMath.HalfButterfly(cospi[32], output[5], cospi[32], output[6], cosBit); + step[7] = output[7]; + + // Stage 5 merges the even and odd halves into spatial order and clamps every result. + stage++; + range = stageRange[stage]; + output[0] = Av1Transform1dMath.Clamp(step[0] + step[7], range); + output[1] = Av1Transform1dMath.Clamp(step[1] + step[6], range); + output[2] = Av1Transform1dMath.Clamp(step[2] + step[5], range); + output[3] = Av1Transform1dMath.Clamp(step[3] + step[4], range); + output[4] = Av1Transform1dMath.Clamp(step[3] - step[4], range); + output[5] = Av1Transform1dMath.Clamp(step[2] - step[5], range); + output[6] = Av1Transform1dMath.Clamp(step[1] - step[6], range); + output[7] = Av1Transform1dMath.Clamp(step[0] - step[7], range); + } + + /// public static void Transform( ref Av1TransformVector> input, ref Av1TransformVector> output, diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs index 29ebc963c..ee3cea37e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1IdentityInverse1dOperators.Simd.cs @@ -37,6 +37,20 @@ internal readonly partial struct Av1Identity4Inverse1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -71,6 +85,20 @@ internal readonly partial struct Av1Identity8Inverse1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 8, 2, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -105,6 +133,20 @@ internal readonly partial struct Av1Identity16Inverse1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + _ = step; + _ = cosBit; + _ = stageRange; + } } /// @@ -139,4 +181,18 @@ internal readonly partial struct Av1Identity32Inverse1dOperator _ = cosBit; _ = stageRange; } + + /// + public static void Transform( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + ref Av1TransformVector> step, + int cosBit, + Av1TransformStageRange stageRange) + { + Av1IdentityTransform1d.Transform(ref input, ref output, 32, 4, 0); + _ = step; + _ = cosBit; + _ = stageRange; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs index 96b21f3f2..fb8c179c0 100644 --- a/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; @@ -11,7 +12,7 @@ namespace SixLabors.ImageSharp.Formats.Heif; /// Decodes a single AV1-coded HEIF image item. /// /// The destination pixel type. -internal class Av1HeifItemDecoder : IHeifItemDecoder +internal class Av1HeifItemDecoder : IHeifItemDecoder, IHeifAlphaItemDecoder where TPixel : unmanaged, IPixel { /// @@ -43,6 +44,70 @@ internal class Av1HeifItemDecoder : IHeifItemDecoder CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); + Av1CodecConfiguration codecConfiguration = ValidateItemData( + options, + item, + data, + out HeifContentLightLevel? obuContentLightLevel, + out HeifMasteringDisplayColorVolume? obuMasteringDisplayColorVolume); + + using Av1Decoder decoder = new(options.Configuration); + Image image = decoder.Decode(data, colorProfile, codecConfiguration); + HeifMetadata metadata = image.Metadata.GetHeifMetadata(); + metadata.CompressionMethod = this.CompressionMethod; + metadata.BitDepth = codecConfiguration.BitDepth; + metadata.IsMonochrome = codecConfiguration.IsMonochrome; + metadata.ContentLightLevel = item.ContentLightLevel ?? obuContentLightLevel; + metadata.MasteringDisplayColorVolume = item.MasteringDisplayColorVolume ?? obuMasteringDisplayColorVolume; + return image; + } + + /// + public void DecodeAlphaItemData( + DecoderOptions options, + HeifItem item, + Span data, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + Av1CodecConfiguration codecConfiguration = ValidateItemData(options, item, data, out _, out _); + if (!codecConfiguration.IsMonochrome) + { + throw new InvalidImageContentException($"AV1 alpha image item {item.Id} is not monochrome."); + } + + using Av1Decoder decoder = new(options.Configuration); + decoder.DecodeAlpha( + data, + item.CicpProfile, + codecConfiguration, + default, + destination, + outputSize, + destinationRectangle, + premultiplied); + } + + /// + /// Validates an AV1 item description against its encoded payload and returns the required codec configuration. + /// + /// The general options governing the containing HEIF decode. + /// The AV1 image item being validated. + /// The encoded AV1 payload. + /// Receives content-light metadata found in the AV1 payload. + /// Receives mastering-display metadata found in the AV1 payload. + /// The validated item-associated AV1 codec configuration. + private static Av1CodecConfiguration ValidateItemData( + DecoderOptions options, + HeifItem item, + Span data, + out HeifContentLightLevel? obuContentLightLevel, + out HeifMasteringDisplayColorVolume? obuMasteringDisplayColorVolume) + { Av1CodecConfiguration codecConfiguration = item.Av1CodecConfiguration ?? throw new InvalidImageContentException($"AV1 image item {item.Id} has no codec configuration property."); @@ -63,17 +128,9 @@ internal class Av1HeifItemDecoder : IHeifItemDecoder item.ContentLightLevel, item.MasteringDisplayColorVolume, options, - out HeifContentLightLevel? obuContentLightLevel, - out HeifMasteringDisplayColorVolume? obuMasteringDisplayColorVolume); + out obuContentLightLevel, + out obuMasteringDisplayColorVolume); - using Av1Decoder decoder = new(options.Configuration); - Image image = decoder.Decode(data, colorProfile, codecConfiguration); - HeifMetadata metadata = image.Metadata.GetHeifMetadata(); - metadata.CompressionMethod = this.CompressionMethod; - metadata.BitDepth = codecConfiguration.BitDepth; - metadata.IsMonochrome = codecConfiguration.IsMonochrome; - metadata.ContentLightLevel = item.ContentLightLevel ?? obuContentLightLevel; - metadata.MasteringDisplayColorVolume = item.MasteringDisplayColorVolume ?? obuMasteringDisplayColorVolume; - return image; + return codecConfiguration; } } diff --git a/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaCompositor.cs b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaCompositor.cs new file mode 100644 index 000000000..d37b5eabe --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaCompositor.cs @@ -0,0 +1,173 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers; +using System.Numerics.Tensors; +using SixLabors.ImageSharp.Formats.Heif.Components; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing.Processors.Transforms; + +namespace SixLabors.ImageSharp.Formats.Heif.Components.Alpha; + +/// +/// Composes a reconstructed HEIF luma plane directly into the alpha channel of a packed destination frame. +/// +internal static class HeifPlanarAlphaCompositor +{ + /// + /// Composes a native codec luma plane into a destination image region without materializing an intermediate image. + /// + /// The destination pixel type. + /// The codec adapter exposing the reconstructed component planes. + /// The native unsigned sample storage type. + /// The SIMD widening operations for the sample type. + /// The configuration used for pooled allocation and pixel conversion. + /// The native reconstructed component planes. + /// The packed destination frame receiving alpha values. + /// The resolved H.273 component-range parameters. + /// The visible luma rectangle within the reconstructed plane. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the top-left portion of the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + public static void Compose( + Configuration configuration, + TBuffer buffer, + ImageFrame destination, + in HeifColorConversionParameters parameters, + Rectangle sourceRectangle, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TLoader : struct, IHeifSampleConverter + { + int sourceWidth = sourceRectangle.Width; + int sourceHeight = sourceRectangle.Height; + int outputWidth = outputSize.Width; + int outputHeight = outputSize.Height; + int composedWidth = destinationRectangle.Width; + int composedHeight = destinationRectangle.Height; + + if (sourceWidth == outputWidth && sourceHeight == outputHeight) + { + using IMemoryOwner componentOwner = configuration.MemoryAllocator.Allocate(composedWidth); + using IMemoryOwner alphaOwner = configuration.MemoryAllocator.Allocate(composedWidth); + using IMemoryOwner colorOwner = configuration.MemoryAllocator.Allocate(composedWidth); + Span alpha = componentOwner.GetSpan()[..composedWidth]; + Span packedAlpha = alphaOwner.GetSpan()[..composedWidth]; + Span packedColor = colorOwner.GetSpan()[..composedWidth]; + + // The overwhelmingly common path reads the codec plane once and immediately packs the corresponding + // destination row. No resize maps or full-plane staging are required. + for (int y = 0; y < composedHeight; y++) + { + ReadOnlySpan source = buffer.GetLumaRowSpan(sourceRectangle.Y + y).Slice(sourceRectangle.X, composedWidth); + NormalizeAlphaRow(source, alpha, in parameters); + ApplyAlphaRow(configuration, destination, destinationRectangle.X, destinationRectangle.Y + y, alpha, packedAlpha, packedColor, premultiplied); + } + + return; + } + + // Alpha scaling must match KnownResamplers.Box. That public instance is exposed as IResampler, while + // ResizeKernelMap requires the concrete struct so Radius and GetValue remain statically dispatched. + // BoxResampler is stateless, making its default value behaviorally identical to the known instance. + BoxResampler boxResampler = default; + using ResizeKernelMap horizontalKernels = ResizeKernelMap.Calculate(in boxResampler, outputWidth, sourceWidth, configuration.MemoryAllocator); + using ResizeKernelMap verticalKernels = ResizeKernelMap.Calculate(in boxResampler, outputHeight, sourceHeight, configuration.MemoryAllocator); + using HeifPlanarAlphaResizeWorker worker = new( + configuration, + buffer, + destination, + in parameters, + sourceRectangle, + destinationRectangle, + horizontalKernels, + verticalKernels, + premultiplied); + + worker.Compose(); + } + + /// + /// Widens and normalizes one native luma row to unbounded alpha values before packing or resampling. + /// + /// The native unsigned sample storage type. + /// The SIMD widening operations for the sample type. + /// The native luma samples. + /// The normalized alpha samples. + /// The resolved H.273 component-range parameters. + public static void NormalizeAlphaRow( + ReadOnlySpan source, + Span destination, + in HeifColorConversionParameters parameters) + where TSample : unmanaged + where TLoader : struct, IHeifSampleConverter + { + HeifSampleConversion.ConvertSamplesToFloat(source, destination); + + // Alpha auxiliaries use the luma code-value range but no color matrix. TensorPrimitives keeps this bulk + // normalization SIMD-first on every supported architecture and clamps before resampling, matching the + // established conversion to a bounded L16 plane. + TensorPrimitives.Subtract(destination, parameters.LumaBias, destination); + TensorPrimitives.Multiply(destination, 1F / parameters.LumaScale, destination); + TensorPrimitives.Clamp(destination, 0F, 1F, destination); + } + + /// + /// Packs and composes one normalized alpha row into the destination frame. + /// + /// The destination pixel type. + /// The configuration used for pixel conversion. + /// The packed destination frame receiving alpha values. + /// The horizontal start of the destination region. + /// The destination row receiving alpha values. + /// The normalized alpha samples. + /// The reusable 16-bit alpha packing row. + /// The reusable high-bit-depth destination color row. + /// Whether stored color samples must be converted to unassociated alpha. + public static void ApplyAlphaRow( + Configuration configuration, + ImageFrame destination, + int destinationX, + int destinationY, + ReadOnlySpan alpha, + Span packedAlpha, + Span packedColor, + bool premultiplied) + where TPixel : unmanaged, IPixel + { + int width = alpha.Length; + Span destinationRow = destination.PixelBuffer.DangerousGetRowSpan(destinationY).Slice(destinationX, width); + PixelOperations pixelOperations = PixelOperations.Instance; + + HeifSampleConversion.PackL16(alpha, packedAlpha); + pixelOperations.ToRgba64(configuration, destinationRow, packedColor); + if (premultiplied) + { + for (int x = 0; x < width; x++) + { + Rgba64 pixel = packedColor[x]; + pixel.A = packedAlpha[x].PackedValue; + + // Transparent associated samples have no recoverable color. Nonzero samples use the pixel type's + // established conversion so unassociation retains ImageSharp's clamping and rounding behavior. + packedColor[x] = pixel.A == 0 + ? new Rgba64(0, 0, 0, 0) + : Rgba64.FromAssociatedScaledVector4(pixel.ToScaledVector4()); + } + } + else + { + for (int x = 0; x < width; x++) + { + packedColor[x].A = packedAlpha[x].PackedValue; + } + } + + pixelOperations.FromRgba64(configuration, packedColor, destinationRow); + } +} diff --git a/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaResizeWorker.cs b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaResizeWorker.cs new file mode 100644 index 000000000..4b5424092 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaResizeWorker.cs @@ -0,0 +1,323 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing.Processors.Transforms; + +namespace SixLabors.ImageSharp.Formats.Heif.Components.Alpha; + +/// +/// Resizes a native HEIF luma plane and composes the result as alpha using a bounded sliding window. +/// +/// The destination pixel type. +/// The codec adapter exposing the reconstructed component planes. +/// The native unsigned sample storage type. +/// The SIMD widening operations for the sample type. +internal sealed class HeifPlanarAlphaResizeWorker : IDisposable + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TLoader : struct, IHeifSampleConverter +{ + /// + /// The configuration used for pooled allocation and pixel conversion. + /// + private readonly Configuration configuration; + + /// + /// The codec-native component planes. + /// + private readonly TBuffer buffer; + + /// + /// The packed color frame receiving alpha values. + /// + private readonly ImageFrame destination; + + /// + /// The resolved H.273 component-range parameters. + /// + private readonly HeifColorConversionParameters parameters; + + /// + /// The visible luma rectangle within the reconstructed plane. + /// + private readonly Rectangle sourceRectangle; + + /// + /// The destination region receiving the resized alpha plane. + /// + private readonly Rectangle destinationRectangle; + + /// + /// The horizontal box-filter kernels for the full presented width. + /// + private readonly ResizeKernelMap horizontalKernels; + + /// + /// The vertical box-filter kernels for the full presented height. + /// + private readonly ResizeKernelMap verticalKernels; + + /// + /// The transposed horizontally filtered rows retained by the sliding window. + /// + private readonly Buffer2D transposedFirstPassBuffer; + + /// + /// The reusable normalized source or resized destination row. + /// + private readonly IMemoryOwner componentOwner; + + /// + /// The reusable replicated source row consumed by the shared resize kernels. + /// + private readonly IMemoryOwner sourceVectorOwner; + + /// + /// The reusable 16-bit source and destination alpha packing row. + /// + private readonly IMemoryOwner alphaOwner; + + /// + /// The reusable high-bit-depth destination color row. + /// + private readonly IMemoryOwner colorOwner; + + /// + /// Whether stored color samples must be converted to unassociated alpha. + /// + private readonly bool premultiplied; + + /// + /// The number of source rows retained when the window advances. + /// + private readonly int windowBandHeight; + + /// + /// The total number of source rows retained by the bounded working window. + /// + private readonly int workerHeight; + + /// + /// The source-row interval currently represented by the transposed first-pass buffer. + /// + private RowInterval currentWindow; + + /// + /// Initializes a new instance of the class. + /// + /// The configuration used for pooled allocation and pixel conversion. + /// The codec-native component planes. + /// The packed color frame receiving alpha values. + /// The resolved H.273 component-range parameters. + /// The visible luma rectangle within the reconstructed plane. + /// The destination region receiving the top-left portion of the presented alpha plane. + /// The horizontal box-filter kernels for the full presented width. + /// The vertical box-filter kernels for the full presented height. + /// Whether stored color samples must be converted to unassociated alpha. + public HeifPlanarAlphaResizeWorker( + Configuration configuration, + TBuffer buffer, + ImageFrame destination, + in HeifColorConversionParameters parameters, + Rectangle sourceRectangle, + Rectangle destinationRectangle, + ResizeKernelMap horizontalKernels, + ResizeKernelMap verticalKernels, + bool premultiplied) + { + this.configuration = configuration; + this.buffer = buffer; + this.destination = destination; + this.parameters = parameters; + this.sourceRectangle = sourceRectangle; + this.destinationRectangle = destinationRectangle; + this.premultiplied = premultiplied; + + this.horizontalKernels = horizontalKernels; + this.verticalKernels = verticalKernels; + + // Retaining one complete maximum-diameter band is sufficient for every vertical kernel that crosses a + // window boundary. Those first-pass rows can be copied forward instead of normalized and filtered again. + this.windowBandHeight = this.verticalKernels.MaxDiameter; + + // As in ResizeWorker, the first pass is stored transposed as [destination X][source Y]. Bounding the source-Y + // dimension by the configured working-buffer limit keeps memory independent of the complete alpha-plane size. + int workingBufferLimitInBytes = Math.Min( + configuration.WorkingBufferSizeHintInBytes, + configuration.MemoryAllocator.GetBufferCapacityInBytes()); + + int windowBandCount = ResizeHelper.CalculateResizeWorkerHeightInWindowBands( + this.windowBandHeight, + destinationRectangle.Width, + workingBufferLimitInBytes); + + // A whole number of bands lets Slide retain exactly one overlap band and fill the remaining window with rows + // that have not entered the first pass before. + this.workerHeight = Math.Min(sourceRectangle.Height, windowBandCount * this.windowBandHeight); + this.transposedFirstPassBuffer = configuration.MemoryAllocator.Allocate2D( + this.workerHeight, + destinationRectangle.Width, + preferContiguosImageBuffers: true, + options: AllocationOptions.Clean); + + this.componentOwner = configuration.MemoryAllocator.Allocate(Math.Max(sourceRectangle.Width, destinationRectangle.Width)); + this.sourceVectorOwner = configuration.MemoryAllocator.Allocate(sourceRectangle.Width); + this.alphaOwner = configuration.MemoryAllocator.Allocate(Math.Max(sourceRectangle.Width, destinationRectangle.Width)); + this.colorOwner = configuration.MemoryAllocator.Allocate(destinationRectangle.Width); + this.currentWindow = new RowInterval(0, this.workerHeight); + } + + /// + /// Releases all allocator-owned working buffers. + /// + public void Dispose() + { + this.transposedFirstPassBuffer.Dispose(); + this.componentOwner.Dispose(); + this.sourceVectorOwner.Dispose(); + this.alphaOwner.Dispose(); + this.colorOwner.Dispose(); + } + + /// + /// Resizes and composes the complete requested destination rectangle. + /// + public void Compose() + { + // Populate the horizontal first pass for the initial bounded source-row interval. Later windows retain their + // overlap and calculate only newly entering rows. + this.CalculateFirstPassValues(this.currentWindow); + + Span transposed = this.transposedFirstPassBuffer.DangerousGetSingleSpan(); + Span resizedAlpha = this.componentOwner.GetSpan()[..this.destinationRectangle.Width]; + Span packedAlpha = this.alphaOwner.GetSpan()[..this.destinationRectangle.Width]; + Span packedColor = this.colorOwner.GetSpan()[..this.destinationRectangle.Width]; + ReadOnlySpan verticalKernelSpan = this.verticalKernels.GetKernelSpan(); + ref ResizeKernel verticalKernelBase = ref MemoryMarshal.GetReference(verticalKernelSpan); + ref float resizedAlphaBase = ref MemoryMarshal.GetReference(resizedAlpha); + int currentWindowMin = this.currentWindow.Min; + int currentWindowMax = this.currentWindow.Max; + nuint width = (uint)this.destinationRectangle.Width; + nuint workerHeight = (uint)this.workerHeight; + nuint twoWorkerHeights = workerHeight * 2; + + for (int y = 0; y < this.destinationRectangle.Height; y++) + { + ref ResizeKernel kernel = ref Unsafe.Add(ref verticalKernelBase, y); + int kernelEnd = kernel.StartIndex + kernel.Length; + + // Destination kernels advance monotonically through source Y. Slide until the complete kernel lies in + // the cached first-pass interval; the retained overlap prevents any shared source row being recalculated. + while (kernelEnd > currentWindowMax) + { + this.Slide(); + currentWindowMin = this.currentWindow.Min; + currentWindowMax = this.currentWindow.Max; + } + + // Values for one destination X are contiguous along source Y in the transposed buffer. ConvolveCore + // therefore reads the vertical kernel without gathers, while workerHeight advances to the next X column. + ref Vector4 column = ref transposed[kernel.StartIndex - currentWindowMin]; + nuint x = 0; + for (; x + 1 < width; x += 2) + { + Unsafe.Add(ref resizedAlphaBase, x) = kernel.ConvolveCore(ref column).X; + ref Vector4 nextColumn = ref Unsafe.Add(ref column, workerHeight); + Unsafe.Add(ref resizedAlphaBase, x + 1) = kernel.ConvolveCore(ref nextColumn).X; + column = ref Unsafe.Add(ref column, twoWorkerHeights); + } + + if (x < width) + { + Unsafe.Add(ref resizedAlphaBase, x) = kernel.ConvolveCore(ref column).X; + } + + HeifPlanarAlphaCompositor.ApplyAlphaRow( + this.configuration, + this.destination, + this.destinationRectangle.X, + this.destinationRectangle.Y + y, + resizedAlpha, + packedAlpha, + packedColor, + this.premultiplied); + } + } + + /// + /// Advances the bounded working window while preserving its overlapping source-row band. + /// + private void Slide() + { + // The old bottom band is the only set of first-pass rows that a future kernel can share with the new window. + // Its height equals the largest vertical-kernel diameter, covering the maximum possible overlap. + int minimumY = this.currentWindow.Max - this.windowBandHeight; + int maximumY = Math.Min(minimumY + this.workerHeight, this.sourceRectangle.Height); + + // Buffer2D columns represent source Y because the first pass is transposed. Move the retained bottom band to + // offset zero for every destination-X column before replacing the remainder of the window. + this.transposedFirstPassBuffer.DangerousCopyColumns( + this.workerHeight - this.windowBandHeight, + 0, + this.windowBandHeight); + + this.currentWindow = new RowInterval(minimumY, maximumY); + + // The retained band already contains normalized and horizontally filtered values. Only rows below it are new. + this.CalculateFirstPassValues(this.currentWindow.Slice(this.windowBandHeight)); + } + + /// + /// Normalizes and horizontally filters the source rows entering the current working window. + /// + /// The source-row interval requiring first-pass values. + private void CalculateFirstPassValues(RowInterval interval) + { + int sourceWidth = this.sourceRectangle.Width; + int destinationWidth = this.destinationRectangle.Width; + Span normalized = this.componentOwner.GetSpan()[..sourceWidth]; + Span sourceAlpha = this.alphaOwner.GetSpan()[..sourceWidth]; + Span sourceVectors = this.sourceVectorOwner.GetSpan()[..sourceWidth]; + Span transposed = this.transposedFirstPassBuffer.DangerousGetSingleSpan(); + ReadOnlySpan horizontalKernelSpan = this.horizontalKernels.GetKernelSpan(); + ref ResizeKernel horizontalKernelBase = ref MemoryMarshal.GetReference(horizontalKernelSpan); + nuint workerHeight = (uint)this.workerHeight; + + for (int y = interval.Min; y < interval.Max; y++) + { + ReadOnlySpan source = this.buffer.GetLumaRowSpan(this.sourceRectangle.Y + y).Slice(this.sourceRectangle.X, sourceWidth); + HeifPlanarAlphaCompositor.NormalizeAlphaRow(source, normalized, in this.parameters); + + // ResizeKernel is the same SIMD convolution primitive used by the general image resizer. Replicating alpha + // into Vector4 lets that kernel operate on the planar row, while the L16 round trip preserves the result of + // the removed Image path without materializing the complete alpha image. + HeifSampleConversion.PackL16(normalized, sourceAlpha); + PixelOperations.Instance.ToVector4(this.configuration, sourceAlpha, sourceVectors, PixelConversionModifiers.Scale); + + // The source row is horizontally filtered once for every destination X and stored at [X][window Y]. A + // vertical kernel can then reuse this first-pass row wherever adjacent destination kernels overlap it. + ref Vector4 firstPass = ref transposed[y - this.currentWindow.Min]; + int x = 0; + for (; x + 1 < destinationWidth; x += 2) + { + ref ResizeKernel kernel0 = ref Unsafe.Add(ref horizontalKernelBase, x); + ref ResizeKernel kernel1 = ref Unsafe.Add(ref horizontalKernelBase, x + 1); + Unsafe.Add(ref firstPass, (nuint)x * workerHeight) = kernel0.Convolve(sourceVectors); + Unsafe.Add(ref firstPass, (nuint)(x + 1) * workerHeight) = kernel1.Convolve(sourceVectors); + } + + if (x < destinationWidth) + { + ref ResizeKernel kernel = ref Unsafe.Add(ref horizontalKernelBase, x); + Unsafe.Add(ref firstPass, (nuint)x * workerHeight) = kernel.Convolve(sourceVectors); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Components/Alpha/IHeifAlphaItemDecoder.cs b/src/ImageSharp/Formats/Heif/Components/Alpha/IHeifAlphaItemDecoder.cs new file mode 100644 index 000000000..bb4e1cf87 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Components/Alpha/IHeifAlphaItemDecoder.cs @@ -0,0 +1,35 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.PixelFormats; + +namespace SixLabors.ImageSharp.Formats.Heif.Components.Alpha; + +/// +/// Decodes one coded HEIF auxiliary alpha item directly into a packed color frame. +/// +/// The destination color pixel type. +internal interface IHeifAlphaItemDecoder + where TPixel : unmanaged, IPixel +{ + /// + /// Decodes and composes one coded auxiliary alpha item. + /// + /// The general options governing the containing HEIF decode. + /// The auxiliary image item whose encoded payload is being decoded. + /// The encoded auxiliary payload. + /// The packed color frame receiving alpha values. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the top-left portion of the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + /// The token used to cancel the payload decode. + public void DecodeAlphaItemData( + DecoderOptions options, + HeifItem item, + Span data, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied, + CancellationToken cancellationToken); +} diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs index 2f27d104a..082933fb1 100644 --- a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs @@ -7,7 +7,6 @@ using SixLabors.ImageSharp.Advanced; using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; -using static SixLabors.ImageSharp.Formats.Heif.Components.HeifColorConverterBase; namespace SixLabors.ImageSharp.Formats.Heif.Components; @@ -64,7 +63,7 @@ internal static class HeifPlanarColorConverter return; } - ConvertToRgb( + ConvertToRgb( configuration, buffer, image, @@ -99,7 +98,7 @@ internal static class HeifPlanarColorConverter where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged - where TLoader : struct, IHeifSampleLoader + where TLoader : struct, IHeifSampleConverter { HeifColorConverterBase colorConverter = HeifColorConverterBase.Create(mode, in parameters, buffer.IsMonochrome); YuvToRgbRowConverter converter = new( @@ -139,7 +138,7 @@ internal static class HeifPlanarColorConverter where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged - where TStorer : struct, IHeifSampleStorer + where TStorer : struct, IHeifSampleConverter { HeifColorConverterBase colorConverter = HeifColorConverterBase.Create(mode, in parameters, buffer.IsMonochrome); RgbToYuvRowConverter converter = new( @@ -209,7 +208,7 @@ internal static class HeifPlanarColorConverter where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged - where TLoader : struct, IHeifSampleLoader + where TLoader : struct, IHeifSampleConverter { /// /// The configuration used for packed-pixel conversion. @@ -364,7 +363,7 @@ internal static class HeifPlanarColorConverter Span blue = scratch.Slice(width * 2, width); int sourceY = y + this.sourceY; ReadOnlySpan luma = this.buffer.GetLumaRowSpan(sourceY).Slice(this.sourceX, width); - ConvertSamplesToFloat(luma, red); + HeifSampleConversion.ConvertSamplesToFloat(luma, red); int packedOffset = width * 3; if (!this.isMonochrome) @@ -384,8 +383,8 @@ internal static class HeifPlanarColorConverter ReadOnlySpan cr1 = this.buffer.GetChromaRedRowSpan(y1); if (this.subsamplingX == 0) { - ConvertSamplesToFloat(cb0.Slice(this.sourceX, width), green); - ConvertSamplesToFloat(cr0.Slice(this.sourceX, width), blue); + HeifSampleConversion.ConvertSamplesToFloat(cb0.Slice(this.sourceX, width), green); + HeifSampleConversion.ConvertSamplesToFloat(cr0.Slice(this.sourceX, width), blue); } else { @@ -397,7 +396,7 @@ internal static class HeifPlanarColorConverter bool isCenteredX = this.chromaPositionX == 1; - ReconstructChromaRow( + HeifSampleConversion.ReconstructChromaRow( cb0, cb1, y1Weight, @@ -413,7 +412,7 @@ internal static class HeifPlanarColorConverter } reconstructed = this.reconstructCompleteRow ? reconstructed : blue; - ReconstructChromaRow( + HeifSampleConversion.ReconstructChromaRow( cr0, cr1, y1Weight, @@ -452,7 +451,7 @@ internal static class HeifPlanarColorConverter } Span packed = MemoryMarshal.Cast(packedStorage)[..width]; - PackRgba64(red, green, blue, packed); + HeifSampleConversion.PackRgba64(red, green, blue, packed); PixelOperations.Instance.FromRgba64(this.configuration, packed, destination); } } @@ -468,7 +467,7 @@ internal static class HeifPlanarColorConverter where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged - where TStorer : struct, IHeifSampleStorer + where TStorer : struct, IHeifSampleConverter { /// /// The configuration used for packed-pixel conversion. @@ -585,7 +584,7 @@ internal static class HeifPlanarColorConverter for (int y = 0; y < this.image.Height; y++) { this.ConvertSourceRow(y, packed, luma0, blue0, red0); - WriteSamples( + HeifSampleConversion.WriteSamples( luma0, this.buffer.GetLumaRowSpan(y), this.colorConverter.LumaScale, @@ -611,7 +610,7 @@ internal static class HeifPlanarColorConverter // unit removes per-row state and lets the selected chroma position choose or average the two rows. int sourceY = destinationY << 1; this.ConvertSourceRow(sourceY, packed, luma0, blue0, red0); - WriteSamples( + HeifSampleConversion.WriteSamples( luma0, this.buffer.GetLumaRowSpan(sourceY), this.colorConverter.LumaScale, @@ -622,7 +621,7 @@ internal static class HeifPlanarColorConverter if (hasSecondRow) { this.ConvertSourceRow(sourceY + 1, packed, luma1, blue1, red1); - WriteSamples( + HeifSampleConversion.WriteSamples( luma1, this.buffer.GetLumaRowSpan(sourceY + 1), this.colorConverter.LumaScale, @@ -671,7 +670,7 @@ internal static class HeifPlanarColorConverter } PixelOperations.Instance.ToRgb48(this.configuration, source, packed); - DeinterleaveRgb48(packed, luma, chromaBlue, chromaRed); + HeifSampleConversion.DeinterleaveRgb48(packed, luma, chromaBlue, chromaRed); this.colorConverter.ConvertFromRgbInPlace(luma, chromaBlue, chromaRed, UShortMaximum); } @@ -696,14 +695,14 @@ internal static class HeifPlanarColorConverter Span redDestination = this.buffer.GetChromaRedRowSpan(destinationY); if (this.subsamplingX == 0) { - WriteSamples( + HeifSampleConversion.WriteSamples( blue0, blueDestination, this.colorConverter.ChromaScale, this.colorConverter.ChromaBias, this.chromaMaximum); - WriteSamples( + HeifSampleConversion.WriteSamples( red0, redDestination, this.colorConverter.ChromaScale, @@ -714,7 +713,7 @@ internal static class HeifPlanarColorConverter } bool isCenteredX = this.chromaPositionX == 1; - WriteSubsampledSamples( + HeifSampleConversion.WriteSubsampledSamples( blue0, blue1, blueDestination, @@ -724,7 +723,7 @@ internal static class HeifPlanarColorConverter this.colorConverter.ChromaBias, this.chromaMaximum); - WriteSubsampledSamples( + HeifSampleConversion.WriteSubsampledSamples( red0, red1, redDestination, diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Samples.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifSampleConversion.cs similarity index 83% rename from src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Samples.cs rename to src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifSampleConversion.cs index dbfdfb2c6..a511604c5 100644 --- a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Samples.cs +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifSampleConversion.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; @@ -9,74 +10,16 @@ using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats.Heif.Components; -/// +/// /// Provides SIMD sample widening, chroma reconstruction, planar storage, and packed output for HEIF color conversion. -/// -internal abstract partial class HeifColorConverterBase +/// +internal static class HeifSampleConversion { /// /// The largest value represented by a 16-bit packed RGB component. /// private const float UShortMaximum = ushort.MaxValue; - /// - /// Defines the SIMD widening operations for one reconstructed HEIF sample type. - /// - /// The reconstructed sample type. - public interface IHeifSampleLoader - where TSample : unmanaged - { - /// - /// Loads and widens four samples to single-precision lanes. - /// - /// The first source sample. - /// The widened samples. - public static abstract Vector128 LoadVector128(ref TSample source); - - /// - /// Loads and widens eight samples to single-precision lanes. - /// - /// The first source sample. - /// The widened samples. - public static abstract Vector256 LoadVector256(ref TSample source); - - /// - /// Loads and widens sixteen samples to single-precision lanes. - /// - /// The first source sample. - /// The widened samples. - public static abstract Vector512 LoadVector512(ref TSample source); - } - - /// - /// Defines the SIMD narrowing and storage operations for one encoded HEIF sample type. - /// - /// The encoded sample type. - public interface IHeifSampleStorer - where TSample : unmanaged - { - /// - /// Narrows and stores four integer samples. - /// - /// The integer samples. - /// The first destination sample. - public static abstract void Store(Vector128 source, ref TSample destination); - - /// - /// Narrows and stores eight integer samples. - /// - /// The integer samples. - /// The first destination sample. - public static abstract void Store(Vector256 source, ref TSample destination); - - /// - /// Narrows and stores sixteen integer samples. - /// - /// The integer samples. - /// The first destination sample. - public static abstract void Store(Vector512 source, ref TSample destination); - } - /// /// Widens reconstructed integer samples into a pooled float component row. /// @@ -86,7 +29,7 @@ internal abstract partial class HeifColorConverterBase /// The destination component row. public static void ConvertSamplesToFloat(ReadOnlySpan source, Span destination) where TSample : unmanaged - where TLoader : struct, IHeifSampleLoader + where TLoader : struct, IHeifSampleConverter { ref TSample sourceBase = ref MemoryMarshal.GetReference(source); ref float destinationBase = ref MemoryMarshal.GetReference(destination); @@ -154,7 +97,7 @@ internal abstract partial class HeifColorConverterBase Span scratch0, Span scratch1) where TSample : unmanaged - where TLoader : struct, IHeifSampleLoader + where TLoader : struct, IHeifSampleConverter { int sourceLength = subX == 0 ? destination.Length : (destination.Length + 1) >> 1; Span top = scratch0[..sourceLength]; @@ -420,8 +363,10 @@ internal abstract partial class HeifColorConverterBase // reinterpreting adjacent 16-bit samples as one unrelated 32-bit integer. Vector128 redVector = Vector128.ConvertToSingle( Vector128.Create((uint)pixel0.R, pixel1.R, pixel2.R, pixel3.R)); + Vector128 greenVector = Vector128.ConvertToSingle( Vector128.Create((uint)pixel0.G, pixel1.G, pixel2.G, pixel3.G)); + Vector128 blueVector = Vector128.ConvertToSingle( Vector128.Create((uint)pixel0.B, pixel1.B, pixel2.B, pixel3.B)); @@ -452,7 +397,7 @@ internal abstract partial class HeifColorConverterBase /// The largest encoded sample value. public static void WriteSamples(ReadOnlySpan source, Span destination, float scale, float bias, float maximum) where TSample : unmanaged - where TStorer : struct, IHeifSampleStorer + where TStorer : struct, IHeifSampleConverter { ref float sourceBase = ref MemoryMarshal.GetReference(source); ref TSample destinationBase = ref MemoryMarshal.GetReference(destination); @@ -518,7 +463,7 @@ internal abstract partial class HeifColorConverterBase float bias, float maximum) where TSample : unmanaged - where TStorer : struct, IHeifSampleStorer + where TStorer : struct, IHeifSampleConverter { ref float row0Base = ref MemoryMarshal.GetReference(row0); ref float row1Base = ref MemoryMarshal.GetReference(row1); @@ -710,6 +655,92 @@ internal abstract partial class HeifColorConverterBase } } + /// + /// Packs normalized monochrome samples into 16-bit luminance pixels. + /// + /// The normalized monochrome samples. + /// The destination luminance pixels. + public static void PackL16(ReadOnlySpan source, Span destination) + { + ref float sourceBase = ref MemoryMarshal.GetReference(source); + ref L16 destinationBase = ref MemoryMarshal.GetReference(destination); + int length = destination.Length; + int i = 0; + + if (Vector512.IsHardwareAccelerated) + { + Vector512 maximum = Vector512.Create(UShortMaximum); + Vector512 redWeight = Vector512.Create(0.2126F); + Vector512 greenWeight = Vector512.Create(0.7152F); + Vector512 blueWeight = Vector512.Create(0.0722F); + Vector512 roundingOffset = Vector512.Create(0.5F); + int oneVectorFromEnd = length - Vector512.Count; + for (; i <= oneVectorFromEnd; i += Vector512.Count) + { + Vector512 value = Vector512.Clamp( + Unsafe.As>(ref Unsafe.Add(ref sourceBase, i)), + Vector512.Zero, + Vector512.One) * maximum; + + // L16 uses its BT.709 luminance expression even when all three source components are equal. Preserve + // that exact arithmetic order so the SIMD path remains byte-identical to L16.FromScaledVector4. + Vector512 luminance = ((value * redWeight) + (value * greenWeight)) + (value * blueWeight); + Vector512 samples = Vector512.ConvertToInt32(luminance + roundingOffset); + Vector256 packed = Vector256.Narrow(samples.GetLower().AsUInt32(), samples.GetUpper().AsUInt32()); + packed.StoreUnsafe(ref Unsafe.As(ref Unsafe.Add(ref destinationBase, i))); + } + } + + if (Vector256.IsHardwareAccelerated) + { + Vector256 maximum = Vector256.Create(UShortMaximum); + Vector256 redWeight = Vector256.Create(0.2126F); + Vector256 greenWeight = Vector256.Create(0.7152F); + Vector256 blueWeight = Vector256.Create(0.0722F); + Vector256 roundingOffset = Vector256.Create(0.5F); + int oneVectorFromEnd = length - Vector256.Count; + for (; i <= oneVectorFromEnd; i += Vector256.Count) + { + Vector256 value = Vector256.Clamp( + Unsafe.As>(ref Unsafe.Add(ref sourceBase, i)), + Vector256.Zero, + Vector256.One) * maximum; + + Vector256 luminance = ((value * redWeight) + (value * greenWeight)) + (value * blueWeight); + Vector256 samples = Vector256.ConvertToInt32(luminance + roundingOffset); + Vector128 packed = Vector128.Narrow(samples.GetLower().AsUInt32(), samples.GetUpper().AsUInt32()); + packed.StoreUnsafe(ref Unsafe.As(ref Unsafe.Add(ref destinationBase, i))); + } + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 maximum = Vector128.Create(UShortMaximum); + Vector128 redWeight = Vector128.Create(0.2126F); + Vector128 greenWeight = Vector128.Create(0.7152F); + Vector128 blueWeight = Vector128.Create(0.0722F); + Vector128 roundingOffset = Vector128.Create(0.5F); + int oneVectorFromEnd = length - Vector128.Count; + for (; i <= oneVectorFromEnd; i += Vector128.Count) + { + Vector128 value = Vector128.Clamp( + Unsafe.As>(ref Unsafe.Add(ref sourceBase, i)), + Vector128.Zero, + Vector128.One) * maximum; + + Vector128 luminance = ((value * redWeight) + (value * greenWeight)) + (value * blueWeight); + Vector128 samples = Vector128.ConvertToInt32(luminance + roundingOffset); + Vector64 packed = Vector128.Narrow(samples.AsUInt32(), Vector128.Zero).GetLower(); + packed.StoreUnsafe(ref Unsafe.As(ref Unsafe.Add(ref destinationBase, i))); + } + } + + for (; i < length; i++) + { + Unsafe.Add(ref destinationBase, i) = L16.FromScaledVector4(new Vector4(Unsafe.Add(ref sourceBase, i))); + } + } + /// /// Reads an eight-bit or 16-bit unsigned sample without an intermediate conversion buffer. /// @@ -854,125 +885,4 @@ internal abstract partial class HeifColorConverterBase Unsafe.As>(ref Unsafe.Add(ref destination, 2)) = upper; } - /// - /// Widens reconstructed eight-bit samples using exact unsigned conversions. - /// - public readonly struct HeifByteSampleLoader : IHeifSampleLoader - { - /// - public static Vector128 LoadVector128(ref byte source) - { - uint packed = Unsafe.ReadUnaligned(ref source); - Vector128 samples16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - return Vector128.ConvertToSingle(Vector128.WidenLower(samples16)); - } - - /// - public static Vector256 LoadVector256(ref byte source) - { - ulong packed = Unsafe.ReadUnaligned(ref source); - Vector128 samples16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); - Vector256 samples32 = Vector256.Create(Vector128.WidenLower(samples16), Vector128.WidenUpper(samples16)); - return Vector256.ConvertToSingle(samples32); - } - - /// - public static Vector512 LoadVector512(ref byte source) - { - Vector128 packed = Unsafe.ReadUnaligned>(ref source); - (Vector128 lower16, Vector128 upper16) = Vector128.Widen(packed); - Vector256 lower32 = Vector256.Create(Vector128.WidenLower(lower16), Vector128.WidenUpper(lower16)); - Vector256 upper32 = Vector256.Create(Vector128.WidenLower(upper16), Vector128.WidenUpper(upper16)); - return Vector512.ConvertToSingle(Vector512.Create(lower32, upper32)); - } - } - - /// - /// Widens reconstructed high-bit-depth samples using exact unsigned conversions. - /// - public readonly struct HeifUShortSampleLoader : IHeifSampleLoader - { - /// - public static Vector128 LoadVector128(ref ushort source) - { - ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); - Vector128 samples16 = Vector128.CreateScalarUnsafe(packed).AsUInt16(); - return Vector128.ConvertToSingle(Vector128.WidenLower(samples16)); - } - - /// - public static Vector256 LoadVector256(ref ushort source) - { - Vector128 samples16 = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source)); - Vector256 samples32 = Vector256.Create(Vector128.WidenLower(samples16), Vector128.WidenUpper(samples16)); - return Vector256.ConvertToSingle(samples32); - } - - /// - public static Vector512 LoadVector512(ref ushort source) - { - Vector256 samples16 = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source)); - (Vector256 lower32, Vector256 upper32) = Vector256.Widen(samples16); - return Vector512.ConvertToSingle(Vector512.Create(lower32, upper32)); - } - } - - /// - /// Narrows encoded integer lanes to eight-bit samples. - /// - public readonly struct HeifByteSampleStorer : IHeifSampleStorer - { - /// - public static void Store(Vector128 source, ref byte destination) - { - Vector128 samples16 = Vector128.Narrow(source.AsUInt32(), Vector128.Zero); - Vector128 samples8 = Vector128.Narrow(samples16, Vector128.Zero); - - // The lower four bytes contain the four source lanes after the two narrowing stages. - Unsafe.WriteUnaligned(ref destination, samples8.AsUInt32().ToScalar()); - } - - /// - public static void Store(Vector256 source, ref byte destination) - { - Store(source.GetLower(), ref destination); - Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector128.Count)); - } - - /// - public static void Store(Vector512 source, ref byte destination) - { - Store(source.GetLower(), ref destination); - Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector256.Count)); - } - } - - /// - /// Narrows encoded integer lanes to unsigned 16-bit samples. - /// - public readonly struct HeifUShortSampleStorer : IHeifSampleStorer - { - /// - public static void Store(Vector128 source, ref ushort destination) - { - Vector128 samples = Vector128.Narrow(source.AsUInt32(), Vector128.Zero); - - // The lower four UInt16 values are contiguous and can be committed with one unaligned store. - Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), samples.AsUInt64().ToScalar()); - } - - /// - public static void Store(Vector256 source, ref ushort destination) - { - Store(source.GetLower(), ref destination); - Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector128.Count)); - } - - /// - public static void Store(Vector512 source, ref ushort destination) - { - Store(source.GetLower(), ref destination); - Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector256.Count)); - } - } } diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Simd.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Simd.cs index e65edef02..ed3285923 100644 --- a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Simd.cs +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Simd.cs @@ -5,7 +5,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Common.Helpers; -using static SixLabors.ImageSharp.Formats.Heif.Components.HeifColorConverterBase; namespace SixLabors.ImageSharp.Formats.Heif.Components; @@ -131,9 +130,9 @@ internal static partial class HeifYuv420ToRgb8Converter Vector512 cr = LoadRepeatedVector512(ref Unsafe.Add(ref chromaRedBase, x >> 1)); TOperator.Convert(y, cb, cr, in parameters.SixteenLane, out Vector512 r, out Vector512 g, out Vector512 b); - HeifByteSampleStorer.Store(r, ref Unsafe.Add(ref redBase, x)); - HeifByteSampleStorer.Store(g, ref Unsafe.Add(ref greenBase, x)); - HeifByteSampleStorer.Store(b, ref Unsafe.Add(ref blueBase, x)); + HeifByteSampleConverter.Store(r, ref Unsafe.Add(ref redBase, x)); + HeifByteSampleConverter.Store(g, ref Unsafe.Add(ref greenBase, x)); + HeifByteSampleConverter.Store(b, ref Unsafe.Add(ref blueBase, x)); } } @@ -148,9 +147,9 @@ internal static partial class HeifYuv420ToRgb8Converter Vector256 cr = LoadRepeatedVector256(ref Unsafe.Add(ref chromaRedBase, x >> 1)); TOperator.Convert(y, cb, cr, in parameters.EightLane, out Vector256 r, out Vector256 g, out Vector256 b); - HeifByteSampleStorer.Store(r, ref Unsafe.Add(ref redBase, x)); - HeifByteSampleStorer.Store(g, ref Unsafe.Add(ref greenBase, x)); - HeifByteSampleStorer.Store(b, ref Unsafe.Add(ref blueBase, x)); + HeifByteSampleConverter.Store(r, ref Unsafe.Add(ref redBase, x)); + HeifByteSampleConverter.Store(g, ref Unsafe.Add(ref greenBase, x)); + HeifByteSampleConverter.Store(b, ref Unsafe.Add(ref blueBase, x)); } } @@ -165,9 +164,9 @@ internal static partial class HeifYuv420ToRgb8Converter Vector128 cr = LoadRepeatedVector128(ref Unsafe.Add(ref chromaRedBase, x >> 1)); TOperator.Convert(y, cb, cr, in parameters.FourLane, out Vector128 r, out Vector128 g, out Vector128 b); - HeifByteSampleStorer.Store(r, ref Unsafe.Add(ref redBase, x)); - HeifByteSampleStorer.Store(g, ref Unsafe.Add(ref greenBase, x)); - HeifByteSampleStorer.Store(b, ref Unsafe.Add(ref blueBase, x)); + HeifByteSampleConverter.Store(r, ref Unsafe.Add(ref redBase, x)); + HeifByteSampleConverter.Store(g, ref Unsafe.Add(ref greenBase, x)); + HeifByteSampleConverter.Store(b, ref Unsafe.Add(ref blueBase, x)); } } diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs new file mode 100644 index 000000000..686da1dd0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs @@ -0,0 +1,167 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Components; + +/// +/// Defines SIMD widening and narrowing operations for one native HEIF sample representation. +/// +/// The native sample type. +internal interface IHeifSampleConverter + where TSample : unmanaged +{ + /// + /// Loads and widens four samples to single-precision lanes. + /// + /// The first source sample. + /// The widened samples. + public static abstract Vector128 LoadVector128(ref TSample source); + + /// + /// Loads and widens eight samples to single-precision lanes. + /// + /// The first source sample. + /// The widened samples. + public static abstract Vector256 LoadVector256(ref TSample source); + + /// + /// Loads and widens sixteen samples to single-precision lanes. + /// + /// The first source sample. + /// The widened samples. + public static abstract Vector512 LoadVector512(ref TSample source); + + /// + /// Narrows and stores four integer samples. + /// + /// The integer samples. + /// The first destination sample. + public static abstract void Store(Vector128 source, ref TSample destination); + + /// + /// Narrows and stores eight integer samples. + /// + /// The integer samples. + /// The first destination sample. + public static abstract void Store(Vector256 source, ref TSample destination); + + /// + /// Narrows and stores sixteen integer samples. + /// + /// The integer samples. + /// The first destination sample. + public static abstract void Store(Vector512 source, ref TSample destination); +} + +/// +/// Converts between eight-bit native samples and the planar conversion pipeline. +/// +internal readonly struct HeifByteSampleConverter : IHeifSampleConverter +{ + /// + public static Vector128 LoadVector128(ref byte source) + { + uint packed = Unsafe.ReadUnaligned(ref source); + Vector128 samples16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + return Vector128.ConvertToSingle(Vector128.WidenLower(samples16)); + } + + /// + public static Vector256 LoadVector256(ref byte source) + { + ulong packed = Unsafe.ReadUnaligned(ref source); + Vector128 samples16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()); + Vector256 samples32 = Vector256.Create(Vector128.WidenLower(samples16), Vector128.WidenUpper(samples16)); + return Vector256.ConvertToSingle(samples32); + } + + /// + public static Vector512 LoadVector512(ref byte source) + { + Vector128 packed = Unsafe.ReadUnaligned>(ref source); + (Vector128 lower16, Vector128 upper16) = Vector128.Widen(packed); + Vector256 lower32 = Vector256.Create(Vector128.WidenLower(lower16), Vector128.WidenUpper(lower16)); + Vector256 upper32 = Vector256.Create(Vector128.WidenLower(upper16), Vector128.WidenUpper(upper16)); + return Vector512.ConvertToSingle(Vector512.Create(lower32, upper32)); + } + + /// + public static void Store(Vector128 source, ref byte destination) + { + Vector128 samples16 = Vector128.Narrow(source.AsUInt32(), Vector128.Zero); + Vector128 samples8 = Vector128.Narrow(samples16, Vector128.Zero); + + // The lower four bytes contain the four source lanes after the two narrowing stages. + Unsafe.WriteUnaligned(ref destination, samples8.AsUInt32().ToScalar()); + } + + /// + public static void Store(Vector256 source, ref byte destination) + { + Store(source.GetLower(), ref destination); + Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector128.Count)); + } + + /// + public static void Store(Vector512 source, ref byte destination) + { + Store(source.GetLower(), ref destination); + Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector256.Count)); + } +} + +/// +/// Converts between unsigned 16-bit native samples and the planar conversion pipeline. +/// +internal readonly struct HeifUShortSampleConverter : IHeifSampleConverter +{ + /// + public static Vector128 LoadVector128(ref ushort source) + { + ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref source)); + Vector128 samples16 = Vector128.CreateScalarUnsafe(packed).AsUInt16(); + return Vector128.ConvertToSingle(Vector128.WidenLower(samples16)); + } + + /// + public static Vector256 LoadVector256(ref ushort source) + { + Vector128 samples16 = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source)); + Vector256 samples32 = Vector256.Create(Vector128.WidenLower(samples16), Vector128.WidenUpper(samples16)); + return Vector256.ConvertToSingle(samples32); + } + + /// + public static Vector512 LoadVector512(ref ushort source) + { + Vector256 samples16 = Unsafe.ReadUnaligned>(ref Unsafe.As(ref source)); + (Vector256 lower32, Vector256 upper32) = Vector256.Widen(samples16); + return Vector512.ConvertToSingle(Vector512.Create(lower32, upper32)); + } + + /// + public static void Store(Vector128 source, ref ushort destination) + { + Vector128 samples = Vector128.Narrow(source.AsUInt32(), Vector128.Zero); + + // The lower four UInt16 values are contiguous and can be committed with one unaligned store. + Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), samples.AsUInt64().ToScalar()); + } + + /// + public static void Store(Vector256 source, ref ushort destination) + { + Store(source.GetLower(), ref destination); + Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector128.Count)); + } + + /// + public static void Store(Vector512 source, ref ushort destination) + { + Store(source.GetLower(), ref destination); + Store(source.GetUpper(), ref Unsafe.Add(ref destination, Vector256.Count)); + } +} diff --git a/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs index 93b66f5b1..2aa83e144 100644 --- a/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs @@ -5,6 +5,7 @@ using System.Buffers; using System.Buffers.Binary; using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; @@ -15,7 +16,7 @@ namespace SixLabors.ImageSharp.Formats.Heif; /// Decodes the image items referenced by a HEIF grid derived-image item. /// /// The destination pixel type. -internal class GridHeifItemDecoder : IHeifItemDecoder +internal class GridHeifItemDecoder : IHeifItemDecoder, IHeifAlphaItemDecoder where TPixel : unmanaged, IPixel { /// @@ -85,64 +86,12 @@ internal class GridHeifItemDecoder : IHeifItemDecoder CicpProfile? colorProfile, CancellationToken cancellationToken) { - if (data.Length < 8) - { - throw new InvalidImageContentException("The HEIF image grid descriptor is truncated."); - } - - byte version = data[0]; - if (version != 0) - { - throw new InvalidImageContentException($"The HEIF image grid descriptor has unsupported version {version}."); - } - - byte flags = data[1]; - int rows = data[2] + 1; - int columns = data[3] + 1; - bool usesLargeDimensions = (flags & 1) != 0; - int descriptorLength = usesLargeDimensions ? 12 : 8; - if (data.Length != descriptorLength) - { - throw new InvalidImageContentException("The HEIF image grid descriptor has an invalid length."); - } - - uint outputWidth; - uint outputHeight; - if (usesLargeDimensions) - { - outputWidth = BinaryPrimitives.ReadUInt32BigEndian(data[4..]); - outputHeight = BinaryPrimitives.ReadUInt32BigEndian(data[8..]); - } - else - { - outputWidth = BinaryPrimitives.ReadUInt16BigEndian(data[4..]); - outputHeight = BinaryPrimitives.ReadUInt16BigEndian(data[6..]); - } - - if (outputWidth is 0 or > int.MaxValue || outputHeight is 0 or > int.MaxValue) - { - throw new InvalidImageContentException("The HEIF image grid descriptor has invalid output dimensions."); - } - - List linked = this.tileItemIds is null ? [] : new(this.tileItemIds); - if (this.tileItemIds is null) - { - foreach (HeifItemLink link in this.itemLinks) - { - if (link.Type == Heif4CharCode.Dimg && link.SourceId == gridItem.Id) - { - // The order of dimg destinations is the normative row-major order of the grid cells. - linked.AddRange(link.DestinationIds); - } - } - } - - int tileCount = rows * columns; - if (linked.Count != tileCount) - { - string message = $"The HEIF image grid requires {tileCount} tiles, but its derived-image references contain {linked.Count}."; - throw new InvalidImageContentException(message); - } + GridDescriptor descriptor = ParseGridDescriptor(data); + int rows = descriptor.Rows; + int columns = descriptor.Columns; + int outputWidth = descriptor.OutputSize.Width; + int outputHeight = descriptor.OutputSize.Height; + List linked = this.GetLinkedTileIds(gridItem, descriptor); // Each compressed tile decoder returns an owned Image. Keep every tile alive until // the final grid has copied its pixels, then dispose all intermediates together. @@ -153,37 +102,10 @@ internal class GridHeifItemDecoder : IHeifItemDecoder { cancellationToken.ThrowIfCancellationRequested(); HeifItem item = this.items.First(item => item.Id == id); - if (tileType == default) - { - tileType = item.Type; - } - else if (item.Type != tileType) - { - throw new InvalidImageContentException("All HEIF image grid tiles must use the same coding format."); - } - - if (item.Type == Heif4CharCode.Av01) - { - Av1CodecConfiguration itemConfiguration = item.Av1CodecConfiguration - ?? throw new InvalidImageContentException($"AV1 image grid tile {item.Id} has no codec configuration property."); - - if (av1GridConfiguration is null) - { - av1GridConfiguration = itemConfiguration; - } - else if (!av1GridConfiguration.HasMatchingImageConfiguration(itemConfiguration)) - { - // All grid cells share one output sample layout. Reject differing AV1 descriptions before - // allocating and copying tiles so channel precision or chroma geometry cannot change by cell. - throw new InvalidImageContentException("All AV1 image grid tiles must use matching codec configurations."); - } - } + ValidateTileConfiguration(item, ref tileType, ref av1GridConfiguration); - IHeifItemDecoder? decoder = HeifCompressionFactory.GetDecoder(item.Type); - if (decoder is null) - { - throw new ImageFormatException($"The HEIF image grid uses unsupported tile type '{item.Type}'."); - } + IHeifItemDecoder? decoder = HeifCompressionFactory.GetDecoder(item.Type) + ?? throw new ImageFormatException($"The HEIF image grid uses unsupported tile type '{item.Type}'."); if (!this.buffers.TryGetValue(item.Id, out IMemoryOwner? itemMemory)) { @@ -223,7 +145,7 @@ internal class GridHeifItemDecoder : IHeifItemDecoder throw new InvalidImageContentException("The HEIF image grid edge tiles do not overlap the output canvas."); } - Image result = new(options.Configuration, (int)outputWidth, (int)outputHeight, firstTile.Metadata.DeepClone()); + Image result = new(options.Configuration, outputWidth, outputHeight, firstTile.Metadata.DeepClone()); ImageFrame destination = result.Frames.RootFrame; for (int tileIndex = 0; tileIndex < gridTiles.Count; tileIndex++) { @@ -238,8 +160,8 @@ internal class GridHeifItemDecoder : IHeifItemDecoder int row = tileIndex / columns; int destinationX = column * tileWidth; int destinationY = row * tileHeight; - int copyWidth = Math.Min(tileWidth, (int)outputWidth - destinationX); - int copyHeight = Math.Min(tileHeight, (int)outputHeight - destinationY); + int copyWidth = Math.Min(tileWidth, outputWidth - destinationX); + int copyHeight = Math.Min(tileHeight, outputHeight - destinationY); ImageFrame source = tile.Frames.RootFrame; // The descriptor may crop only the rightmost column and bottom row. Copying bounded row spans @@ -253,4 +175,242 @@ internal class GridHeifItemDecoder : IHeifItemDecoder return result; } + + /// + public void DecodeAlphaItemData( + DecoderOptions options, + HeifItem gridItem, + Span data, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied, + CancellationToken cancellationToken) + { + GridDescriptor descriptor = ParseGridDescriptor(data); + List linked = this.GetLinkedTileIds(gridItem, descriptor); + Heif4CharCode tileType = default; + Av1CodecConfiguration? av1GridConfiguration = null; + Size tileSize = default; + + // Validate the complete grid before mutating the color frame. IgnoreImageData can then omit a failed alpha + // grid without leaving a partially composed prefix in the returned image. + foreach (uint id in linked) + { + HeifItem item = this.items.First(item => item.Id == id); + ValidateTileConfiguration(item, ref tileType, ref av1GridConfiguration); + if (HeifCompressionFactory.GetDecoder(item.Type) is not IHeifAlphaItemDecoder) + { + throw new ImageFormatException($"The HEIF alpha grid uses unsupported tile type '{item.Type}'."); + } + + if (!this.buffers.ContainsKey(item.Id)) + { + throw new InvalidImageContentException($"HEIF alpha grid tile {item.Id} has no data extents."); + } + + if (item.Extent == default) + { + throw new InvalidImageContentException($"HEIF alpha grid tile {item.Id} has no spatial extent."); + } + + if (tileSize == default) + { + tileSize = item.Extent; + } + else if (item.Extent != tileSize) + { + throw new InvalidImageContentException("The HEIF alpha grid contains tiles with mismatched dimensions."); + } + } + + int gridWidth = descriptor.OutputSize.Width; + int gridHeight = descriptor.OutputSize.Height; + if (((long)tileSize.Width * descriptor.Columns) < gridWidth || ((long)tileSize.Height * descriptor.Rows) < gridHeight) + { + throw new InvalidImageContentException("The HEIF alpha grid tiles do not cover the output canvas."); + } + + if (((long)tileSize.Width * (descriptor.Columns - 1)) >= gridWidth || + ((long)tileSize.Height * (descriptor.Rows - 1)) >= gridHeight) + { + throw new InvalidImageContentException("The HEIF alpha grid edge tiles do not overlap the output canvas."); + } + + if (descriptor.OutputSize != outputSize || destinationRectangle.Size != outputSize) + { + throw new InvalidImageContentException("The HEIF alpha grid dimensions do not match the color grid dimensions."); + } + + for (int tileIndex = 0; tileIndex < linked.Count; tileIndex++) + { + cancellationToken.ThrowIfCancellationRequested(); + HeifItem item = this.items.First(item => item.Id == linked[tileIndex]); + IHeifAlphaItemDecoder decoder = (IHeifAlphaItemDecoder)HeifCompressionFactory.GetDecoder(item.Type)!; + IMemoryOwner itemMemory = this.buffers[item.Id]; + int column = tileIndex % descriptor.Columns; + int row = tileIndex / descriptor.Columns; + int destinationX = destinationRectangle.X + (column * tileSize.Width); + int destinationY = destinationRectangle.Y + (row * tileSize.Height); + int copyWidth = Math.Min(tileSize.Width, destinationRectangle.Right - destinationX); + int copyHeight = Math.Min(tileSize.Height, destinationRectangle.Bottom - destinationY); + Rectangle tileDestination = new(destinationX, destinationY, copyWidth, copyHeight); + + decoder.DecodeAlphaItemData( + options, + item, + itemMemory.GetSpan(), + destination, + tileSize, + tileDestination, + premultiplied, + cancellationToken); + } + } + + /// + /// Parses and validates the fixed HEIF image-grid descriptor fields used by both color and alpha composition. + /// + /// The complete image-grid descriptor payload. + /// The validated row, column, and output dimensions. + private static GridDescriptor ParseGridDescriptor(ReadOnlySpan data) + { + if (data.Length < 8) + { + throw new InvalidImageContentException("The HEIF image grid descriptor is truncated."); + } + + byte version = data[0]; + if (version != 0) + { + throw new InvalidImageContentException($"The HEIF image grid descriptor has unsupported version {version}."); + } + + bool usesLargeDimensions = (data[1] & 1) != 0; + int descriptorLength = usesLargeDimensions ? 12 : 8; + if (data.Length != descriptorLength) + { + throw new InvalidImageContentException("The HEIF image grid descriptor has an invalid length."); + } + + uint outputWidth = usesLargeDimensions + ? BinaryPrimitives.ReadUInt32BigEndian(data[4..]) + : BinaryPrimitives.ReadUInt16BigEndian(data[4..]); + + uint outputHeight = usesLargeDimensions + ? BinaryPrimitives.ReadUInt32BigEndian(data[8..]) + : BinaryPrimitives.ReadUInt16BigEndian(data[6..]); + + if (outputWidth is 0 or > int.MaxValue || outputHeight is 0 or > int.MaxValue) + { + throw new InvalidImageContentException("The HEIF image grid descriptor has invalid output dimensions."); + } + + return new GridDescriptor(data[2] + 1, data[3] + 1, new Size((int)outputWidth, (int)outputHeight)); + } + + /// + /// Resolves and validates the row-major tile identifiers for a grid descriptor. + /// + /// The grid item whose derived-image references are being resolved. + /// The validated grid dimensions. + /// The exact row-major tile identifiers required by the descriptor. + private List GetLinkedTileIds(HeifItem gridItem, in GridDescriptor descriptor) + { + List linked = this.tileItemIds is null ? [] : new(this.tileItemIds); + if (this.tileItemIds is null) + { + foreach (HeifItemLink link in this.itemLinks) + { + if (link.Type == Heif4CharCode.Dimg && link.SourceId == gridItem.Id) + { + // The order of dimg destinations is the normative row-major order of the grid cells. + linked.AddRange(link.DestinationIds); + } + } + } + + int tileCount = descriptor.Rows * descriptor.Columns; + if (linked.Count != tileCount) + { + string message = $"The HEIF image grid requires {tileCount} tiles, but its derived-image references contain {linked.Count}."; + throw new InvalidImageContentException(message); + } + + return linked; + } + + /// + /// Validates the coding format and common AV1 sample layout of one grid tile. + /// + /// The coded grid tile being validated. + /// The common coding type established by the first grid tile. + /// The common AV1 sample layout established by the first AV1 grid tile. + private static void ValidateTileConfiguration( + HeifItem item, + ref Heif4CharCode tileType, + ref Av1CodecConfiguration? av1GridConfiguration) + { + if (tileType == default) + { + tileType = item.Type; + } + else if (item.Type != tileType) + { + throw new InvalidImageContentException("All HEIF image grid tiles must use the same coding format."); + } + + if (item.Type != Heif4CharCode.Av01) + { + return; + } + + Av1CodecConfiguration itemConfiguration = item.Av1CodecConfiguration + ?? throw new InvalidImageContentException($"AV1 image grid tile {item.Id} has no codec configuration property."); + + if (av1GridConfiguration is null) + { + av1GridConfiguration = itemConfiguration; + } + else if (!av1GridConfiguration.HasMatchingImageConfiguration(itemConfiguration)) + { + // All grid cells share one output sample layout. Reject differing AV1 descriptions before allocating + // or composing tiles so channel precision and chroma geometry cannot change between cells. + throw new InvalidImageContentException("All AV1 image grid tiles must use matching codec configurations."); + } + } + + /// + /// Contains the bounded row, column, and output dimensions from one image-grid descriptor. + /// + private readonly struct GridDescriptor + { + /// + /// Initializes a new instance of the struct. + /// + /// The number of grid rows. + /// The number of grid columns. + /// The output canvas dimensions. + public GridDescriptor(int rows, int columns, Size outputSize) + { + this.Rows = rows; + this.Columns = columns; + this.OutputSize = outputSize; + } + + /// + /// Gets the number of grid rows. + /// + public int Rows { get; } + + /// + /// Gets the number of grid columns. + /// + public int Columns { get; } + + /// + /// Gets the output canvas dimensions. + /// + public Size OutputSize { get; } + } } diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 3ba01425f..fdb36ea0f 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -6,6 +6,7 @@ using System.Buffers.Binary; using System.Text; using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Formats.Heif.Hevc; using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.Memory; @@ -330,33 +331,14 @@ internal sealed class HeifDecoderCore : ImageDecoderCore for (int frameIndex = 0; frameIndex < colorFrames.Length; frameIndex++) { cancellationToken.ThrowIfCancellationRequested(); - ImageFrame? alphaFrame = null; HeifSequenceSample alphaSample = alphaTrack.Samples[sampleIndices[frameIndex]]; this.ExecuteImageDataSegmentAction( - () => alphaFrame = this.DecodeSequenceFrame(stream, alphaTrack, alphaSample)); - - if (alphaFrame is null) - { - continue; - } - - if (alphaFrame.Size == colorFrames[frameIndex].Size) - { - using (alphaFrame) - { - this.ApplyAlpha(colorFrames[frameIndex], alphaFrame, colorTrack.IsPremultiplied); - } - } - else - { - // Auxiliary planes may use a lower resolution. Adopt the decoded frame into a temporary - // image so the established box resampler can resize it without another source-frame clone. - using Image alphaImage = new(this.configuration, new ImageMetadata(), [alphaFrame]); - alphaImage.Mutate( - context => context.Resize(colorFrames[frameIndex].Width, colorFrames[frameIndex].Height, KnownResamplers.Box)); - - this.ApplyAlpha(colorFrames[frameIndex], alphaImage.Frames.RootFrame, colorTrack.IsPremultiplied); - } + () => this.DecodeSequenceAlphaFrame( + stream, + alphaTrack, + alphaSample, + colorFrames[frameIndex], + colorTrack.IsPremultiplied)); } } @@ -499,19 +481,8 @@ internal sealed class HeifDecoderCore : ImageDecoderCore Av1CodecConfiguration codecConfiguration = track.Av1CodecConfiguration ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); - using IMemoryOwner sampleOwner = this.configuration.MemoryAllocator.Allocate(sample.Length); + using IMemoryOwner sampleOwner = this.ReadSequenceSample(stream, track, sample); Span sampleData = sampleOwner.GetSpan()[..sample.Length]; - stream.Position = sample.Offset; - HeifBoxReader.ReadExactly(stream, sampleData, "The HEIF image-sequence sample is truncated."); - - codecConfiguration.ValidateSampleData( - sampleData, - sample.IsSync, - track.ContentLightLevel, - track.MasteringDisplayColorVolume, - this.Options, - out _, - out _); using Av1Decoder decoder = new(this.configuration); ImageFrame frame = decoder.DecodeFrame( @@ -529,6 +500,89 @@ internal sealed class HeifDecoderCore : ImageDecoderCore return frame; } + /// + /// Decodes one AV1 auxiliary sample and composes its native luma plane directly into a color frame. + /// + /// The destination color pixel type. + /// The complete seekable HEIF stream. + /// The alpha track supplying the codec configuration and color description. + /// The validated alpha sample range. + /// The decoded color frame receiving alpha values. + /// Whether stored color samples must be converted to unassociated alpha. + private void DecodeSequenceAlphaFrame( + BufferedReadStream stream, + HeifSequenceTrack track, + HeifSequenceSample sample, + ImageFrame destination, + bool premultiplied) + where TPixel : unmanaged, IPixel + { + if (track.CodecType != Heif4CharCode.Av01) + { + throw new ImageFormatException($"No decoder is available for image-sequence alpha sample type '{track.CodecType}'."); + } + + Av1CodecConfiguration codecConfiguration = track.Av1CodecConfiguration + ?? throw new InvalidImageContentException("The AV1 alpha image-sequence track has no codec configuration."); + + if (!codecConfiguration.IsMonochrome) + { + throw new InvalidImageContentException("An AV1 alpha image-sequence track must be encoded as monochrome."); + } + + using IMemoryOwner sampleOwner = this.ReadSequenceSample(stream, track, sample); + Span sampleData = sampleOwner.GetSpan()[..sample.Length]; + using Av1Decoder decoder = new(this.configuration); + decoder.DecodeAlpha( + sampleData, + track.CicpProfile, + codecConfiguration, + new Size(track.CodedWidth, track.CodedHeight), + destination, + destination.Size, + destination.Bounds, + premultiplied); + } + + /// + /// Reads and validates one bounded AV1 sequence sample into allocator-owned codec input storage. + /// + /// The complete seekable HEIF stream. + /// The track supplying the codec configuration and color description. + /// The validated sample range. + /// The allocator-owned buffer containing the validated coded sample. + private IMemoryOwner ReadSequenceSample( + BufferedReadStream stream, + HeifSequenceTrack track, + HeifSequenceSample sample) + { + Av1CodecConfiguration codecConfiguration = track.Av1CodecConfiguration + ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); + + IMemoryOwner sampleOwner = this.configuration.MemoryAllocator.Allocate(sample.Length); + try + { + Span sampleData = sampleOwner.GetSpan()[..sample.Length]; + stream.Position = sample.Offset; + HeifBoxReader.ReadExactly(stream, sampleData, "The HEIF image-sequence sample is truncated."); + codecConfiguration.ValidateSampleData( + sampleData, + sample.IsSync, + track.ContentLightLevel, + track.MasteringDisplayColorVolume, + this.Options, + out _, + out _); + + return sampleOwner; + } + catch + { + sampleOwner.Dispose(); + throw; + } + } + /// /// Updates image-level metadata from the selected color and optional alpha sequence tracks. /// @@ -2129,18 +2183,9 @@ internal sealed class HeifDecoderCore : ImageDecoderCore try { - Image? alphaImage = null; - bool alphaPremultiplied = false; + bool hasAlpha = false; this.ExecuteImageDataSegmentAction( - () => alphaImage = this.DecodeAlphaPlane(itemToDecode, buffers, cancellationToken, out alphaPremultiplied)); - - using (alphaImage) - { - if (alphaImage is not null) - { - this.ApplyAlpha(image, alphaImage, alphaPremultiplied); - } - } + () => hasAlpha = this.DecodeAlphaPlane(itemToDecode, buffers, image.Frames.RootFrame, cancellationToken)); if (!this.Options.SkipMetadata) { @@ -2167,7 +2212,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore // and a thumbnail fallback when the primary image compression is not available. HeifMetadata meta = image.Metadata.GetHeifMetadata(); meta.CompressionMethod = itemDecoder.CompressionMethod; - meta.HasAlpha = alphaImage is not null; + meta.HasAlpha = hasAlpha; if (this.Options.SkipMetadata) { // AV1 item decoders still parse encoded metadata to enforce codec/container equivalence and select @@ -2557,20 +2602,21 @@ internal sealed class HeifDecoderCore : ImageDecoderCore } /// - /// Decodes the direct or per-grid-tile alpha auxiliary plane associated with a color image item. + /// Decodes and composes the direct or per-grid-tile alpha auxiliary associated with a color image item. /// + /// The destination color pixel type. /// The color image item whose alpha plane is requested. /// The assembled item payloads. + /// The decoded color frame receiving alpha values. /// The token used to cancel the auxiliary payload decode. - /// Indicates whether the color samples are premultiplied by the decoded alpha. - /// The normalized 16-bit alpha plane, or when the item has no alpha auxiliary. - private Image? DecodeAlphaPlane( + /// when an auxiliary alpha plane was decoded and composed. + private bool DecodeAlphaPlane( HeifItem colorItem, DisposableDictionary> buffers, - CancellationToken cancellationToken, - out bool premultiplied) + ImageFrame destination, + CancellationToken cancellationToken) + where TPixel : unmanaged, IPixel { - premultiplied = false; HeifItem? alphaItem = this.FindAlphaItem(colorItem); if (alphaItem is not null) { @@ -2590,29 +2636,44 @@ internal sealed class HeifDecoderCore : ImageDecoderCore throw new ImageFormatException("The alpha auxiliary image and color image use different presentation transforms."); } - IHeifItemDecoder? decoder = this.GetItemDecoder(alphaItem, buffers); - if (decoder is null) + IHeifItemDecoder? itemDecoder = this.GetItemDecoder(alphaItem, buffers); + if (itemDecoder is not IHeifAlphaItemDecoder decoder) { throw new ImageFormatException($"The alpha auxiliary item uses unsupported item type '{alphaItem.Type}'."); } - premultiplied = this.itemLinks.Any( + bool premultiplied = this.itemLinks.Any( link => link.Type == Heif4CharCode.Prem && link.SourceId == colorItem.Id && link.DestinationIds.Contains(alphaItem.Id)); - return this.DecodeImageItem(alphaItem, decoder, buffers, cancellationToken); + if (!buffers.TryGetValue(alphaItem.Id, out IMemoryOwner? itemMemory)) + { + throw new InvalidImageContentException($"Item {alphaItem.Id} has no data extents."); + } + + decoder.DecodeAlphaItemData( + this.payloadOptions, + alphaItem, + itemMemory.GetSpan(), + destination, + destination.Size, + destination.Bounds, + premultiplied, + cancellationToken); + + return true; } if (colorItem.Type != Heif4CharCode.Grid) { - return null; + return false; } List? alphaTileIds = this.FindGridAlphaTiles(colorItem); if (alphaTileIds is null) { - return null; + return false; } if (!buffers.TryGetValue(colorItem.Id, out IMemoryOwner? gridMemory)) @@ -2622,71 +2683,23 @@ internal sealed class HeifDecoderCore : ImageDecoderCore // The color grid descriptor defines the same row/column layout and output canvas for per-tile alpha // auxiliaries. Supplying their IDs lets the existing grid compositor preserve that normative ordering. - GridHeifItemDecoder gridDecoder = new( + GridHeifItemDecoder gridDecoder = new( this.items, this.itemLinks, buffers, alphaTileIds); - return gridDecoder.DecodeItemData(this.payloadOptions, colorItem, gridMemory.GetSpan(), null, cancellationToken); - } - - /// - /// Composes a normalized alpha plane into a decoded color image. - /// - /// The decoded color pixel format. - /// The decoded color image. - /// The normalized 16-bit alpha plane. - /// Whether the stored color values must be converted to unassociated alpha. - private void ApplyAlpha(Image image, Image alphaImage, bool premultiplied) - where TPixel : unmanaged, IPixel - { - if (alphaImage.Width != image.Width || alphaImage.Height != image.Height) - { - // HEIF permits auxiliary alpha dimensions to differ from the master image. libavif uses a box filter - // for this plane scaling, which maps directly to ImageSharp's existing resampler. - alphaImage.Mutate(context => context.Resize(image.Width, image.Height, KnownResamplers.Box)); - } - - this.ApplyAlpha(image.Frames.RootFrame, alphaImage.Frames.RootFrame, premultiplied); - } - - /// - /// Composes one same-sized auxiliary alpha frame into a decoded color frame. - /// - /// The destination color pixel format. - /// The decoded color frame receiving alpha values. - /// The decoded same-sized 16-bit alpha frame. - /// Whether the encoded color samples are premultiplied by alpha. - private void ApplyAlpha(ImageFrame colorFrame, ImageFrame alphaFrame, bool premultiplied) - where TPixel : unmanaged, IPixel - { - using IMemoryOwner rowOwner = this.configuration.MemoryAllocator.Allocate(colorFrame.Width); - Span rgbaRow = rowOwner.GetSpan()[..colorFrame.Width]; - PixelOperations pixelOperations = PixelOperations.Instance; - for (int y = 0; y < colorFrame.Height; y++) - { - Span colorRow = colorFrame.PixelBuffer.DangerousGetRowSpan(y); - Span alphaRow = alphaFrame.PixelBuffer.DangerousGetRowSpan(y); - pixelOperations.ToRgba64(this.configuration, colorRow, rgbaRow); - for (int x = 0; x < colorFrame.Width; x++) - { - Rgba64 pixel = rgbaRow[x]; - pixel.A = alphaRow[x].PackedValue; - if (premultiplied) - { - // libavif defines transparent premultiplied samples as transparent black. For nonzero alpha, - // reuse the packed pixel's associated-input conversion so clamping and rounding follow ImageSharp. - pixel = pixel.A == 0 - ? new Rgba64(0, 0, 0, 0) - : Rgba64.FromAssociatedScaledVector4(pixel.ToScaledVector4()); - } - - rgbaRow[x] = pixel; - } + gridDecoder.DecodeAlphaItemData( + this.payloadOptions, + colorItem, + gridMemory.GetSpan(), + destination, + destination.Size, + destination.Bounds, + false, + cancellationToken); - pixelOperations.FromRgba64(this.configuration, rgbaRow, colorRow); - } + return true; } /// diff --git a/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs b/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs index 79ace6407..299b1fff2 100644 --- a/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs @@ -448,7 +448,7 @@ internal sealed class HeifEncoderCore throw new NotSupportedException("Legacy JPEG image items do not support lossless encoding."); } - if (this.encoder.BitDepth is not null && this.encoder.BitDepth != HeifBitDepth.Bit8) + if (this.encoder.BitDepth is not null and not HeifBitDepth.Bit8) { throw new NotSupportedException("Legacy JPEG image items support only 8-bit component encoding."); } diff --git a/src/ImageSharp/Formats/Heif/Hevc/Color/HevcYuvConverter.cs b/src/ImageSharp/Formats/Heif/Hevc/Color/HevcYuvConverter.cs index fa43a24bf..92a8af514 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/Color/HevcYuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/Color/HevcYuvConverter.cs @@ -2,9 +2,9 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Components; +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; -using static SixLabors.ImageSharp.Formats.Heif.Components.HeifColorConverterBase; namespace SixLabors.ImageSharp.Formats.Heif.Hevc.Color; @@ -46,6 +46,44 @@ internal static class HevcYuvConverter sourceY); } + /// + /// Composes a visible HEVC luma rectangle into a packed color frame as auxiliary alpha. + /// + /// The destination color pixel type. + /// The configuration used for allocation and pixel conversion. + /// The reconstructed HEVC picture containing the alpha luma plane. + /// The packed color frame receiving alpha values. + /// The effective H.273 color description defining the luma range. + /// The progressive-frame 4:2:0 chroma sample location. + /// The visible luma rectangle within the coded picture. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the top-left portion of the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + public static void ComposeAlpha( + Configuration configuration, + HevcPictureBuffer picture, + ImageFrame destination, + CicpProfile colorProfile, + HevcChromaSampleLocation chromaSampleLocation, + Rectangle sourceRectangle, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel + { + HeifColorConversionParameters parameters = GetConversionParameters(picture, colorProfile, out _); + HevcPlanarSampleBuffer buffer = new(picture, chromaSampleLocation); + HeifPlanarAlphaCompositor.Compose( + configuration, + buffer, + destination, + in parameters, + sourceRectangle, + outputSize, + destinationRectangle, + premultiplied); + } + /// /// Converts packed pixels to the configured HEVC component planes. /// @@ -65,7 +103,7 @@ internal static class HevcYuvConverter { HeifColorConversionParameters parameters = GetConversionParameters(picture, colorProfile, out HeifColorConversionMode mode); HevcPlanarSampleBuffer buffer = new(picture, chromaSampleLocation); - HeifPlanarColorConverter.ConvertFromRgb( + HeifPlanarColorConverter.ConvertFromRgb( configuration, image, buffer, diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContext.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContext.cs index a2b2cd611..728297f9c 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContext.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContext.cs @@ -9,7 +9,33 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; internal struct HevcCabacContext { /// - /// Maps each packed context state to the state that follows its most-probable symbol. + /// The packed probability-state index and most-probable-symbol value. + /// + private byte state; + + /// + /// Initializes a new instance of the struct. + /// + /// The luma quantization parameter that selects the initial probability. + /// The syntax-element initialization value. + public HevcCabacContext(int quantizationParameter, byte initializationValue) + { + int clippedQuantizationParameter = Math.Clamp(quantizationParameter, 0, 51); + int slope = ((initializationValue >> 4) * 5) - 45; + int offset = ((initializationValue & 15) << 3) - 16; + int initializationState = Math.Clamp( + ((slope * clippedQuantizationParameter) >> 4) + offset, + 1, + 126); + + bool mostProbableSymbol = initializationState >= 64; + this.state = (byte)( + ((mostProbableSymbol ? initializationState - 64 : 63 - initializationState) << 1) + + (mostProbableSymbol ? 1 : 0)); + } + + /// + /// Gets a mapping from each packed context state to the state that follows its most-probable symbol. /// // ReadOnlySpan allows the compiler to embed both normative tables in static data instead of allocating // mutable arrays when this type is initialized. @@ -26,7 +52,7 @@ internal struct HevcCabacContext ]; /// - /// Maps each packed context state to the state that follows its least-probable symbol. + /// Gets a mapping from each packed context state to the state that follows its least-probable symbol. /// private static ReadOnlySpan LeastProbableStateTransitions => [ @@ -40,32 +66,6 @@ internal struct HevcCabacContext 72, 73, 72, 73, 74, 75, 74, 75, 74, 75, 76, 77, 76, 77, 126, 127 ]; - /// - /// The packed probability-state index and most-probable-symbol value. - /// - private byte state; - - /// - /// Initializes a new instance of the struct. - /// - /// The luma quantization parameter that selects the initial probability. - /// The syntax-element initialization value. - public HevcCabacContext(int quantizationParameter, byte initializationValue) - { - int clippedQuantizationParameter = Math.Clamp(quantizationParameter, 0, 51); - int slope = ((initializationValue >> 4) * 5) - 45; - int offset = ((initializationValue & 15) << 3) - 16; - int initializationState = Math.Clamp( - ((slope * clippedQuantizationParameter) >> 4) + offset, - 1, - 126); - - bool mostProbableSymbol = initializationState >= 64; - this.state = (byte)( - ((mostProbableSymbol ? initializationState - 64 : 63 - initializationState) << 1) - + (mostProbableSymbol ? 1 : 0)); - } - /// /// Gets the probability-state index used to select the least-probable-symbol range. /// diff --git a/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs index e6899994d..23a74809f 100644 --- a/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Formats.Heif.Hevc; using SixLabors.ImageSharp.Formats.Heif.Hevc.Color; using SixLabors.ImageSharp.Metadata; @@ -13,7 +14,7 @@ namespace SixLabors.ImageSharp.Formats.Heif; /// Decodes a single HEVC-coded HEIF image item. /// /// The destination pixel type. -internal sealed class HevcHeifItemDecoder : IHeifItemDecoder +internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IHeifAlphaItemDecoder where TPixel : unmanaged, IPixel { /// @@ -41,9 +42,121 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder Span data, CicpProfile? colorProfile, CancellationToken cancellationToken) + { + using HevcPictureDecoder decoder = DecodePicture( + options, + item, + data, + colorProfile, + cancellationToken, + out HevcCodecConfiguration codecConfiguration, + out HevcSequenceParameterSet sequenceParameterSet, + out CicpProfile effectiveColorProfile, + out HevcChromaSampleLocation chromaSampleLocation); + + ImageFrame? frame = null; + try + { + frame = new ImageFrame(options.Configuration, sequenceParameterSet.DisplayWidth, sequenceParameterSet.DisplayHeight); + HevcYuvConverter.ConvertToRgb( + options.Configuration, + decoder.Picture, + frame, + effectiveColorProfile, + chromaSampleLocation, + sequenceParameterSet.ConformanceWindowLeftOffset, + sequenceParameterSet.ConformanceWindowTopOffset); + + ImageMetadata metadata = new() + { + CicpProfile = effectiveColorProfile.DeepClone() + }; + + HeifMetadata heifMetadata = metadata.GetHeifMetadata(); + heifMetadata.CompressionMethod = this.CompressionMethod; + heifMetadata.BitDepth = codecConfiguration.BitDepth; + heifMetadata.IsMonochrome = codecConfiguration.IsMonochrome; + return new Image(options.Configuration, metadata, [frame]); + } + catch + { + // Ownership transfers only after the image constructor accepts the completely converted frame. + frame?.Dispose(); + throw; + } + } + + /// + public void DecodeAlphaItemData( + DecoderOptions options, + HeifItem item, + Span data, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied, + CancellationToken cancellationToken) + { + using HevcPictureDecoder decoder = DecodePicture( + options, + item, + data, + item.CicpProfile, + cancellationToken, + out _, + out HevcSequenceParameterSet sequenceParameterSet, + out CicpProfile effectiveColorProfile, + out HevcChromaSampleLocation chromaSampleLocation); + + Rectangle sourceRectangle = new( + sequenceParameterSet.ConformanceWindowLeftOffset, + sequenceParameterSet.ConformanceWindowTopOffset, + sequenceParameterSet.DisplayWidth, + sequenceParameterSet.DisplayHeight); + + if (decoder.Picture.ChromaFormat != 0) + { + throw new InvalidImageContentException($"HEVC alpha image item {item.Id} is not monochrome."); + } + + HevcYuvConverter.ComposeAlpha( + options.Configuration, + decoder.Picture, + destination, + effectiveColorProfile, + chromaSampleLocation, + sourceRectangle, + outputSize, + destinationRectangle, + premultiplied); + } + + /// + /// Validates and reconstructs one HEVC image item while retaining the native picture for its caller. + /// + /// The general options governing the containing HEIF decode. + /// The HEVC image item being decoded. + /// The encoded HEVC payload. + /// The container color description that takes precedence over bitstream color information. + /// The token used to cancel the payload decode. + /// Receives the validated HEVC codec configuration. + /// Receives the sequence parameters describing the visible picture. + /// Receives the effective CICP description used for presentation. + /// Receives the progressive-frame chroma sample location. + /// The decoder owning the reconstructed native picture. Ownership transfers to the caller. + private static HevcPictureDecoder DecodePicture( + DecoderOptions options, + HeifItem item, + Span data, + CicpProfile? colorProfile, + CancellationToken cancellationToken, + out HevcCodecConfiguration codecConfiguration, + out HevcSequenceParameterSet sequenceParameterSet, + out CicpProfile effectiveColorProfile, + out HevcChromaSampleLocation chromaSampleLocation) { cancellationToken.ThrowIfCancellationRequested(); - HevcCodecConfiguration codecConfiguration = item.HevcCodecConfiguration + codecConfiguration = item.HevcCodecConfiguration ?? throw new InvalidImageContentException($"HEVC image item {item.Id} has no codec configuration property."); if (item.ChannelBitDepths is not null) @@ -53,12 +166,12 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder HevcImageItemBitstream bitstream = new(data, codecConfiguration); HevcPictureParameterSet pictureParameterSet = bitstream.SliceSegments[0].PictureParameterSet; - HevcSequenceParameterSet sequenceParameterSet = pictureParameterSet.SequenceParameterSet; + sequenceParameterSet = pictureParameterSet.SequenceParameterSet; HevcVideoUsabilityInformation? vui = sequenceParameterSet.VideoUsabilityInformation; // ISO BMFF color information takes precedence when both the container and HEVC VUI describe the image. // Otherwise, retain the VUI values used by conversion so bitstream-only color information reaches metadata. - CicpProfile effectiveColorProfile = colorProfile is not null + effectiveColorProfile = colorProfile is not null ? new CicpProfile( (byte)colorProfile.ColorPrimaries, (byte)colorProfile.TransferCharacteristics, @@ -70,42 +183,20 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder vui?.ColorDescriptionPresent == true ? vui.MatrixCoefficients : (byte)CicpMatrixCoefficients.Unspecified, vui?.VideoSignalTypePresent == true && vui.FullRange); - HevcChromaSampleLocation chromaSampleLocation = vui?.ChromaLocationInfoPresent == true + chromaSampleLocation = vui?.ChromaLocationInfoPresent == true ? vui.ChromaSampleLocationTopField : HevcChromaSampleLocation.Left; - using HevcPictureDecoder decoder = new(options.Configuration, pictureParameterSet); - decoder.Decode(bitstream); - cancellationToken.ThrowIfCancellationRequested(); - - ImageFrame? frame = null; + HevcPictureDecoder decoder = new(options.Configuration, pictureParameterSet); try { - frame = new ImageFrame(options.Configuration, sequenceParameterSet.DisplayWidth, sequenceParameterSet.DisplayHeight); - HevcYuvConverter.ConvertToRgb( - options.Configuration, - decoder.Picture, - frame, - effectiveColorProfile, - chromaSampleLocation, - sequenceParameterSet.ConformanceWindowLeftOffset, - sequenceParameterSet.ConformanceWindowTopOffset); - - ImageMetadata metadata = new() - { - CicpProfile = effectiveColorProfile.DeepClone() - }; - - HeifMetadata heifMetadata = metadata.GetHeifMetadata(); - heifMetadata.CompressionMethod = this.CompressionMethod; - heifMetadata.BitDepth = codecConfiguration.BitDepth; - heifMetadata.IsMonochrome = codecConfiguration.IsMonochrome; - return new Image(options.Configuration, metadata, [frame]); + decoder.Decode(bitstream); + cancellationToken.ThrowIfCancellationRequested(); + return decoder; } catch { - // Ownership transfers only after the image constructor accepts the completely converted frame. - frame?.Dispose(); + decoder.Dispose(); throw; } } diff --git a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs index 3b2ab914b..39dec7ba9 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/ColorConverters/JpegColorConverter.Operator.cs @@ -4,7 +4,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.Formats.Jpeg.Components; diff --git a/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/L16.PixelOperations.cs b/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/L16.PixelOperations.cs index b79d2a5b2..348ff35c4 100644 --- a/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/L16.PixelOperations.cs +++ b/src/ImageSharp/PixelFormats/PixelImplementations/PixelOperations/L16.PixelOperations.cs @@ -2,6 +2,10 @@ // Licensed under the Six Labors Split License. using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; namespace SixLabors.ImageSharp.PixelFormats; @@ -18,9 +22,163 @@ public partial struct L16 // Alpha is implicitly one, so both outward representations already contain associated color components. /// - protected override void ToAssociatedVector4(Configuration configuration, ReadOnlySpan source, Span destination) => this.ToUnassociatedVector4(configuration, source, destination); + protected override void ToUnassociatedVector4( + Configuration configuration, + ReadOnlySpan source, + Span destination) + => ConvertToVector4(source, destination); /// - protected override void ToAssociatedScaledVector4(Configuration configuration, ReadOnlySpan source, Span destination) => this.ToUnassociatedScaledVector4(configuration, source, destination); + protected override void ToUnassociatedScaledVector4( + Configuration configuration, + ReadOnlySpan source, + Span destination) + => ConvertToVector4(source, destination); + + /// + protected override void ToAssociatedVector4( + Configuration configuration, + ReadOnlySpan source, + Span destination) + => this.ToUnassociatedVector4(configuration, source, destination); + + /// + protected override void ToAssociatedScaledVector4( + Configuration configuration, + ReadOnlySpan source, + Span destination) + => this.ToUnassociatedScaledVector4(configuration, source, destination); + + /// + /// Expands packed luminance samples into normalized RGB vectors with opaque alpha. + /// + /// The packed luminance samples. + /// The destination vectors. + private static void ConvertToVector4(ReadOnlySpan source, Span destination) + { + ref ushort sourceBase = ref Unsafe.As(ref MemoryMarshal.GetReference(source)); + ref Vector4 destinationBase = ref MemoryMarshal.GetReference(destination); + int length = source.Length; + int i = 0; + + if (Vector512.IsHardwareAccelerated) + { + Vector512 maximum = Vector512.Create((float)ushort.MaxValue); + int samplesPerVector = Vector512.Count; + int oneVectorFromEnd = length - samplesPerVector; + + for (; i <= oneVectorFromEnd; i += samplesPerVector) + { + Vector512 packed = Vector512.LoadUnsafe(ref sourceBase, (nuint)i); + (Vector512 lower, Vector512 upper) = Vector512.Widen(packed); + + StoreLuminanceVectors(Vector512.ConvertToSingle(lower.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)i)); + StoreLuminanceVectors(Vector512.ConvertToSingle(upper.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)(i + (samplesPerVector / 2)))); + } + } + + if (Vector256.IsHardwareAccelerated) + { + Vector256 maximum = Vector256.Create((float)ushort.MaxValue); + int samplesPerVector = Vector256.Count; + int oneVectorFromEnd = length - samplesPerVector; + + for (; i <= oneVectorFromEnd; i += samplesPerVector) + { + Vector256 packed = Vector256.LoadUnsafe(ref sourceBase, (nuint)i); + (Vector256 lower, Vector256 upper) = Vector256.Widen(packed); + + StoreLuminanceVectors(Vector256.ConvertToSingle(lower.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)i)); + StoreLuminanceVectors(Vector256.ConvertToSingle(upper.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)(i + (samplesPerVector / 2)))); + } + } + + if (Vector128.IsHardwareAccelerated) + { + Vector128 maximum = Vector128.Create((float)ushort.MaxValue); + int samplesPerVector = Vector128.Count; + int oneVectorFromEnd = length - samplesPerVector; + + for (; i <= oneVectorFromEnd; i += samplesPerVector) + { + Vector128 packed = Vector128.LoadUnsafe(ref sourceBase, (nuint)i); + (Vector128 lower, Vector128 upper) = Vector128.Widen(packed); + + StoreLuminanceVectors(Vector128.ConvertToSingle(lower.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)i)); + StoreLuminanceVectors(Vector128.ConvertToSingle(upper.AsInt32()) / maximum, ref Unsafe.Add(ref destinationBase, (uint)(i + (samplesPerVector / 2)))); + } + } + + for (; i < length; i++) + { + Unsafe.Add(ref destinationBase, (uint)i) = Unsafe.As(ref Unsafe.Add(ref sourceBase, (uint)i)).ToVector4(); + } + } + + /// + /// Replicates sixteen normalized luminance samples into sixteen RGB vectors with opaque alpha. + /// + /// The normalized luminance samples. + /// The first destination vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreLuminanceVectors(Vector512 source, ref Vector4 destination) + { + Vector512 rgbMask = Vector512.Create(-1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1, 0, -1, -1, -1, 0); + Vector512 opaqueAlpha = Vector512.Create(0F, 0F, 0F, 1F, 0F, 0F, 0F, 1F, 0F, 0F, 0F, 1F, 0F, 0F, 0F, 1F); + Vector512 indices0 = Vector512.Create(0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3); + Vector512 indices1 = Vector512.Create(4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7); + Vector512 indices2 = Vector512.Create(8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11); + Vector512 indices3 = Vector512.Create(12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15); + ref Vector512 destinationBase = ref Unsafe.As>(ref destination); + + // Native indexed shuffles expand four luminance values per store. Clearing every fourth lane before + // inserting one preserves the implicit opaque alpha without scalar lane extraction. + destinationBase = (Vector512.ShuffleNative(source, indices0) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 1) = (Vector512.ShuffleNative(source, indices1) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 2) = (Vector512.ShuffleNative(source, indices2) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 3) = (Vector512.ShuffleNative(source, indices3) & rgbMask.AsSingle()) | opaqueAlpha; + } + + /// + /// Replicates eight normalized luminance samples into eight RGB vectors with opaque alpha. + /// + /// The normalized luminance samples. + /// The first destination vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreLuminanceVectors(Vector256 source, ref Vector4 destination) + { + Vector256 rgbMask = Vector256.Create(-1, -1, -1, 0, -1, -1, -1, 0); + Vector256 opaqueAlpha = Vector256.Create(0F, 0F, 0F, 1F, 0F, 0F, 0F, 1F); + Vector256 indices0 = Vector256.Create(0, 0, 0, 0, 1, 1, 1, 1); + Vector256 indices1 = Vector256.Create(2, 2, 2, 2, 3, 3, 3, 3); + Vector256 indices2 = Vector256.Create(4, 4, 4, 4, 5, 5, 5, 5); + Vector256 indices3 = Vector256.Create(6, 6, 6, 6, 7, 7, 7, 7); + ref Vector256 destinationBase = ref Unsafe.As>(ref destination); + + destinationBase = (Vector256.ShuffleNative(source, indices0) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 1) = (Vector256.ShuffleNative(source, indices1) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 2) = (Vector256.ShuffleNative(source, indices2) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 3) = (Vector256.ShuffleNative(source, indices3) & rgbMask.AsSingle()) | opaqueAlpha; + } + + /// + /// Replicates four normalized luminance samples into four RGB vectors with opaque alpha. + /// + /// The normalized luminance samples. + /// The first destination vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static void StoreLuminanceVectors(Vector128 source, ref Vector4 destination) + { + Vector128 rgbMask = Vector128.Create(-1, -1, -1, 0); + Vector128 opaqueAlpha = Vector128.Create(0F, 0F, 0F, 1F); + ref Vector128 destinationBase = ref Unsafe.As>(ref destination); + + // The immediate controls broadcast one source lane to RGB. The mask replaces the fourth lane with the + // implicit alpha value without extracting an individual sample from the SIMD register. + destinationBase = (Vector128_.ShuffleNative(source, 0b_00_00_00_00) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 1) = (Vector128_.ShuffleNative(source, 0b_01_01_01_01) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 2) = (Vector128_.ShuffleNative(source, 0b_10_10_10_10) & rgbMask.AsSingle()) | opaqueAlpha; + Unsafe.Add(ref destinationBase, 3) = (Vector128_.ShuffleNative(source, 0b_11_11_11_11) & rgbMask.AsSingle()) | opaqueAlpha; + } } } diff --git a/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs b/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs index c9942418c..32949c667 100644 --- a/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs +++ b/tests/ImageSharp.Benchmarks/Codecs/Heif/Av1TransformBenchmarks.cs @@ -142,6 +142,21 @@ public class Av1TransformBenchmarks return this.coefficients[^1]; } + /// + /// Measures the Vector512 thirty-two-by-thirty-two forward DCT traversal. + /// + /// The last coefficient written by the transform. + [Benchmark] + [BenchmarkCategory("Forward32x32")] + public int Forward32x32Vector512() + { + Av1Transform2dFlipConfiguration config = CreateForwardConfiguration(Av1TransformSize.Size32x32, 10); + Av1ForwardTransformer.Transform2dVector512( + this.spatial, this.coefficients, 32, ref config, this.workspace); + + return this.coefficients[^1]; + } + /// /// Measures runtime dispatch of a thirty-two-by-thirty-two forward DCT block. /// @@ -270,6 +285,21 @@ public class Av1TransformBenchmarks return this.reconstruction[^1]; } + /// + /// Measures the Vector512 thirty-two-by-thirty-two inverse DCT and byte reconstruction traversal. + /// + /// The last reconstructed sample. + [Benchmark] + [BenchmarkCategory("Inverse32x32")] + public byte Inverse32x32Vector512() + { + Av1Transform2dFlipConfiguration config = CreateInverseConfiguration(Av1TransformSize.Size32x32, 8); + Av1Inverse2dTransformer.Transform2dVector512( + this.coefficients, this.prediction, 32, this.reconstruction, 32, ref config, this.workspace, 8); + + return this.reconstruction[^1]; + } + /// /// Measures runtime dispatch of a thirty-two-by-thirty-two inverse DCT and byte reconstruction block. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs index 5971b84f0..ae660a562 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ForwardTransformTests.cs @@ -4,19 +4,56 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; +using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1ForwardTransformTests { + /// + /// The hardware configurations covering every transform SIMD tier and the scalar fallback. + /// + private const HwIntrinsics TransformConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + /// /// Gets every normative transform size, type, and bit-depth combination exercised by the forward and inverse suites. /// public static TheoryData ValidTransformCases { get; } = CreateValidTransformCases(); + /// + /// Verifies DCT operator parity across the supported hardware feature levels. + /// [Fact] public void DctOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertDctOperatorParity, TransformConfigurations); + + /// + /// Verifies ADST operator parity across the supported hardware feature levels. + /// + [Fact] + public void AdstOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertAdstOperatorParity, TransformConfigurations); + + /// + /// Verifies identity operator parity across the supported hardware feature levels. + /// + [Fact] + public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertIdentityOperatorParity, TransformConfigurations); + + /// + /// Verifies the complete sixteen-lane two-dimensional traversal matrix across hardware feature levels. + /// + [Fact] + public void Vector512KernelsMatchScalarForEveryApplicableConfiguration() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertVector512TransformParity, TransformConfigurations); + + /// + /// Verifies the forward DCT operators against their scalar implementations. + /// + private static void AssertDctOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); @@ -25,16 +62,20 @@ public class Av1ForwardTransformTests AssertOperatorParity(64); } - [Fact] - public void AdstOperatorsProduceIdenticalScalarAndSimdResults() + /// + /// Verifies the forward ADST operators against their scalar implementations. + /// + private static void AssertAdstOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); AssertOperatorParity(16); } - [Fact] - public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() + /// + /// Verifies the forward identity operators against their scalar implementations. + /// + private static void AssertIdentityOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); @@ -81,6 +122,11 @@ public class Av1ForwardTransformTests Assert.Equal(0, allocated); } + /// + /// Compares one forward transform operator across scalar and all SIMD lane widths. + /// + /// The forward transform operator. + /// The transform length. private static void AssertOperatorParity(int length) where TOperator : struct, IAv1Transform1dOperator { @@ -98,6 +144,9 @@ public class Av1ForwardTransformTests Av1TransformVector> input256 = default; Av1TransformVector> output256 = default; Av1TransformVector> step256 = default; + Av1TransformVector> input512 = default; + Av1TransformVector> output512 = default; + Av1TransformVector> step512 = default; for (int index = 0; index < length; index++) { @@ -116,16 +165,36 @@ public class Av1ForwardTransformTests GetInputValue(index, 5), GetInputValue(index, 6), GetInputValue(index, 7)); + + input512[index] = Vector512.Create( + GetInputValue(index, 0), + GetInputValue(index, 1), + GetInputValue(index, 2), + GetInputValue(index, 3), + GetInputValue(index, 4), + GetInputValue(index, 5), + GetInputValue(index, 6), + GetInputValue(index, 7), + GetInputValue(index, 8), + GetInputValue(index, 9), + GetInputValue(index, 10), + GetInputValue(index, 11), + GetInputValue(index, 12), + GetInputValue(index, 13), + GetInputValue(index, 14), + GetInputValue(index, 15)); + } TOperator.Transform(ref input128, ref output128, ref step128, cosBit, stageRange); TOperator.Transform(ref input256, ref output256, ref step256, cosBit, stageRange); + TOperator.Transform(ref input512, ref output512, ref step512, cosBit, stageRange); int[] scalarInput = new int[length]; int[] scalarOutput = new int[length]; int[] scalarStep = new int[length]; - for (int lane = 0; lane < Vector256.Count; lane++) + for (int lane = 0; lane < Vector512.Count; lane++) { for (int index = 0; index < length; index++) { @@ -136,7 +205,12 @@ public class Av1ForwardTransformTests for (int index = 0; index < length; index++) { - Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); + Assert.Equal(scalarOutput[index], output512[index].GetElement(lane)); + + if (lane < Vector256.Count) + { + Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); + } if (lane < Vector128.Count) { @@ -146,6 +220,35 @@ public class Av1ForwardTransformTests } } + /// + /// Runs every valid forward transform configuration capable of filling a sixteen-lane tile. + /// + private static void AssertVector512TransformParity() + { + for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) + { + if (transformSize.GetWidth() < Vector512.Count || transformSize.GetHeight() < Vector512.Count) + { + continue; + } + + for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) + { + Av1Transform2dFlipConfiguration allowedConfig = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, 8); + if (!allowedConfig.IsAllowed()) + { + continue; + } + + for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) + { + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateForward(transformType, transformSize, bitDepth); + DispatchColumn(transformType, transformSize, bitDepth, ref config); + } + } + } + } + /// /// Creates the complete normative transform matrix shared by the forward and inverse parity tests. /// @@ -352,7 +455,22 @@ public class Av1ForwardTransformTests Assert.Equal(scalar, vector256); } + + if (width >= Vector512.Count && height >= Vector512.Count) + { + int[] vector512 = new int[coefficientCount]; + int[] vector512Workspace = new int[workspaceLength]; + Av1ForwardTransformer.Transform2dVector512(input, vector512, (uint)inputStride, ref config, vector512Workspace); + + Assert.Equal(scalar, vector512); + } } + /// + /// Produces deterministic bounded input for one transform position and SIMD lane. + /// + /// The position within the transform. + /// The SIMD lane index. + /// The input value. private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 1023) - 511; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs index 5ac451f61..b832ec808 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseQuantizationTests.cs @@ -2,7 +2,7 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1; -using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 2f94b7019..5674158cc 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -6,14 +6,44 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse; +using SixLabors.ImageSharp.Tests.TestUtilities; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1InverseTransformTests { + /// + /// The hardware configurations covering every transform SIMD tier and the scalar fallback. + /// + private const HwIntrinsics TransformConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + + /// + /// Verifies DCT operator parity across the supported hardware feature levels. + /// [Fact] public void DctOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertDctOperatorParity, TransformConfigurations); + + /// + /// Verifies ADST operator parity across the supported hardware feature levels. + /// + [Fact] + public void AdstOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertAdstOperatorParity, TransformConfigurations); + + /// + /// Verifies identity operator parity across the supported hardware feature levels. + /// + [Fact] + public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertIdentityOperatorParity, TransformConfigurations); + + /// + /// Verifies the inverse DCT operators against their scalar implementations. + /// + private static void AssertDctOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); @@ -22,16 +52,20 @@ public class Av1InverseTransformTests AssertOperatorParity(64); } - [Fact] - public void AdstOperatorsProduceIdenticalScalarAndSimdResults() + /// + /// Verifies the inverse ADST operators against their scalar implementations. + /// + private static void AssertAdstOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); AssertOperatorParity(16); } - [Fact] - public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() + /// + /// Verifies the inverse identity operators against their scalar implementations. + /// + private static void AssertIdentityOperatorParity() { AssertOperatorParity(4); AssertOperatorParity(8); @@ -39,6 +73,13 @@ public class Av1InverseTransformTests AssertOperatorParity(32); } + /// + /// Verifies the complete sixteen-lane two-dimensional inverse traversal matrix across hardware feature levels. + /// + [Fact] + public void Vector512KernelsMatchScalarForEveryApplicableConfiguration() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertVector512TransformParity, TransformConfigurations); + [Theory] [InlineData((int)Av1TransformSize.Size4x4, 0, -4)] [InlineData((int)Av1TransformSize.Size8x8, -1, -4)] @@ -209,6 +250,11 @@ public class Av1InverseTransformTests Assert.All(reconstruction, value => Assert.Equal((short)0, value)); } + /// + /// Compares one inverse transform operator across scalar and all SIMD lane widths. + /// + /// The inverse transform operator. + /// The transform length. private static void AssertOperatorParity(int length) where TOperator : struct, IAv1Transform1dOperator { @@ -226,6 +272,9 @@ public class Av1InverseTransformTests Av1TransformVector> input256 = default; Av1TransformVector> output256 = default; Av1TransformVector> step256 = default; + Av1TransformVector> input512 = default; + Av1TransformVector> output512 = default; + Av1TransformVector> step512 = default; for (int index = 0; index < length; index++) { @@ -244,16 +293,36 @@ public class Av1InverseTransformTests GetInputValue(index, 5), GetInputValue(index, 6), GetInputValue(index, 7)); + + input512[index] = Vector512.Create( + GetInputValue(index, 0), + GetInputValue(index, 1), + GetInputValue(index, 2), + GetInputValue(index, 3), + GetInputValue(index, 4), + GetInputValue(index, 5), + GetInputValue(index, 6), + GetInputValue(index, 7), + GetInputValue(index, 8), + GetInputValue(index, 9), + GetInputValue(index, 10), + GetInputValue(index, 11), + GetInputValue(index, 12), + GetInputValue(index, 13), + GetInputValue(index, 14), + GetInputValue(index, 15)); + } TOperator.Transform(ref input128, ref output128, ref step128, cosBit, stageRange); TOperator.Transform(ref input256, ref output256, ref step256, cosBit, stageRange); + TOperator.Transform(ref input512, ref output512, ref step512, cosBit, stageRange); int[] scalarInput = new int[length]; int[] scalarOutput = new int[length]; int[] scalarStep = new int[length]; - for (int lane = 0; lane < Vector256.Count; lane++) + for (int lane = 0; lane < Vector512.Count; lane++) { for (int index = 0; index < length; index++) { @@ -264,7 +333,12 @@ public class Av1InverseTransformTests for (int index = 0; index < length; index++) { - Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); + Assert.Equal(scalarOutput[index], output512[index].GetElement(lane)); + + if (lane < Vector256.Count) + { + Assert.Equal(scalarOutput[index], output256[index].GetElement(lane)); + } if (lane < Vector128.Count) { @@ -274,6 +348,44 @@ public class Av1InverseTransformTests } } + /// + /// Runs every valid inverse transform configuration capable of filling a sixteen-lane tile. + /// + private static void AssertVector512TransformParity() + { + for (Av1TransformSize transformSize = 0; transformSize < Av1TransformSize.AllSizes; transformSize++) + { + if (transformSize.GetWidth() < Vector512.Count || transformSize.GetHeight() < Vector512.Count) + { + continue; + } + + for (Av1TransformType transformType = 0; transformType < Av1TransformType.AllTransformTypes; transformType++) + { + Av1Transform2dFlipConfiguration allowedConfig = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, 8); + if (!allowedConfig.IsAllowed()) + { + continue; + } + + for (int bitDepth = 8; bitDepth <= 12; bitDepth += 2) + { + Av1Transform2dFlipConfiguration config = Av1Transform2dFlipConfiguration.CreateInverse(transformType, transformSize, bitDepth); + DispatchColumn(transformType, transformSize, bitDepth, ref config); + } + } + } + } + + /// + /// Verifies that a matching one-dimensional forward and inverse operator pair reconstructs bounded input. + /// + /// The forward transform operator. + /// The inverse transform operator. + /// The compound transform type. + /// The transform-block dimensions. + /// The power-of-two scale applied by the operator pair. + /// The maximum permitted reconstruction error. private static void AssertRoundTrip(Av1TransformType transformType, Av1TransformSize transformSize, int scaleLog2, int allowedError) where TForwardOperator : struct, IAv1Transform1dOperator where TInverseOperator : struct, IAv1Transform1dOperator @@ -533,6 +645,18 @@ public class Av1InverseTransformTests Assert.Equal(scalar, vector256); } + + if (width >= Vector512.Count && height >= Vector512.Count) + { + byte[] vector512 = new byte[writeStride * height]; + int[] vector512Workspace = new int[workspaceLength]; + Array.Fill(vector512, byte.MaxValue); + + Av1Inverse2dTransformer.Transform2dVector512( + coefficients, prediction, readStride, vector512, writeStride, ref config, vector512Workspace, bitDepth); + + Assert.Equal(scalar, vector512); + } } /// @@ -594,7 +718,25 @@ public class Av1InverseTransformTests Assert.Equal(scalar, vector256); } + + if (width >= Vector512.Count && height >= Vector512.Count) + { + short[] vector512 = new short[writeStride * height]; + int[] vector512Workspace = new int[workspaceLength]; + Array.Fill(vector512, short.MinValue); + + Av1Inverse2dTransformer.Transform2dVector512( + coefficients, prediction, readStride, vector512, writeStride, ref config, vector512Workspace, bitDepth); + + Assert.Equal(scalar, vector512); + } } + /// + /// Produces deterministic bounded input for one transform position and SIMD lane. + /// + /// The position within the transform. + /// The SIMD lane index. + /// The input value. private static int GetInputValue(int index, int lane) => (((index * 73) + (lane * 151)) % 1023) - 511; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs index 38ee48541..0ec3a7145 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1YuvConverterTests.cs @@ -2,11 +2,14 @@ // Licensed under the Six Labors Split License. using System; +using System.Numerics; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Color; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.Tests.TestUtilities; using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -17,6 +20,12 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1YuvConverterTests { + /// + /// The hardware configurations covering 512-bit, 256-bit, 128-bit, and scalar conversion paths. + /// + private const HwIntrinsics AlphaConfigurations = + HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + /// /// Verifies known RGB-to-YUV values across coefficient, identity, and YCgCo matrices and sample ranges. /// @@ -918,6 +927,255 @@ public class Av1YuvConverterTests ImageComparer.Tolerant(0.002F).VerifySimilarity(image, actual); } + /// + /// Verifies that same-sized AV1 alpha composition preserves color and maps full-range luma exactly with and + /// without hardware intrinsics. + /// + [Fact] + public void ComposeAlphaMapsEightBitLumaExactlyAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + ValidateEightBitAlphaComposition, + AlphaConfigurations); + + /// + /// Verifies that scaled 10-bit and 12-bit AV1 alpha composition matches ImageSharp's established box resampler + /// with and without hardware intrinsics. + /// + [Fact] + public void ComposeAlphaScalesHighBitDepthLumaAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + ValidateHighBitDepthAlphaScaling, + AlphaConfigurations); + + /// + /// Verifies that alpha scaling remains exact when the bounded working buffer must advance through multiple + /// source-row windows. + /// + [Fact] + public void ComposeAlphaScalesAcrossMultipleWorkingWindows() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + ValidateSlidingWindowAlphaScaling, + AlphaConfigurations); + + /// + /// Verifies exact limited-range endpoints and out-of-range clamping for every supported AV1 alpha bit depth. + /// + [Fact] + public void ComposeAlphaExpandsLimitedRangeAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + ValidateLimitedRangeAlphaComposition, + AlphaConfigurations); + + /// + /// Exercises direct full-range byte alpha composition against exact code-value expansion. + /// + private static void ValidateEightBitAlphaComposition() + { + const int width = 19; + const int height = 5; + + ObuSequenceHeader sequenceHeader = CreateSequenceHeader(width, height, colorFormat: Av1ColorFormat.Yuv400); + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + using Image destination = new(width, height); + Buffer2DRegion luma = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0); + for (int y = 0; y < height; y++) + { + Span sourceRow = luma.DangerousGetRowSpan(y); + Span destinationRow = destination.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < width; x++) + { + sourceRow[x] = (byte)((x * 11) + (y * 7)); + destinationRow[x] = new Rgba64((ushort)(1000 + x), (ushort)(2000 + y), 3000, ushort.MaxValue); + } + } + + Av1YuvConverter.ComposeAlpha( + Configuration.Default, + frameBuffer, + destination.Frames.RootFrame, + destination.Size, + destination.Bounds, + false); + + for (int y = 0; y < height; y++) + { + Span sourceRow = luma.DangerousGetRowSpan(y); + Span actualRow = destination.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < width; x++) + { + Assert.Equal((ushort)(1000 + x), actualRow[x].R); + Assert.Equal((ushort)(2000 + y), actualRow[x].G); + Assert.Equal((ushort)3000, actualRow[x].B); + Assert.Equal((ushort)(sourceRow[x] * 257), actualRow[x].A); + } + } + } + + /// + /// Exercises the direct box-resize path for both supported high-bit-depth sample layouts. + /// + private static void ValidateHighBitDepthAlphaScaling() + { + const int sourceWidth = 5; + const int sourceHeight = 3; + const int destinationWidth = 9; + const int destinationHeight = 7; + + foreach (Av1BitDepth bitDepth in new[] { Av1BitDepth.TenBit, Av1BitDepth.TwelveBit }) + { + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + sourceWidth, + sourceHeight, + colorFormat: Av1ColorFormat.Yuv400, + bitDepth: bitDepth); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + using Image expected = new(sourceWidth, sourceHeight); + using Image destination = new(destinationWidth, destinationHeight, new Rgba64(1000, 2000, 3000, ushort.MaxValue)); + ushort maximum = (ushort)((1 << bitDepth.GetBitCount()) - 1); + for (int y = 0; y < sourceHeight; y++) + { + Span sourceRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0); + Span expectedRow = expected.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < sourceWidth; x++) + { + sourceRow[x] = (ushort)(((x * 223) + (y * 151)) & maximum); + expectedRow[x] = L16.FromScaledVector4(new Vector4((float)sourceRow[x] / maximum)); + } + } + + expected.Mutate(context => context.Resize(destinationWidth, destinationHeight, KnownResamplers.Box)); + Av1YuvConverter.ComposeAlpha( + Configuration.Default, + frameBuffer, + destination.Frames.RootFrame, + destination.Size, + destination.Bounds, + false); + + for (int y = 0; y < destinationHeight; y++) + { + Span expectedRow = expected.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + Span actualRow = destination.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < destinationWidth; x++) + { + Assert.Equal((ushort)1000, actualRow[x].R); + Assert.Equal((ushort)2000, actualRow[x].G); + Assert.Equal((ushort)3000, actualRow[x].B); + Assert.Equal(expectedRow[x].PackedValue, actualRow[x].A); + } + } + } + } + + /// + /// Exercises overlapping box kernels across multiple transposed source-row windows. + /// + private static void ValidateSlidingWindowAlphaScaling() + { + const int sourceWidth = 13; + const int sourceHeight = 41; + const int destinationWidth = 23; + const int destinationHeight = 17; + + Configuration configuration = Configuration.CreateDefaultInstance(); + configuration.WorkingBufferSizeHintInBytes = 1; + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + sourceWidth, + sourceHeight, + colorFormat: Av1ColorFormat.Yuv400, + bitDepth: Av1BitDepth.TwelveBit); + + using Av1FrameBuffer frameBuffer = new(configuration, sequenceHeader, Av1ColorFormat.Yuv400, false); + using Image expected = new(configuration, sourceWidth, sourceHeight); + using Image destination = new(configuration, destinationWidth, destinationHeight, new Rgba64(1000, 2000, 3000, ushort.MaxValue)); + for (int y = 0; y < sourceHeight; y++) + { + Span sourceRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, y, 0, 0); + Span expectedRow = expected.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < sourceWidth; x++) + { + sourceRow[x] = (ushort)(((x * 277) + (y * 193)) & 4095); + expectedRow[x] = L16.FromScaledVector4(new Vector4(sourceRow[x] / 4095F)); + } + } + + expected.Mutate(context => context.Resize(destinationWidth, destinationHeight, KnownResamplers.Box)); + Av1YuvConverter.ComposeAlpha( + configuration, + frameBuffer, + destination.Frames.RootFrame, + destination.Size, + destination.Bounds, + false); + + for (int y = 0; y < destinationHeight; y++) + { + Span expectedRow = expected.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + Span actualRow = destination.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < destinationWidth; x++) + { + Assert.Equal((ushort)1000, actualRow[x].R); + Assert.Equal((ushort)2000, actualRow[x].G); + Assert.Equal((ushort)3000, actualRow[x].B); + Assert.Equal(expectedRow[x].PackedValue, actualRow[x].A); + } + } + } + + /// + /// Exercises luma-range expansion and clamping for 8-bit, 10-bit, and 12-bit alpha samples. + /// + private static void ValidateLimitedRangeAlphaComposition() + { + foreach (Av1BitDepth bitDepth in new[] { Av1BitDepth.EightBit, Av1BitDepth.TenBit, Av1BitDepth.TwelveBit }) + { + int bitCount = bitDepth.GetBitCount(); + ushort minimum = (ushort)(16 << (bitCount - 8)); + ushort maximum = (ushort)(235 << (bitCount - 8)); + ushort storageMaximum = (ushort)((1 << bitCount) - 1); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + 4, + 1, + fullRange: false, + colorFormat: Av1ColorFormat.Yuv400, + bitDepth: bitDepth); + + using Av1FrameBuffer frameBuffer = new(Configuration.Default, sequenceHeader, Av1ColorFormat.Yuv400, false); + using Image destination = new(4, 1, new Rgba64(1000, 2000, 3000, ushort.MaxValue)); + if (bitDepth == Av1BitDepth.EightBit) + { + Span luma = frameBuffer.DeriveBlockPointer(Av1Plane.Y, 0, 0).DangerousGetRowSpan(0); + luma[0] = 0; + luma[1] = (byte)minimum; + luma[2] = (byte)maximum; + luma[3] = byte.MaxValue; + } + else + { + Span luma = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0); + luma[0] = 0; + luma[1] = minimum; + luma[2] = maximum; + luma[3] = storageMaximum; + } + + Av1YuvConverter.ComposeAlpha( + Configuration.Default, + frameBuffer, + destination.Frames.RootFrame, + destination.Size, + destination.Bounds, + false); + + Span actual = destination.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0); + Assert.Equal((ushort)0, actual[0].A); + Assert.Equal((ushort)0, actual[1].A); + Assert.Equal(ushort.MaxValue, actual[2].A); + Assert.Equal(ushort.MaxValue, actual[3].A); + } + } + /// /// Creates a sequence header containing the color signaling required by a conversion test. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs index 5d12a13a9..5407eaa70 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs @@ -244,6 +244,7 @@ public class HeifSequenceParserTests [InvalidAv1SampleByte], source.AsSpan(OrangeAv1ConfigurationOffset, OrangeAv1ConfigurationLength), false); + DecoderOptions options = new() { SegmentIntegrityHandling = handling }; Assert.Throws(() => Image.Load(options, data)); @@ -261,6 +262,7 @@ public class HeifSequenceParserTests [InvalidAv1SampleByte], source.AsSpan(OrangeAv1ConfigurationOffset, OrangeAv1ConfigurationLength), false); + DecoderOptions options = new() { SegmentIntegrityHandling = SegmentIntegrityHandling.IgnoreImageData }; using Image image = Image.Load(options, data); @@ -270,35 +272,50 @@ public class HeifSequenceParserTests } /// - /// Verifies that frame-aligned color and auxiliary AV1 samples are decoded together and the auxiliary luma - /// samples become the alpha channel of each presented color frame. + /// Verifies that an AV1 alpha track whose sequence header is not monochrome is rejected at the codec boundary. /// [Fact] - public void DecodeComposesFrameAlignedAv1AlphaSamples() + public void DecodeRejectsNonMonochromeAv1AlphaSamples() { byte[] source = TestFile.Create(TestImages.Heif.Orange4x4).Bytes; - byte[] data = CreateDecodableAv1SequenceWithAlphaContainer( + byte[] data = CreateAv1SequenceWithNonMonochromeAlphaContainer( source.AsSpan(OrangeAv1SampleOffset, OrangeAv1SampleLength), source.AsSpan(OrangeAv1ConfigurationOffset, OrangeAv1ConfigurationLength)); - using Image expectedColor = Image.Load(source); - using Image expectedAlpha = Image.Load(source); - using Image actual = Image.Load(data); + Assert.Throws(() => + { + using Image image = Image.Load(data); + }); + } - Assert.Equal(2, actual.Frames.Count); - Assert.True(actual.Metadata.GetHeifMetadata().HasAlpha); - foreach (ImageFrame frame in actual.Frames) + /// + /// Verifies that a genuine libavif alpha sequence composes its first retained frame from the linked monochrome + /// auxiliary track instead of returning the color frame as opaque. + /// + [Fact] + public void DecodeComposesFirstRealLibavifAlphaSequenceFrame() + { + DecoderOptions options = new() { MaxFrames = 1 }; + TestFile file = TestFile.Create(TestImages.Heif.Animated8BitWithAlphaExifXmp); + + using Image image = Image.Load(options, file.Bytes); + + Assert.Single(image.Frames); + Assert.True(image.Metadata.GetHeifMetadata().HasAlpha); + bool hasNonOpaqueSample = false; + for (int y = 0; y < image.Height && !hasNonOpaqueSample; y++) { - for (int y = 0; y < frame.Height; y++) + foreach (Rgba32 pixel in image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y)) { - for (int x = 0; x < frame.Width; x++) + if (pixel.A != byte.MaxValue) { - Rgba64 expected = Rgba64.FromRgba32(expectedColor[x, y]); - expected.A = expectedAlpha[x, y].PackedValue; - Assert.Equal(expected.ToRgba32(), frame[x, y]); + hasNonOpaqueSample = true; + break; } } } + + Assert.True(hasNonOpaqueSample); } /// @@ -1005,13 +1022,12 @@ public class HeifSequenceParserTests } /// - /// Builds two frame-aligned color and alpha AV1 tracks using a sample that is independently decodable as both - /// color and monochrome luma, allowing alpha composition to be tested without an encoder dependency. + /// Builds two frame-aligned AV1 tracks that intentionally reuse a color sample for the declared alpha track. /// /// The AV1 sample payload stored in every color and alpha frame. /// The AV1CodecConfigurationBox payload describing the sample. /// The complete synthetic AVIF byte stream. - private static byte[] CreateDecodableAv1SequenceWithAlphaContainer(ReadOnlySpan sample, ReadOnlySpan configuration) + private static byte[] CreateAv1SequenceWithNonMonochromeAlphaContainer(ReadOnlySpan sample, ReadOnlySpan configuration) { uint colorChunkOffset = FileTypeBoxLength + SyntheticFileLength; uint alphaChunkOffset = colorChunkOffset + (uint)(sample.Length * 2);