diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 5e6ad0385..dc221f0cb 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -29,13 +29,15 @@ Checkboxes may be marked complete only when the implementation and the verificat ## Delivery dashboard -Last reconciled with the source tree on 2026-08-29 against production checkpoint `019ac5648b4380a6ff4865072c96fde29ce09fad`. Committed checkpoints include the AV1 transform architecture, OBU framing, intra-block copy, 12-profile reconstruction matrix, layered-item properties, layered reference/header/CDF/motion-field state, inter-frame intra blocks, SIMD-first translational prediction, complete single-reference inter reconstruction, compound reference trees and modes, paired reference-MV derivation, reference-dependent bounded sequence decoding, allocation-free SIMD-first equal averaging, selected inter-intra prediction, selectable compound blending, OBMC, scaled-reference reconstruction, local warped prediction, non-translational global prediction, official motion-vector conformance, and selected spatial-layer presentation. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. +Last reconciled with the source tree on 2026-08-29 against production checkpoint `beb2ab86ce5e4052c667df30af8a48a860814031`. Committed checkpoints include the AV1 transform architecture, OBU framing, intra-block copy, 12-profile reconstruction matrix, layered-item properties, layered reference/header/CDF/motion-field state, inter-frame intra blocks, SIMD-first translational prediction, complete single-reference inter reconstruction, compound reference trees and modes, paired reference-MV derivation, reference-dependent bounded sequence decoding, allocation-free SIMD-first equal averaging, selected inter-intra prediction, selectable compound blending, OBMC, scaled-reference reconstruction, local warped prediction, non-translational global prediction, official motion-vector conformance, selected spatial-layer presentation, and progressive color and auxiliary-alpha conformance. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. Commit `1c58d855f70b024170ced9eb0a7005f0f9c955ad` records the complete official motion-vector conformance checkpoint. The official IVF has SHA-1 `F064290D7FCD3B3DE19020E8AEC6C43C88D3A505`, matching the pinned libaom test-data manifest, and SHA-256 `222A9050059B254DAB17CFB802FF829C778E3F93AF18961A622C8268576C1395`; its pinned-libaom Y4M has SHA-256 `D97AC78C81782CF1507549368047769DC677DBE205706458D1EE9C807DE6EC78`. Both source targets build with zero warnings and errors, the `net10.0` test-project analyzer build completes with zero errors and 1,014 pre-existing repository warnings, Roslynk reports zero compiler errors, 3,985 focused `net10.0` cases pass without failures or skips, and `git diff --check` is clean. Commit `019ac5648b4380a6ff4865072c96fde29ce09fad` records the complete selected-spatial-layer presentation checkpoint. An essential `lsel` selecting the genuine 40x40 base layer of the committed 40x40-to-80x80 progressive AVIF returns that exact pinned-libaom native reconstruction, then scales its YUV444 planes to the 80x80 `ispe` extent before color conversion and matches pinned libavif's RGBA output exactly. Retained reconstructed planes remain at coded dimensions; presentation planes use group-safe allocator-owned rows and are returned exactly once under a 1,024-byte constrained allocator. Portable `Vector128` and scalar two-times filters match pinned libyuv edge, quarter-sample, and rounding results at 8 and 12 bits through `FeatureTestRunner`. Both source targets build with zero warnings and errors, the `net10.0` test-project analyzer build completes with zero errors and 1,014 pre-existing repository warnings, Roslynk reports zero compiler errors, 156 focused reconstruction and color cases pass without failures or skips, and `git diff --check` is clean. -The working tree contains a complete, verified progressive color and auxiliary-alpha coverage checkpoint awaiting commit. Both pinned progressive AVIF fixtures pass their exact native-plane and pinned-libavif presentation comparisons through normal dispatch, AVX-512-disabled, AVX-disabled, and scalar `FeatureTestRunner` configurations. The YUV444-alpha fixture also compares every composed alpha sample with the independent final native auxiliary plane. Both complete production presentation paths pass under a 1,024-byte constrained tracked allocator with every allocation returned exactly once. Both source targets build with zero warnings and errors, the `net10.0` test-project analyzer build completes with zero errors and 1,014 pre-existing repository warnings, Roslynk reports zero compiler errors, 156 focused reconstruction and color cases pass without failures or skips, and `git diff --check` is clean. +Commit `beb2ab86ce5e4052c667df30af8a48a860814031` records the complete progressive color and auxiliary-alpha conformance checkpoint. Both pinned progressive AVIF fixtures pass their exact native-plane and pinned-libavif presentation comparisons through normal dispatch, AVX-512-disabled, AVX-disabled, and scalar `FeatureTestRunner` configurations. The YUV444-alpha fixture also compares every composed alpha sample with the independent final native auxiliary plane. Both complete production presentation paths pass under a 1,024-byte constrained tracked allocator with every allocation returned exactly once. Both source targets build with zero warnings and errors, the `net10.0` test-project analyzer build completes with zero errors and 1,014 pre-existing repository warnings, Roslynk reports zero compiler errors, 156 focused reconstruction and color cases pass without failures or skips, and `git diff --check` is clean. + +The working tree contains a complete, verified twelve-bit inverse-transform arithmetic checkpoint awaiting commit. ADST4 retains pinned libaom's signed 32-bit sine products and factorized sums and widens only its terminal rounding; Identity4 and Identity16 widen their fixed-point product and rounding bias only for the 20-bit twelve-bit row stage. The established 8/10-bit and twelve-bit column paths remain unchanged. Exact boundary vectors cover both 128-bit and 256-bit operators through normal, AVX-512-disabled, AVX-disabled, and scalar `FeatureTestRunner` configurations. Both source targets build with zero warnings and errors, the `net10.0` test-project analyzer build completes with zero errors and 1,014 pre-existing repository warnings, Roslynk reports zero compiler errors, 510 focused inverse-transform cases and 44 production reconstruction cases pass without failures or skips, and `git diff --check` is clean. Status meanings: @@ -47,13 +49,13 @@ Status meanings: Current development stage: **Stage 3 — complete AV1 still-image decoding.** The decoder retains reference/header/CDF/motion-field state, derives frame-level skip-mode references, consumes temporal segment prediction, decodes intra-coded blocks inside inter frames, and reconstructs translational single-reference, compound, inter-intra, OBMC, scaled-reference, local warped, and non-translational global prediction before residual traversal. Commit `1c58d855f70b024170ced9eb0a7005f0f9c955ad` adds exact official four-frame coverage through all ordinary inter modes, all three motion modes, every regular/smooth/sharp dual-filter pair, sub-8x8 chroma prediction, and no-round compound intermediates. Commit `019ac5648b4380a6ff4865072c96fde29ce09fad` adds exact selected-layer native reconstruction and scaled presentation. Neither AV1 nor HEVC production encoding is implemented. -Immediate checkpoint: **commit the verified progressive color and auxiliary-alpha coverage before advancing.** Both pinned progressive fixtures now pass exact production presentation under every required dispatch fallback and constrained tracked allocation; the alpha-bearing fixture additionally passes an independent sample-for-sample auxiliary-plane comparison. +Immediate checkpoint: **commit the verified twelve-bit inverse-transform arithmetic correction before advancing.** The corrected operators match pinned libaom at the exact overflow boundaries, preserve narrower stage paths, and pass the complete focused transform and production reconstruction evidence. | Order | Delivery stage | State | Delivered state | Gate that remains open | | --- | --- | --- | --- | --- | | 1 | Baseline, provenance, documentation, and public contract | In progress | Pinned codec references, a bounded image-only scope, encoder options, typed bit depth, decoder-option propagation, and extensive HEIF documentation exist. | Complete the all-file documentation audit, record a fresh Release baseline, finish distinct public HEIC/AVIF save boundaries, and close API review. | | 2 | Bounded HEIF item and image-sequence container | In progress | Still-item parsing, grids, auxiliary alpha, metadata properties, bounded image-sequence tracks, Identify, and all-sync AV1 sequence presentation are connected. | Complete adversarial boundary coverage, remaining item/property behavior, reference-dependent sequence reconstruction, and the bounded sequence writer. | -| 3 | Still-image AV1 and HEVC decoding | **Current** | HEVC reconstruction reaches exact HM/libheif fixtures across the recorded 8/10/12-bit and chroma cases. AV1 work includes bounded OBU framing, reconstruction, filters, grain, color, transforms, intra-block copy, an exact independent 12-profile bit-depth/chroma matrix through every dispatch tier, retained layered reference/header/CDF/motion-field state, temporal segment prediction, inter-frame intra blocks, SIMD-first translational prediction, exact single-reference inter reconstruction, exact equal-average and selectable compound reconstruction, inter-intra prediction, committed OBMC, scaled-reference reconstruction, local warped prediction, non-translational global prediction, selected spatial-layer presentation, and verified progressive color and auxiliary-alpha output. | Correct the audited 12-bit inverse-transform arithmetic, remove every other valid AV1 still-image unsupported branch with independent vectors, and then complete the remaining HEVC profile and Range Extensions matrix. | +| 3 | Still-image AV1 and HEVC decoding | **Current** | HEVC reconstruction reaches exact HM/libheif fixtures across the recorded 8/10/12-bit and chroma cases. AV1 work includes bounded OBU framing, reconstruction, filters, grain, color, corrected twelve-bit inverse transforms, intra-block copy, an exact independent 12-profile bit-depth/chroma matrix through every dispatch tier, retained layered reference/header/CDF/motion-field state, temporal segment prediction, inter-frame intra blocks, SIMD-first translational prediction, exact single-reference inter reconstruction, exact equal-average and selectable compound reconstruction, inter-intra prediction, committed OBMC, scaled-reference reconstruction, local warped prediction, non-translational global prediction, selected spatial-layer presentation, and verified progressive color and auxiliary-alpha output. | Remove every other valid AV1 still-image unsupported branch with independent vectors; then complete the remaining HEVC profile and Range Extensions matrix. | | 4 | Complete decoded presentation and animation | In progress | Shared SIMD-first AV1/HEVC color conversion, ICC application, grids, transforms, direct planar alpha composition, frame metadata, repetition, and independently decodable AV1 sequence samples exist. | Close the full color/ICC cross-product, HEVC sequence decoding, AV1/HEVC reference-dependent samples, frame-local metadata/alpha behavior, and independent animated decode vectors. | | 5 | AV1/AVIF encoding | Not started | RGB-to-planar conversion, forward transforms, OBU writer foundations, options, and container-writing infrastructure exist. | `HeifEncoderCore` still rejects AV1. Implement a real independently decodable lossy/lossless AV1 payload and the complete AVIF item/metadata matrix. | | 6 | HEVC/HEIC encoding | Not started | Shared input color conversion, options, and HEIF writer infrastructure exist. | `HeifEncoderCore` still rejects HEVC. Implement a real independently decodable lossy/lossless HEVC payload and the complete HEIC item/metadata matrix. | @@ -104,8 +106,8 @@ Immediate checkpoint: **commit the verified progressive color and auxiliary-alph - [x] Verify every connected inter mode and filter with exact independent content. The official pinned-libaom sequence supplies complete bit-mask coverage for all twelve ordinary inter modes, all three motion modes, and all nine regular/smooth/sharp dual-filter pairs. The committed average, selectable compound, inter-intra, OBMC, scaled-reference, local-warp, and global-warp AVIF fixtures separately require every connected reconstruction variant and pass exact native-plane comparison; the presentation-bearing fixtures also pass exact pinned-libavif output. - [x] Return the explicitly selected spatial layer or the final displayed layer, keeping reference reconstruction separate from display-only film grain. The essential-`lsel` production fixture reconstructs the selected 40x40 YUV444 base exactly, scales native component planes to the 80x80 item extent with pinned-libyuv integer rounding, and matches pinned-libavif RGBA presentation under normal/scalar dispatch. The committed final-layer fixture and film-grain matrix remain exact. Constrained tracked allocation returns every short-lived presentation plane exactly once. Commit `019ac5648b4380a6ff4865072c96fde29ce09fad` records the checkpoint. - [x] Verify color and auxiliary-alpha output exactly against both pinned libavif progressive fixtures under normal SIMD dispatch and all required `FeatureTestRunner` fallbacks. Both fixtures pass exact public presentation under normal, AVX-512-disabled, AVX-disabled, and scalar dispatch; the alpha-bearing fixture additionally matches every composed alpha sample with its pinned native auxiliary plane. Both full production paths pass under a 1,024-byte constrained tracked allocator with exactly-once returns. Both Release source targets build with zero warnings and errors, the test-project analyzer build completes with zero errors and 1,014 pre-existing warnings, Roslynk reports zero compiler errors, and all 156 focused reconstruction and color cases pass without failures or skips. -- [ ] **Queued until the progressive color and auxiliary-alpha checkpoint commit:** correct the audited 12-bit inverse ADST4, Identity4, and Identity16 SIMD arithmetic by widening only the libaom-widened multiply/accumulate operations, with exact conformant-range vectors and `FeatureTestRunner` coverage. -- [ ] Continue inventorying and removing every remaining valid AV1 still-image unsupported branch, adding exact independent compression-tool fixtures to the profile-matrix regression gate. +- [x] Correct the audited 12-bit inverse ADST4, Identity4, and Identity16 SIMD arithmetic by widening only the libaom-widened multiply/accumulate operations, with exact conformant-range vectors and `FeatureTestRunner` coverage. Both 128-bit and 256-bit operators match pinned outputs through every required hardware fallback; 510 focused inverse-transform cases, 44 production reconstruction cases, both zero-warning source builds, the zero-error analyzer build, zero Roslynk compiler errors, and clean `git diff --check` complete the verification. +- [ ] **Queued until the twelve-bit inverse-transform checkpoint commit:** continue inventorying and removing every remaining valid AV1 still-image unsupported branch, adding exact independent compression-tool fixtures to the profile-matrix regression gate. - [ ] Complete the remaining HEVC still-image profile and Range Extensions matrix with exact independent native-plane and presentation evidence. - [ ] Close shared decoded presentation, ICC, alpha, grid, transform, metadata, and animated AV1/HEVC decode gates. - [ ] Implement and independently verify real AV1/AVIF still encoding. @@ -838,7 +840,8 @@ The dashboard and immediate execution queue define the remaining critical path. - [x] Implement and independently verify local warped motion through pinned projection, direct 8/10/12-bit predictor coverage, a genuine traced two-frame bounded AVIF, exact final native and presentation comparisons, constrained allocation, and normal/scalar dispatch. Commit `0b2db86809e26757460e719a6b59c0fe58bb0586` records the checkpoint. - [x] Implement and independently verify non-translational global prediction with a genuine traced bounded sequence, exact native and presentation comparisons, constrained allocation, normal/scalar dispatch, and direct 8/10/12-bit compound production coverage. Commit `c5637ea0187df35b385bf43e2fe85cd955f01099` records the checkpoint. - [x] Complete progressive color and auxiliary-alpha verification through exact native and presentation comparisons, every required `FeatureTestRunner` fallback, and constrained tracked allocation. -- [ ] **Queued until the progressive color and auxiliary-alpha checkpoint commit:** correct the audited 12-bit inverse-transform arithmetic, remove every other unsupported valid AV1 still-image syntax path, and prove the complete AVIF decode matrix with independent inputs and scalar/SIMD parity. +- [x] Correct the audited 12-bit inverse ADST4, Identity4, and Identity16 arithmetic through exact pinned boundary vectors and SIMD/scalar feature isolation. +- [ ] **Queued until the twelve-bit inverse-transform checkpoint commit:** remove every other unsupported valid AV1 still-image syntax path and prove the complete AVIF decode matrix with independent inputs and scalar/SIMD parity. - [ ] Close Phase 4 by completing the remaining HEVC profile and Range Extensions matrix with exact native-plane and presented-image evidence. - [ ] Close Phase 5 and the decode portion of the bounded sequence ledger: color, ICC, alpha, grids, presentation transforms, reference-dependent samples, and complete animated AVIF/HEIC decode. - [ ] Close the still-image portions of Phases 0, 1, and 2 that remain as release gates: documentation, provenance, public format boundaries, API review, parser hardening, and malformed-input coverage. diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs index fad1e3f53..c60891e65 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs @@ -70,6 +70,40 @@ internal static class Av1IdentityTransform1d } } + /// + /// Scales four independent identity-transform axes with widened fixed-point products. + /// + /// The source values for four transform axes. + /// The destination values for four transform axes. + /// The number of values in each axis. + /// The fixed-point identity scale. + /// The number of fractional bits in . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void TransformWidened(ref Av1TransformVector> input, ref Av1TransformVector> output, int length, int multiplier, int fractionalBits) + { + for (int i = 0; i < length; i++) + { + output[i] = Av1Transform1dMath.MultiplyRoundWidened(input[i], multiplier, fractionalBits); + } + } + + /// + /// Scales eight independent identity-transform axes with widened fixed-point products. + /// + /// The source values for eight transform axes. + /// The destination values for eight transform axes. + /// The number of values in each axis. + /// The fixed-point identity scale. + /// The number of fractional bits in . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void TransformWidened(ref Av1TransformVector> input, ref Av1TransformVector> output, int length, int multiplier, int fractionalBits) + { + for (int i = 0; i < length; i++) + { + output[i] = Av1Transform1dMath.MultiplyRoundWidened(input[i], multiplier, fractionalBits); + } + } + /// /// Scales sixteen independent identity-transform axes in parallel. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs index ce6a60871..30c0c132a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Adst4Operator.cs @@ -84,9 +84,8 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - TransformCore(ref input, ref output, cosBit); + TransformCore(ref input, ref output, cosBit, stageRange[0] >= WidenedIntermediateBitCount); _ = step; - _ = stageRange; } /// @@ -97,9 +96,8 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - TransformCore(ref input, ref output, cosBit); + TransformCore(ref input, ref output, cosBit, stageRange[0] >= WidenedIntermediateBitCount); _ = step; - _ = stageRange; } /// @@ -108,7 +106,12 @@ internal static partial class Av1Inverse2dTransformer /// The source values for four transform axes. /// The destination values for four transform axes. /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. + private static void TransformCore( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + int cosBit, + bool widenedRound) { ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); Vector128 x0 = input.V0; @@ -116,8 +119,18 @@ internal static partial class Av1Inverse2dTransformer Vector128 x2 = input.V2; Vector128 x3 = input.V3; - // The products retain the sine-table scale across the complete matrix. The bounded transform inputs make - // the optimized kernels' wrapping 32-bit multiply/add sequence valid until the terminal rounding shift. + // Pinned libaom retains the sine-table scale in Int32 products and sums, but performs the twelve-bit row + // kernel's terminal scaling and rounding in Int64. This is the only stage whose rounding bias can overflow + // a valid Int32 fixed-point sum. + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); @@ -130,7 +143,12 @@ internal static partial class Av1Inverse2dTransformer /// The source values for eight transform axes. /// The destination values for eight transform axes. /// The fixed-point precision of the sine constants. - private static void TransformCore(ref Av1TransformVector> input, ref Av1TransformVector> output, int cosBit) + /// Whether the terminal fixed-point rounding requires signed 64-bit lanes. + private static void TransformCore( + ref Av1TransformVector> input, + ref Av1TransformVector> output, + int cosBit, + bool widenedRound) { ReadOnlySpan sinpi = Av1SinusConstants.SinusPi(cosBit); Vector256 x0 = input.V0; @@ -138,6 +156,15 @@ internal static partial class Av1Inverse2dTransformer Vector256 x2 = input.V2; Vector256 x3 = input.V3; + if (widenedRound) + { + output.V0 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); + output.V1 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); + output.V2 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); + output.V3 = Av1Transform1dMath.MultiplyAdd4WidenedRound(sinpi[1] + sinpi[2], x0, -sinpi[3], x1, sinpi[4] - sinpi[1], x2, sinpi[2] - sinpi[4], x3, cosBit); + return; + } + output.V0 = Av1Transform1dMath.MultiplyAdd4(sinpi[1], x0, sinpi[3], x1, sinpi[4], x2, sinpi[2], x3, cosBit); output.V1 = Av1Transform1dMath.MultiplyAdd4(sinpi[2], x0, sinpi[3], x1, -sinpi[1], x2, -sinpi[4], x3, cosBit); output.V2 = Av1Transform1dMath.MultiplyAdd4(sinpi[3], x0, 0, x1, -sinpi[3], x2, sinpi[3], x3, cosBit); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs index 011d3a4c0..bd4c44f54 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity16Operator.cs @@ -48,10 +48,19 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + // The doubled scale exceeds Int32 only for the 20-bit twelve-bit row range. Widen that exact product and + // rounding sequence, matching libaom without changing the established lower-range SIMD path. + if (stageRange[0] >= WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + _ = step; _ = cosBit; - _ = stageRange; } /// @@ -62,10 +71,17 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + if (stageRange[0] >= WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 16, 2 * Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + _ = step; _ = cosBit; - _ = stageRange; } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs index 7113d8986..53ea2cc99 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Identity4Operator.cs @@ -48,10 +48,19 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + // Only a twelve-bit row transform has the 20-bit input range that can overflow this fixed-point product. + // Match libaom's high-bit-depth kernel there while retaining the compact Int32 path for narrower ranges. + if (stageRange[0] >= WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + _ = step; _ = cosBit; - _ = stageRange; } /// @@ -62,10 +71,17 @@ internal static partial class Av1Inverse2dTransformer int cosBit, Av1TransformStageRange stageRange) { - Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + if (stageRange[0] >= WidenedIntermediateBitCount) + { + Av1IdentityTransform1d.TransformWidened(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + else + { + Av1IdentityTransform1d.Transform(ref input, ref output, 4, Av1Transform1dMath.NewSqrt2, Av1Transform1dMath.NewSqrt2Bits); + } + _ = step; _ = cosBit; - _ = stageRange; } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs index 9d253ed3c..eedb48e77 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.Operator.cs @@ -18,6 +18,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform; /// internal static partial class Av1Inverse2dTransformer { + /// + /// The signed stage width whose fixed-point terminal operations require widened SIMD intermediates. + /// + private const byte WidenedIntermediateBitCount = 20; + /// /// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional inverse transform. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs index 148cee2d5..85df71fff 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs @@ -396,6 +396,44 @@ internal static class Av1Transform1dMath public static Vector512 MultiplyRound(Vector512 value, int multiplier, int fractionalBits) => HalfButterfly(multiplier, value, 0, Vector512.Zero, fractionalBits); + /// + /// Multiplies and rounds four fixed-point values with signed sixty-four-bit intermediate lanes. + /// + /// The four values to scale. + /// The fixed-point multiplier. + /// The number of fractional bits in the multiplier. + /// The four rounded results. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 MultiplyRoundWidened(Vector128 value, int multiplier, int fractionalBits) + { + (Vector128 lower, Vector128 upper) = Vector128.Widen(value); + Vector128 rounding = Vector128.Create(1L << (fractionalBits - 1)); + + // Pinned libaom's high-bit-depth identity kernels multiply in signed 64-bit lanes. Widen before both the + // product and rounding addition so a valid 20-bit twelve-bit row value cannot wrap through Int32. + lower = ((lower * multiplier) + rounding) >> fractionalBits; + upper = ((upper * multiplier) + rounding) >> fractionalBits; + return Vector128.Narrow(lower, upper); + } + + /// + /// Multiplies and rounds eight fixed-point values with signed sixty-four-bit intermediate lanes. + /// + /// The eight values to scale. + /// The fixed-point multiplier. + /// The number of fractional bits in the multiplier. + /// The eight rounded results. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 MultiplyRoundWidened(Vector256 value, int multiplier, int fractionalBits) + { + (Vector256 lower, Vector256 upper) = Vector256.Widen(value); + Vector256 rounding = Vector256.Create(1L << (fractionalBits - 1)); + + lower = ((lower * multiplier) + rounding) >> fractionalBits; + upper = ((upper * multiplier) + rounding) >> fractionalBits; + return Vector256.Narrow(lower, upper); + } + /// /// Multiplies four scalar inputs by fixed-point weights and rounds their sum. /// @@ -504,4 +542,64 @@ internal static class Av1Transform1dMath Vector512 weightedSum = (input0 * weight0) + (input1 * weight1) + (input2 * weight2) + (input3 * weight3); return (weightedSum + Vector512.Create(1 << (fractionalBits - 1))) >> fractionalBits; } + + /// + /// Multiplies four sets of four inputs in signed thirty-two-bit lanes, then widens the terminal rounding step. + /// + /// The first fixed-point weight. + /// The first four input values. + /// The second fixed-point weight. + /// The second four input values. + /// The third fixed-point weight. + /// The third four input values. + /// The fourth fixed-point weight. + /// The fourth four input values. + /// The number of fractional bits in each weight. + /// The four rounded fixed-point sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 MultiplyAdd4WidenedRound( + int weight0, + Vector128 input0, + int weight1, + Vector128 input1, + int weight2, + Vector128 input2, + int weight3, + Vector128 input3, + int fractionalBits) + { + // libaom keeps conformant ADST4 sine products and their factorized sums in Int32, then widens the terminal + // scaling and rounding. Preserve that exact boundary instead of widening every transform multiplication. + Vector128 weightedSum = (input0 * weight0) + (input1 * weight1) + (input2 * weight2) + (input3 * weight3); + return MultiplyRoundWidened(weightedSum, 1, fractionalBits); + } + + /// + /// Multiplies four sets of eight inputs in signed thirty-two-bit lanes, then widens the terminal rounding step. + /// + /// The first fixed-point weight. + /// The first eight input values. + /// The second fixed-point weight. + /// The second eight input values. + /// The third fixed-point weight. + /// The third eight input values. + /// The fourth fixed-point weight. + /// The fourth eight input values. + /// The number of fractional bits in each weight. + /// The eight rounded fixed-point sums. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 MultiplyAdd4WidenedRound( + int weight0, + Vector256 input0, + int weight1, + Vector256 input1, + int weight2, + Vector256 input2, + int weight3, + Vector256 input3, + int fractionalBits) + { + Vector256 weightedSum = (input0 * weight0) + (input1 * weight1) + (input2 * weight2) + (input3 * weight3); + return MultiplyRoundWidened(weightedSum, 1, fractionalBits); + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs index 621c0f3d5..2da220186 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs @@ -41,6 +41,13 @@ public class Av1InverseTransformTests public void IdentityOperatorsProduceIdenticalScalarAndSimdResults() => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertIdentityOperatorParity, TransformConfigurations); + /// + /// Verifies the pinned-libaom widened operations at the twelve-bit inverse row-stage bounds. + /// + [Fact] + public void TwelveBitWideIntermediatesMatchPinnedLibaom() + => FeatureTestRunner.RunWithHwIntrinsicsFeature(AssertTwelveBitWideIntermediateParity, TransformConfigurations); + /// /// Verifies the inverse DCT operators against their scalar implementations. /// @@ -74,6 +81,133 @@ public class Av1InverseTransformTests AssertOperatorParity(32); } + /// + /// Exercises the exact ADST4 rounding and identity-product overflows that are possible at a twenty-bit row range. + /// + private static void AssertTwelveBitWideIntermediateParity() + { + const int cosBit = 12; + Av1TransformStageRange stageRange = default; + for (int index = 0; index < Av1Transform2dFlipConfiguration.MaxStageNumber; index++) + { + stageRange[index] = 20; + } + + Av1TransformVector> adstInput128 = default; + adstInput128.V0 = Vector128.Create(196_118, -196_118, 196_117, -196_117); + adstInput128.V1 = Vector128.Create(196_117, -196_117, 196_117, -196_117); + adstInput128.V2 = Vector128.Create(196_117, -196_117, 196_117, -196_117); + adstInput128.V3 = Vector128.Create(196_117, -196_117, 196_117, -196_117); + Av1TransformVector> adstInput256 = default; + adstInput256.V0 = Vector256.Create(196_118, -196_118, 196_117, -196_117, 196_118, -196_118, 196_117, -196_117); + adstInput256.V1 = Vector256.Create(196_117, -196_117, 196_117, -196_117, 196_117, -196_117, 196_117, -196_117); + adstInput256.V2 = adstInput256.V1; + adstInput256.V3 = adstInput256.V1; + Av1TransformVector> adstOutput128 = default; + Av1TransformVector> adstStep128 = default; + Av1TransformVector> adstOutput256 = default; + Av1TransformVector> adstStep256 = default; + + Av1Inverse2dTransformer.Adst4Operator.Transform( + ref adstInput128, + ref adstOutput128, + ref adstStep128, + cosBit, + stageRange); + + Av1Inverse2dTransformer.Adst4Operator.Transform( + ref adstInput256, + ref adstOutput256, + ref adstStep256, + cosBit, + stageRange); + + // These are the exact outputs of pinned libaom's signed Int64 terminal round. The first positive lane has an + // Int32 fixed-point sum of 2,147,482,471, so adding the 2,048 rounding bias in Int32 would wrap. + Vector128 adstExpected0 = Vector128.Create(524_288, -524_288, 524_287, -524_287); + Vector128 adstExpected1 = Vector128.Create(33_612, -33_612, 33_612, -33_612); + Vector128 adstExpected2 = Vector128.Create(160_112, -160_112, 160_111, -160_111); + Vector128 adstExpected3 = Vector128.Create(77_567, -77_567, 77_566, -77_566); + Assert.Equal(adstExpected0, adstOutput128.V0); + Assert.Equal(adstExpected1, adstOutput128.V1); + Assert.Equal(adstExpected2, adstOutput128.V2); + Assert.Equal(adstExpected3, adstOutput128.V3); + Assert.Equal(Vector256.Create(adstExpected0, adstExpected0), adstOutput256.V0); + Assert.Equal(Vector256.Create(adstExpected1, adstExpected1), adstOutput256.V1); + Assert.Equal(Vector256.Create(adstExpected2, adstExpected2), adstOutput256.V2); + Assert.Equal(Vector256.Create(adstExpected3, adstExpected3), adstOutput256.V3); + + Vector128 identityInput128 = Vector128.Create(524_287, -524_288, 524_286, -524_287); + Vector256 identityInput256 = Vector256.Create( + 524_287, + -524_288, + 524_286, + -524_287, + 370_727, + -370_728, + 262_143, + -262_144); + + AssertWidenedIdentityOperator( + 4, + identityInput128, + Vector128.Create(741_503, -741_504, 741_501, -741_503), + identityInput256, + Vector256.Create(741_503, -741_504, 741_501, -741_503, 524_322, -524_323, 370_751, -370_752), + stageRange); + + AssertWidenedIdentityOperator( + 16, + identityInput128, + Vector128.Create(1_483_005, -1_483_008, 1_483_002, -1_483_005), + identityInput256, + Vector256.Create(1_483_005, -1_483_008, 1_483_002, -1_483_005, 1_048_643, -1_048_646, 741_501, -741_504), + stageRange); + } + + /// + /// Verifies one identity operator against exact pinned-libaom widened fixed-point results. + /// + /// The inverse identity operator. + /// The identity-transform length. + /// The four-lane bounded input. + /// The exact four-lane result. + /// The eight-lane bounded input. + /// The exact eight-lane result. + /// The twelve-bit inverse row-stage range. + private static void AssertWidenedIdentityOperator( + int length, + Vector128 input128, + Vector128 expected128, + Vector256 input256, + Vector256 expected256, + Av1TransformStageRange stageRange) + where TOperator : struct, Av1Inverse2dTransformer.IAv1InverseTransform1dOperator + { + const int cosBit = 12; + Av1TransformVector> values128 = default; + Av1TransformVector> output128 = default; + Av1TransformVector> step128 = default; + Av1TransformVector> values256 = default; + Av1TransformVector> output256 = default; + Av1TransformVector> step256 = default; + + for (int index = 0; index < length; index++) + { + values128[index] = input128; + values256[index] = input256; + } + + TOperator.Transform(ref values128, ref output128, ref step128, cosBit, stageRange); + TOperator.Transform(ref values256, ref output256, ref step256, cosBit, stageRange); + + for (int index = 0; index < length; index++) + { + Assert.Equal(expected128, output128[index]); + Assert.Equal(expected256, output256[index]); + } + } + [Theory] [InlineData((int)Av1TransformSize.Size4x4, 0, -4)] [InlineData((int)Av1TransformSize.Size8x8, -1, -4)]