From 433afd1a928b26d5ea1204e83e31c777d701a891 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sat, 5 Sep 2026 11:48:15 +1000 Subject: [PATCH] Checkpoint AV1 inter encoding and shared search operators Preserve the current encoder, container sequence, and decoder integration work. Record focused Release verification and the remaining interpolation/conformance work in the implementation plan. Non-regular interpolation runtime verification remains open. --- HEIF_IMPLEMENTATION_PLAN.md | 55 +- .../Formats/Heif/Av1/Av1BitStreamWriter.cs | 78 + .../Formats/Heif/Av1/Av1Constants.cs | 45 + src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs | 79 + .../Formats/Heif/Av1/Color/Av1YuvConverter.cs | 72 +- .../Av1/Entropy/Av1FrameEntropyContext.cs | 4 +- .../Av1/Entropy/Av1MotionVectorContext.cs | 104 +- .../Heif/Av1/Entropy/Av1RateDistortion.cs | 225 ++- .../Av1/Entropy/Av1SymbolContextHelper.cs | 61 +- .../Heif/Av1/Entropy/Av1SymbolEncoder.cs | 529 +++-- .../Heif/Av1/Entropy/Av1SymbolWriter.cs | 53 +- .../Av1/Motion/Av1GlobalMotionParameters.cs | 45 + .../Heif/Av1/Motion/Av1MotionVector.cs | 10 + .../Av1/Motion/Av1ReferenceMotionVectors.cs | 459 +++-- .../Av1/OpenBitstreamUnit/ObuFrameHeader.cs | 9 + .../OpenBitstreamUnit/ObuOperatingPoint.cs | 15 + .../Heif/Av1/OpenBitstreamUnit/ObuReader.cs | 153 +- .../OpenBitstreamUnit/ObuSequenceHeader.cs | 8 + .../Heif/Av1/OpenBitstreamUnit/ObuWriter.cs | 654 +++++- .../Av1/Pipeline/Av1EncoderBlockWorkspace.cs | 92 +- .../Heif/Av1/Pipeline/Av1EncoderFrame.cs | 38 + .../Av1/Pipeline/Av1EncoderFrameBuffer.cs | 7 +- ... => Av1EncoderInterPredictionWorkspace.cs} | 35 +- .../Heif/Av1/Pipeline/Av1FrameEncoder.cs | 1700 ++++++++++++++-- ...blockEncoder.IntraBlockCopyModeDecision.cs | 586 ------ .../Av1IntraSuperblockEncoder.ModeDecision.cs | 98 +- .../Av1IntraSuperblockEncoder.Operator.cs | 570 +++--- ...raSuperblockEncoder.PaletteModeDecision.cs | 3 +- ...SuperblockEncoder.ReferenceModeDecision.cs | 1751 +++++++++++++++++ .../Heif/Av1/Pipeline/Av1IntraTileWriter.cs | 170 -- .../Pipeline/Av1ResidualBuilder.Operator.cs | 158 ++ .../Heif/Av1/Pipeline/Av1ResidualBuilder.cs | 220 +++ .../Heif/Av1/Pipeline/Av1TileEncoder.cs | 383 ++++ .../Inter/Av1TranslationalInterPredictor.cs | 2 +- .../Av1/Tiling/Av1EncoderBlockModeInfo.cs | 58 +- .../Heif/Av1/Tiling/Av1EncoderBlockStruct.cs | 12 + .../Av1/Tiling/Av1EncoderPaletteMapBuffer.cs | 18 +- .../Av1/Tiling/Av1EncoderPictureBuffer.cs | 126 +- .../Tiling/Av1EncoderSuperblockWorkspace.cs | 62 +- .../Heif/Av1/Tiling/Av1PictureControlSet.cs | 15 +- .../Av1/Tiling/Av1PictureParentControlSet.cs | 2 +- .../Formats/Heif/Av1/Tiling/Av1TileReader.cs | 9 +- .../Formats/Heif/Av1/Tiling/Av1TileWriter.cs | 337 +++- .../Alpha/HeifPlanarAlphaEncoder.cs | 102 +- .../HeifPlanarColorConverter.cs | 166 +- .../Formats/Heif/GridHeifItemDecoder.cs | 212 +- .../Formats/Heif/HeifDecoderCore.cs | 454 +++-- src/ImageSharp/Formats/Heif/HeifEncoder.cs | 8 +- .../Formats/Heif/HeifEncoderCore.Sequence.cs | 265 ++- .../Formats/Heif/HeifEncoderCore.cs | 623 +++++- src/ImageSharp/Formats/Heif/HeifItem.cs | 15 + .../Linear/FlipProcessor{TPixel}.cs | 16 +- .../Linear/RotateProcessor{TPixel}.cs | 33 +- .../Formats/Heif/Av1/Av1BitStreamTests.cs | 20 + .../Heif/Av1/Av1CoefficientsEntropyTests.cs | 174 +- .../Formats/Heif/Av1/Av1EncoderFrameTests.cs | 282 +++ .../Heif/Av1/Av1EncoderModeInfoBufferTests.cs | 187 +- .../Formats/Heif/Av1/Av1EntropyTests.cs | 218 +- .../Av1/Av1InterpolationFilterEntropyTests.cs | 44 + .../Av1/Av1IntraSuperblockEncoderTests.cs | 224 ++- .../Heif/Av1/Av1MotionVectorEntropyTests.cs | 31 + .../Heif/Av1/Av1ResidualBuilderTests.cs | 140 ++ .../Av1/Av1SingleReferenceEntropyTests.cs | 77 + .../Heif/Av1/Av1TransformBlockEncoderTests.cs | 4 +- .../Formats/Heif/Av1/ObuFrameHeaderTests.cs | 105 +- .../Formats/Heif/HeifEncoderTests.cs | 476 ++++- 66 files changed, 10630 insertions(+), 2356 deletions(-) rename src/ImageSharp/Formats/Heif/Av1/Pipeline/{Av1EncoderIntraBlockCopyWorkspace.cs => Av1EncoderInterPredictionWorkspace.cs} (76%) delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs delete mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 71b0f4961b..b3692f1d4f 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -14,10 +14,10 @@ This plan is the authoritative delivery checklist. A source file, unit test, bui - Production code must not load, invoke, install, or fall back to a native codec. - Existing independent container files may be used only as interoperability inputs. Native AV1 expected output must be generated by the current libaom `main` checkout, and no independent decoder output may substitute for it. -Reference checkout evidence on 2026-08-31: +Reference checkout evidence refreshed on 2026-09-05: -- `D:\GitHub\AOMediaCodec\aom` is clean. Its checked-out `HEAD` is `441c439b9916474cac15d2822af47a9ad70674a8`, while the refreshed `origin/main` used for the current source comparison is `d773924c767f7433d3838dbe1ba90978b10bf675`. -- The most recent independently built reference decoder uses the earlier `origin/main` tree exported to `D:\GitHub\ynse01\aom-main-a40` and built in `D:\GitHub\ynse01\aom-main-a40-build` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727`. The resulting `aomdec` identifies itself as version 3.15.0. This records the binary used for the last external acceptance pass; source comparison continues against the current revision above. +- The current encoder source comparison uses the official libaom `main` revision `d565eec60f084421fa34fc0534b760c6452b6a6c`, exported at `D:\GitHub\ynse01\aom-d565eec6-source`. +- The current independently built reference decoder is `D:\GitHub\ynse01\aom-d565eec6-build-generic2\aomdec.exe`. Its CMake cache identifies the current source export above, and it reports version 3.15.0. On 2026-09-05 it accepted both frames of each retained-reference sequence at efforts five, seven, eight, and nine. This establishes syntax acceptance for those four streams, not complete interpolation or codec conformance. ## Status notation @@ -27,14 +27,14 @@ Reference checkout evidence on 2026-08-31: ## Current source reconciliation -Reconciled with the worktree on 2026-09-04. +Reconciled with the worktree on 2026-09-05. - [~] The bounded container reader, still-image path, sequence parser, AV1 decoder, color pipeline, presentation pipeline, and broad AV1 test suite exist locally. - [x] The inter-frame decoder has verified checkpoints through inter deblocking decisions and reference/mode deltas. - [~] Loop filtering, CDEF, super-resolution, restoration, film grain, layered presentation, alpha composition, and color conversion exist locally. Shared-source cleanup changed the current tree, so final production-path verification is open. - [~] AV1 writer primitives, forward transforms, symbol encoding, and tile-writing source are connected to the public encoder for bounded still-image and all-intra sequence AVIF color with optional auxiliary alpha output. -- [~] The public AV1 encoder supports explicit single-image and bounded all-intra sequence requests. Lossless output is implemented; inter-frame sequence search, grids, orientation handling, and default format registration remain open. +- [~] The public AV1 encoder has local single-image, grid, lossless sequence, and LAST_FRAME lossy sequence paths. Current-tree verification remains open. Additional references, compound prediction, remaining inter tools, orientation handling, and default format registration remain open. - [x] Patented codec production code, registrations, tests, benchmarks, fixtures, reference outputs, and notices were manually deleted and committed by `78a74d448`. - [x] Remaining task-created HM, HEVC, libheif, GPAC, Nokia, FFmpeg, Pillow HEIF, libavif-build, and libjpeg-build directories were traced to their creation commands in the recovered Codex session history and deleted on 2026-08-31. The user-provided repositories and all libaom-only source, build, and reference data were left untouched. - [x] The PNG metadata-suppression fix and three HEIF/AV1 diagnostic-save call-site corrections passed the exact 34 net11.0 ARM CI cases and were committed with the single-reference checkpoint as `54bb6cbe59bd113058854a3ee31448cf61f462ca`. They are infrastructure evidence, not decoder or encoder completion evidence. @@ -42,6 +42,19 @@ Reconciled with the worktree on 2026-09-04. ## Immediate execution queue +Current interpolation-search checkpoint, implemented on 2026-09-05 with focused verification in progress: + +- [x] The 8x8 single-candidate SAD, four-candidate SAD, and variance paths now share closed-generic traversal in `Av1ResidualBuilder`. Its existing byte/ushort residual operators own scalar and SIMD arithmetic; the superblock operators no longer duplicate these row loops or hardware dispatch. The four-candidate path retains one source load/conversion per row, exact eight-sample loads preserve final-row bounds, and both variance moments retain native precision until the existing normalization boundary. Six known-result cases cover 8/10/12-bit signed extrema, distinct source/prediction rows and strides, unaligned starts, exact final-row lengths, and four-candidate output order. Together with nine existing intra-block-copy cases and the extended zero-allocation test, all 16 pass with hardware intrinsics enabled and all 16 pass with them disabled. All 58 public HEIF encoder cases also pass. Verification used serialized net11.0 Release Visual Studio VSTest with stop-on-failure; evidence is `artifacts/TestResults/av1-search-metrics-20260905/search-metrics-r2.trx`, `search-metrics-scalar-r2.trx`, and `search-metrics-encoder-r2.trx` in that directory. The exact Release build has zero errors and the existing 1,009 warnings; Roslyn reports no diagnostics in the changed files. This verifies the search-metric refactor, not the remaining interpolation conformance or end-to-end performance work. +- [~] Effort eight searches the common regular, smooth, and sharp interpolation families; efforts nine and ten enable independent vertical/horizontal filter selection. Lower efforts retain the fixed regular-filter path. +- [~] Filter ranking follows the curve-fit prediction-error model in the official `d565eec60f084421fa34fc0534b760c6452b6a6c` source, before full transform search. The existing rate-distortion type owns the static model tables and paired portable SIMD cubic evaluation. Visible-plane SSE uses the existing SIMD residual reduction, including native-bit-depth normalization and cropped edges. +- [~] Candidate and retained prediction views alternate within the existing inter workspace. At most one normalization copy per active plane retains the chosen predictor for transform search; no new pixel owner, coefficient owner, per-block rent, or reconstructed-frame copy is introduced. Zero-phase axes retain only the cheapest signaled filter instead of repeating equivalent prediction trials. +- [~] The selected filters, skip flags, segment, and primary reference fit the original seven-byte block-mode record and eight-byte macroblock record. Filter costing and writing use the live tile CDFs. Encoder and decoder share the context-combination mapping, and encoder search and writing share filter-symbol eligibility. +- [~] All 67 focused net11.0 Release cases pass through serialized Visual Studio VSTest: model curve samples and skip decisions, 8/10/12-bit quantizer normalization, exact entropy bytes and live adaptation, packed-field independence, tile-boundary contexts, syntax eligibility, and sequence-header signaling. All 12 model cases also pass with hardware intrinsics disabled in the test-host environment. The four retained-reference sequences are accepted by current libaom. These flat-image cases do not force non-regular filter selections; those production cases, broader current-tree verification, allocation assertions, and end-to-end timing remain required before this checkpoint can be committed. +- [x] Focused runtime verification exposed an invalid low-effort inter-frame header: `force_integer_mv` was set while screen-content tools were disabled, making the writer omit the high-precision flag that a conforming reader expects. The frame encoder now retains the inferred false flag and controls integer-only search through the existing effort boundary. All four retained-reference cases assert the parsed precision, quantizer, and filter fields and decode both frames; current libaom accepts the same saved streams. +- [x] The preceding interpolation checkpoint's affected net11.0 Release run passed 2,494 cases with zero failures or skips through one serialized Visual Studio VSTest process with stop-on-failure enabled. It combined 2,269 entropy cases, 167 frame/superblock/transform/picture-storage cases, and 58 public HEIF encoder cases. That build had zero errors and the existing 1,009 test-project warnings, with none in the changed files. Evidence: `artifacts/TestResults/av1-interpolation-20260905/verified-encoder-entropy-r10.trx` and `artifacts/av1-interpolation-net11-release-20260905-r10.log`. This predates the search-metric refactor above and is not the complete current-tree decoder/encoder release matrix. +- [x] Allocation regressions now reflect the implemented lifetimes instead of the former layout: all seven trailing tile-state integers are proved contiguous within the picture's second allocator owner; coefficient level, context, and output owners are proved allocated at construction, reused across costing, writing, finalization, and frame resets, and returned exactly once. The two-owner picture and three-owner symbol-encoder limits are retained. +- [x] Grid regressions use valid 4:2:2/4:2:0 syntax and the inferred monochrome subsampling flags used by production configuration. Odd-dimension and undersized-cell failures assert their grid-specific messages, so malformed AV1 fixtures or an earlier configuration mismatch cannot satisfy those tests. Smaller right/bottom color and alpha cells, separate primary roots, lossless sequences, metadata, and public precision/sampling cases pass in the 58-case set above. + Work must proceed in this order. Do not skip to a later item while an earlier checkpoint is open. ### 1. Finish and verify the AV1-only cleanup @@ -571,7 +584,7 @@ Previously verified algorithm checkpoints remain valuable evidence, but the fina - [x] The exact current-tree presentation matrix passes 12 of 12 cases through ImageSharp's established reference-image API on net10.0 and net11.0. - [x] Verify malformed/truncated data, frame IDs, reference slots, tile bounds, allocation limits, cancellation, and failure unwinding. - [x] Verify still items and bounded sequences from file, memory, non-seekable, and short-read streams. -- [x] Verify ICC, CICP, alpha, grids, pixel aspect ratio, clean aperture, rotation, mirroring, metadata, and every presented sequence frame. +- [~] Verify ICC, CICP, alpha, grids, pixel aspect ratio, clean aperture, rotation, mirroring, metadata, and every presented sequence frame. Grid validation now requires the first cell to be at least 64 samples on both axes, enforces even output and cell dimensions along each subsampled AV1 chroma axis, and requires every cell to cover its row-major output region without exceeding the first cell's dimensions. This accepts the smaller right and bottom cells supported by the writer while also accepting uniform coded cells whose final row and column are cropped to the grid descriptor. Auxiliary alpha uses the same cropped overlap, so padded cells cannot write outside the final frame. Production regressions cover smaller right, bottom, and bottom-right color and alpha cells; Roslynk compiler and scoped analyzer diagnostics are clean, while runtime verification of the current tree remains pending. - [x] Complete the public AVIF format/API review so registered capabilities match implemented behavior. - [x] Remove or reject every valid in-scope AV1 syntax branch that remains silently ignored or unsupported. @@ -792,14 +805,13 @@ Writer primitives are not an encoder. The public encoder remains incomplete unti ### 5. Define and enforce the encoder contract -- [x] Use official libaom `main` at `d773924c767f7433d3838dbe1ba90978b10bf675` as the encoder syntax, probability-model, transform, quantization, filtering, and bitstream reference. +- [x] Use official libaom `main` at `d565eec60f084421fa34fc0534b760c6452b6a6c` as the encoder syntax, probability-model, transform, quantization, filtering, and bitstream reference. - [x] Use the existing PNG, TIFF, and JPEG encoders as the ImageSharp architecture reference: generic `Image` input, encoder options taking precedence over converted format metadata and codec defaults, allocator-owned temporary storage, and deterministic disposal. - [x] Treat source pixel type, source alpha representation, and decoded source bit depth as conversion inputs, never as output-eligibility checks. Do not pre-scan pixels before encoding. - [x] Resolve output configuration once from explicit encoder options, converted `HeifMetadata`, and AV1 defaults in that order. Sanitize only combinations that cannot describe a legal requested output, and never write resolved values back to source metadata. -- [ ] Finalize observable options for quality, effort, lossless mode, bit depth, chroma subsampling, alpha quality, metadata, and bounded sequences. +- [~] Finalize observable options for quality, effort, lossless mode, bit depth, chroma subsampling, alpha quality, metadata, and bounded sequences. Sequence repeat-count options override converted HEIF metadata like the existing animated encoders. Legacy JPEG treats the AV1-specific lossless and bit-depth options as inapplicable and continues with its native eight-bit encoding contract. AV1 sequences now follow the existing animated-image contract: the primary item reuses the first sync sample when the root is animated, while an excluded root is encoded once as the independent primary image and the sequence begins at frame index one. Final verification remains open. - [x] Preserve high-bit-depth source precision through 16-bit RGB and native 10/12-bit component planes. -- [ ] Reject only genuinely unsupported output combinations at the public boundary before writing output. -- [ ] Register only capabilities that the completed encoder proves. +- [~] HEIF is registered through the default configuration module. Keep public AV1 capability claims limited to the paths covered by the encoder verification matrix until the remaining encoder work is complete. Encoder data-flow contract: @@ -822,23 +834,23 @@ Encoder verification contract: - [~] SIMD-first RGB-to-native-plane conversion now feeds eight-bit and high-bit-depth bordered AV1 source frames directly, preserving ImageSharp's arbitrary packed-pixel input contract without an intermediate full-frame native-plane copy. - [~] Auxiliary-alpha encoding now follows the same packed-pixel conversion boundary without scanning pixel contents or cloning the image. Source alpha is converted through ImageSharp's 16-bit pixel contract, deinterleaved with descending Vector512, Vector256, Vector128, and scalar traversal through the shared vector-count helpers, then scaled and rounded once by the existing native-sample writer directly into the final bordered monochrome source frame. One operation-wide allocator owner provides the packed and planar row views; there is no frame-sized alpha staging allocation or second owner. Exact 12-bit precision, physical border extension, the single 12-bytes-per-pixel row rent, and balanced return pass through the production converter. The complete 47-case frame-encoder set passes direct foreground net11 Release VSTest, and current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts the generated 8-, 10-, and 12-bit monochrome payloads. AVIF auxiliary item properties, references, and public activation remain open. - [~] Forward transform families, transform workspace, and an allocation-free DC intra block boundary exist locally. For eight-bit and high-bit-depth samples, the composed boundary now follows current libaom's encoder order: predict into the reconstruction plane, subtract prediction from source, transform, quantize into separate qcoeff and dqcoeff storage, retain EOB and transform type, and inverse-transform only when EOB is nonzero so later blocks consume decoder-identical references. Prediction and subtraction retain their SIMD-first operators, independent source and reconstruction strides are preserved, and no frame-sized or per-block buffer is introduced. The block boundary consumes the real bordered encoder-plane regions and indexes their one-segment owner directly; this preserves physical row strides without a row copy and avoids the per-call enumerator allocation exposed by the initial array-only test. One reusable 61 KiB allocator owner supplies tightly packed residual, aligned transform-coefficient, dequantized-coefficient, and transform scratch spans across transform blocks; quantized coefficients write directly to the retained frame coefficient owner instead of being duplicated. A fixed 8x8 DC-intra superblock baseline now traverses the same recursive preorder and frame-edge pruning as the tile writer, gathers left references into that reusable block workspace, writes luma and chroma coefficient-owner slices in the writer's exact consumption order, and updates the caller-owned reconstruction planes for subsequent predictions. Stage-by-stage scalar-oracle, physical-border, retained-syntax, superblock-to-writer synchronization, high-bit-depth precision, and steady-state zero-allocation coverage passes 8 of 8 through direct net11 VSTest in Release. This is a legal fixed baseline, not complete partition or mode analysis. -- [~] A production single-tile all-intra writer now walks raster superblocks, analyzes each immediately before entropy coding, reuses one decision workspace and one block workspace, and retains decoder-identical reconstructed references across the tile. Its closed byte and high-bit-depth operators feed the existing superblock boundary without runtime sample-type checks. A byte-exact test compares this composed path with an explicit superblock-then-tile-writer oracle, so producer and writer traversal or coefficient-area drift cannot pass unnoticed. A separate clipped 2x2-superblock regression proves global raster indexing by requiring all four coefficient segments and the bottom-right reconstruction to be populated. Multi-tile ownership and the complete frame/OBU operation remain. +- [~] The production tile writer walks raster superblocks, analyzes each immediately before entropy coding, reuses one decision workspace and one block workspace, and retains decoder-identical reconstructed references across each tile. Frames exceeding AV1's 4,096-sample tile-width or 4,096-by-2,304-sample tile-area limit now select the minimum uniform tile-column and tile-row logarithms used by current libaom. Every tile begins from the same normative frame probabilities, appends its independently finalized range-coded bytes to one bounded output allocation, and records only its offset and length in the picture-state owner. Closed byte and high-bit-depth operators feed the existing superblock boundary without runtime sample-type checks. Existing byte-exact and clipped-superblock tests cover traversal and coefficient indexing; a production 4,097-sample-wide lossless case crosses the first tile boundary and checks decoded pixels on both sides. Roslynk reports zero compiler errors; runtime and current-libaom verification of this multi-tile checkpoint remain pending. - [~] A non-owning encoder-frame view now separates visible conversion regions from coded regions and performs complete left, top, right, bottom, and corner extension across each bordered plane. Current libaom uses 8-sample-aligned coded dimensions, a 32-sample-aligned luma stride with chroma stride derived from it, and a 64-pixel luma border for non-resized all-intra encoding. One operation-ready frame owner now rents the aligned Y, U, and V storage contiguously, exposes non-owning `Buffer2D` plane views, and returns the rent exactly once. A 4K 4:2:0 frame occupies about 13.0 MiB at 8-bit or 26.0 MiB at 10/12-bit; source and reconstruction therefore remain distinct frame owners rather than adding a full-frame copy. The corrected tests use this real ownership path and verify the exact 54 KiB 64x64 4:2:0 rent. The frame-encoder operation now instantiates matching source and reconstruction owners with ordinary `using` lifetimes and converts packed pixels directly into the source owner before extension. -- [~] Temporal delimiter, sequence header, frame header, combined-frame tile-group writing, and an internal reduced-still-picture frame operation now exist locally. The remaining required metadata, padding, multi-tile, option, and public encoder paths are not complete. +- [~] Temporal delimiter, sequence header, frame header, combined-frame tile-group writing, uniform multi-tile layout, and reduced and non-reduced frame operations now exist locally. The remaining codec-tool and verification work is tracked below. - [~] Implement superblock and partition analysis for every permitted block size and partition. Efforts zero through eight deliberately split every in-frame node to 8x8 blocks. Effort nine performs recursive live rate-distortion selection at complete 8x8 and 16x16 nodes, while effort ten extends the same search to complete 32x32, 64x64, and 128x128 nodes. Candidate order matches current libaom: `PARTITION_NONE`, `PARTITION_SPLIT`, `PARTITION_HORZ`, `PARTITION_VERT`, the four asymmetric partitions, then `PARTITION_HORZ_4` and `PARTITION_VERT_4`; the two 1-to-4 partitions are excluded at 128x128 as required by current libaom. Invalid chroma geometries are excluded before evaluation. Each candidate saves and restores the exact partition, coefficient, transform, and palette neighbor edges in one aligned block-workspace owner; trials neither allocate nor copy probability state. Recursive split trials publish each selected child's decoded mode, transform, coefficient, and palette contexts before evaluating its next sibling. Coefficient contexts are published per retained transform rather than broadcasting the first transform over an entire partition leaf. Large luma and chroma leaves are evaluated as bounded-64, raster-ordered transform tiles in the existing aligned workspace, and each winning plane is copied to retained storage once. Production picture state retains the compact 8x8 mode allocation below effort nine and explicitly selects 4x4 allocation granularity when sub-8x8 partitions are enabled. Effort-dependent pruning remains. - [~] Implement intra mode search, palette, filter intra, chroma-from-luma, and intra-block copy decisions. Live luma search now covers all 13 zero-angle base modes and all six nonzero adjustments for each of the eight directional modes. Joint spatial chroma search covers the same 61 candidates, combines both chroma planes in one rate-distortion decision, and preserves the winning shared angle adjustment. Chroma-from-luma now searches the complete signed alpha alphabet from reconstructed luma and retains its joint U/V syntax. Filter-intra now searches all five predictors after ordinary luma modes. Palette entropy, retained state, production syntax, exhaustive luma and paired chroma palette selection, adaptive screen-content activation, and joint intra-block-copy mode selection are complete. -- [ ] Implement inter mode search for bounded sequences, including reference selection and the decoder-supported inter tools. +- [~] Implement inter mode search for bounded sequences, including reference selection and the decoder-supported inter tools. The sequence encoder retains the preceding reconstruction and, from effort six, searches a bounded full-pixel frame translation against that LAST_FRAME reference. Candidate discovery uses the existing SIMD-first squared-error kernels over a central analysis window, validates the winner over the complete coded luma plane, and charges its exact uncompressed-header bit count in the inter-frame rate-distortion domain. Pure translation is signaled as an identity-scale rotation/zoom model, matching current libaom's workaround for the AV1 translation-only axis defect. Each 8x8 inter-frame block first retains the complete intra candidate, then compares NEARESTMV, all three legal NEARMV dynamic-list entries, GLOBALMV, and all three legal NEWMV dynamic-list entries against it with live intra/inter, single-reference, mode, DRL, differential-vector, skip, transform, coefficient, and distortion costs. The initial NEWMV search retains the reference's cheap prediction-error stage, but every surviving mode now owns a complete transform, coefficient, skip, and distortion evaluation before mode selection; selected and candidate workspace views exchange ownership only on strict improvement. Inter trials remain in the existing shared workspace until they strictly beat the intra result, so losing trials require no backup buffer or copy. The tile writer emits the matching DRL path and normative context-selected `LAST_FRAME` reference tree instead of forcing every block through a segmentation feature. The selected DRL index reuses the filter-intra byte because those block syntax branches are mutually exclusive, preserving the existing packed state size. Packed short vectors reuse the existing picture-state owner. Effort six keeps a full-pixel fast path; effort seven refines each selected NEWMV through half- and quarter-pixel eight-tap prediction; effort eight adds the final eighth-pixel stage. The frame header advertises the matching precision, and the shared motion-vector entropy path emits fractional and high-precision symbols only when that precision permits them. Roslynk reports no compiler or scoped analyzer diagnostics, but runtime verification is pending. Additional retained reference pictures, compound prediction, and the remaining inter tools remain. - [~] Current-libaom `av1_quantize_fp_no_qmatrix` arithmetic is implemented as a closed generic forward-quantizer family with Vector512, Vector256, Vector128, and scalar paths, raster-order output, coded 64-point coefficient limits, and scan-order EOB selection. High-bit-depth paths widen before multiplying instead of applying the eight-bit coefficient clamp. Lossless blocks use the AV1 4x4 Walsh-Hadamard transform, exact lossless quantization and dequantization, four-by-four-only transform syntax, and non-skipped residual coding. Transform search and coefficient optimization remain. - [~] Implement real rate-distortion selection and make quality and effort change work, size, and output quality. The complete luma and joint chroma candidate sets, including chroma-from-luma, filter-intra, palette, and intra-block copy, now perform live rate-distortion selection. Public quality mapping and effort tiers through exhaustive uniform luma mode/transform search are implemented. Effort nine adds exact recursive 8x8 and 16x16 partition rate-distortion selection, and effort ten extends it through 128x128; effort-dependent pruning and the remaining sequence searches remain. - [~] Frame effort now progressively expands the available current search: zero is DC-only, one adds every zero-angle spatial mode, two adds every legal directional adjustment, three refines the preliminary luma winner's transform type, four adds filter-intra and chroma-from-luma, and five adds adaptive palette and intra-block-copy analysis. Lower tiers do not signal unavailable sequence or frame tools, and tiers below five skip the whole-frame screen-content scan. Effort six enables `TX_MODE_SELECT` and compares the winning ordinary spatial or filter-intra luma mode as one 8x8 transform against four raster-ordered 4x4 transforms; each luma palette candidate owns that size comparison from effort six onward. Effort seven searches every legal 8x8 transform type inside every ordinary spatial candidate rather than refining only the preliminary winner. Effort eight also performs the 8x8-versus-four-4x4 comparison inside every ordinary spatial and filter-intra candidate, matching current libaom's per-candidate uniform-transform ownership. Effort nine additionally searches every legal partition at complete 8x8 and 16x16 nodes in current-libaom order, and effort ten extends that recursive search through 128x128. Prediction and residual construction run once per mode and are reused across its legal transform types. A 128x128 leaf evaluates four 64x64 luma transforms and as many as sixteen 32x32 transforms per 4:4:4 chroma plane, retaining sparse state at coefficient-area offsets. Residual emission follows AV1's bounded-region order, completing Y, U, and V for each 64x64 luma region before advancing. Every 4x4 transform searches all legal types with live coefficient contexts and reconstructed intra references. The search reuses the aligned block workspace, preserves only global improvements, and performs no per-block, per-partition, or per-transform rent. Non-skipped intra-block copy writes and costs the current-libaom unsplit variable-transform root; skipped intra-block copy emits no transform-partition symbol. Effort-dependent model and transform pruning remain. Decoder-visible production cases inspect the emitted restrictions and frame state and decode the produced streams, including real effort-nine streams selecting sub-8x8 and 8x16 rectangular blocks. The complete non-HEVC HEIF/AV1 namespace passes 9,077 of 9,077 through one foreground net11 Release VSTest run. The last independently built `aomdec`, from the then-current `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` snapshot, accepts the previously generated effort-eight and effort-ten payloads as well as the existing palette and intra-block-copy payloads. The affected encoder, partition, and workspace surface passes 139 of 139 through one foreground net11 Release VSTest run. The net11 Release build and Roslynk compiler and analyzer passes report zero errors. - [~] Encoder rate accounting converts the entropy writer's live inverse cumulative distributions into current-libaom fixed-point symbol costs without allocating or duplicating probability state. Read-only luma-mode, directional-delta, filter-intra, chroma-mode, block-skip, transform-size, transform-block-skip, and complete transform-coefficient queries share the exact distributions mutated by the subsequent entropy write. Complete coefficient costing follows current libaom's optimized shape: it returns immediately for an empty transform, uses the EOB-specific base-range context, fuses magnitude, sign, base-range, and Golomb accounting into one reverse traversal, and combines repeated full base-range chunks instead of replaying each emitted symbol. Tile-lifetime level and context scratch is reused, the one-coefficient path neither clears nor initializes the forward-neighbor level map, and steady-state queries allocate nothing. Transform-size writing and costing share one subdivision-depth calculation, while shared closed symbol operations keep the writer and cost mappings for transform skip, transform type, and EOB syntax identical without forcing the estimator through the writer's slower two-pass coefficient traversal. The current-libaom fixed-point RD combiner preserves 64-bit distortion and rounds the weighted 1/512-bit rate at the required boundary. Its key-frame multiplier follows libaom's squared DC-quantizer formula and exact 10/12-bit normalization. Live final-block selection evaluates all 61 legal 8x8 luma candidates: the 13 zero-angle base modes in current-libaom order, followed by six nonzero adjustments for each directional mode. Joint chroma selection evaluates the equivalent 61 spatial candidates, combines U and V distortion plus coefficient rate, and charges one live chroma-mode and shared-angle symbol over the actual subsampled 4x4, 4x8, or 8x8 geometry. Chroma-from-luma subsamples the reconstructed luma block once into fixed-stride Q3 stack scratch, subtracts the rounded mean, evaluates all 33 signed alpha values independently for each plane with complete transform RD, and combines the cached plane results across all 1,088 valid joint pairs with one live sign cost and the conditional U/V magnitude costs. This is the allocation-free equivalent of current libaom's exhaustive 33-value path: it requires 66 evaluation transforms rather than transforming every joint pair, preserves DC-before-CfL-before-spatial tie order, and fixes the implicit chroma transform to DCT-DCT. Filter-intra follows ordinary luma candidates, searches all five predictors in syntax order, and evaluates every legal transform while reusing one prepared prediction and source residual per filter mode. Every candidate includes its live mode, angle, filter mode, alpha, and coefficient rate plus normalized pixel-domain distortion. Each prepared reference edge retains the common-corner prefix and twice the transform dimension required by directional prediction. A shared encoder/decoder availability calculation selects reconstructed top-right and bottom-left extensions according to tile, frame, superblock, and block reconstruction order; unavailable extensions repeat the nearest coded endpoint. Missing top or left edges retain current libaom's perpendicular-sample and bit-depth-midpoint rules. Directional prediction applies the AV1 three-degree adjustment step and reuses transform workspace for zone-three transposition before the transform overwrites it, keeping candidate evaluation allocation-free. The winning luma and chroma signed adjustments are retained in the packed final-block state consumed by the tile writer. The tile writer invokes these reusable workspace-backed selectors after mapping current neighbors and immediately before writing each block, so later decisions see reconstructed samples, coefficient contexts, and CDF updates from every preceding block. Block skip is read only after the callback has combined every coded plane. Luma and chroma candidate scratch is partitioned from the encoder's single aligned reusable block workspace; transform-size search uses that owner for four retained 4x4 transform states, local coefficient contexts, and the compact trial reconstruction needed to preserve the best result. No candidate path rents a buffer per block or per transform. Only a newly winning candidate is copied into retained frame storage. Production fixtures force every luma base predictor, both extreme adjustments in all three directional zones, available top-right and bottom-left extensions, high-bit-depth adjustment propagation, exact signed luma and chroma angle-rate terms, joint U/V decisions, packed chroma state, and 4:2:0, 4:2:2, and 4:4:4 transform geometry. The CfL fixtures derive target chroma from a pilot production encode's actual reconstructed luma through an independent scalar Q3 oracle and prove exact positive/negative alpha syntax plus zero-residual DCT-DCT reconstruction for all three subsampling geometries at 8, 10, and 12 bits. The stable fixed-DC traversal comparison uses neutral samples for which both the baseline and live search are contractually DC and skipped, instead of relying on textured content to happen to select the baseline mode. Luma palette selection now evaluates dominant-color and one-dimensional K-means candidates for every legal size, snaps near-cache colors with the reference threshold and tie order, removes duplicate snapped colors, extends boundary maps from active samples, and performs complete transform rate-distortion search. Ordinary DC and filter-intra candidates pay the palette-disabled symbol whenever screen-content syntax is enabled. The exact net11 Release rebuild reports 1,992 test-project warnings and zero errors, all 58 intra-superblock cases pass, all 8,935 AVIF cases pass, and all 230 HEIF cases pass. Remaining mode decision work includes transform-size coverage for filter-intra and palette, broader joint mode/transform refinement, and effort-dependent pruning. Non-empty intra blocks deliberately remain non-skipped, matching current libaom; later inter mode selection owns its distinct skip-transform RD decision. -- [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Partition, transform, and coefficient neighbor state retains only the above and left context regions used by current libaom; the unused third top-left region, its granularity state, and its unused sentinel are removed. One picture owner now packs segmentation plus every tile's partition, luma, chroma, and transform edges into one clean byte allocation with typed non-owning views; together with the separately typed packed mode-information owner, the complete picture state uses two allocator rents rather than seven. Exact aligned lengths, clean initialization, and balanced exactly-once returns are covered in Release. Multi-tile payload ownership and verified CDF update behavior remain. +- [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Partition, transform, and coefficient neighbor state retains only the above and left context regions used by current libaom; the unused third top-left region, its granularity state, and its unused sentinel are removed. One picture owner packs segmentation, every tile's partition, luma, chroma, and transform edges, CDEF state, preceding quantizer, and encoded payload bounds into one clean byte allocation with typed non-owning views; together with the separately typed packed mode-information owner, the complete picture state uses two allocator rents rather than seven. Each encoded tile has independent neighbor and probability state while sharing the bounded output owner. Earlier aligned-length and balanced-return coverage exists; runtime allocation verification of the current multi-tile layout remains pending. - [~] Encoder mode information now uses a frame-owned integer alias grid over a packed 8-byte value allocation, matching current libaom's `mi_grid_base` and `mi_alloc` relationship without a managed object or reference per 4x4 entry. The visible dimensions are aligned to eight luma samples, the grid stride and allocated row count are aligned to 32 mode-information units, and optional 8x8 allocation granularity reduces the value store in both dimensions exactly as current libaom does. One clean ImageSharp byte owner contains both independently typed regions, reducing libaom's two allocation lifetimes to one without a copy. At 4K, the 4x4 layout occupies about 6.0 MiB in total; the 8x8 layout occupies about 3.0 MiB. Exact geometry, clean allocation, typed lengths, aligned mapping, untouched row padding, and exactly-once return pass 4 of 4 direct net11 VSTest cases in Release. Every coded 4x4 cell covered by square, rectangular, or clipped edge blocks maps to its owning allocation entry before context-dependent symbols are written. Packed syntax, relative neighbor lookup, full block mapping, writer traversal, entropy, and OBU coverage pass 1,947 of 1,947 direct net11 VSTest cases in Release; complete mode decision still remains. -- [~] The final-block decision workspace uses one reusable 8.3 KiB ImageSharp allocator owner. It contains 1,024 explicitly packed 8-byte final-block entries and the 341 preorder partition bytes required by a complete 128x128-through-8x8 quadtree, replacing separate managed arrays. Palette colors now have their own current-block value and are copied only to the picture edges that later blocks can reference, so enabling palette mode does not add 50 bytes to every final-block entry. Construction and the explicit per-superblock reset initialize every syntax field, including the nonzero sentinel that disables filter-intra prediction; pooled quantizer, prediction, partition, and current-palette bytes cannot leak into the next decision pass. Complete mode decision still remains. +- [~] The superblock decision and palette-map workspace uses one reusable 40.3 KiB ImageSharp allocator owner. Its aligned 8.3 KiB decision region contains 1,024 explicitly packed 8-byte final-block entries and the 341 preorder partition bytes required by a complete 128x128-through-8x8 quadtree; its remaining 32 KiB contains the fixed 128x128 luma and chroma palette maps. This improves on libaom's separate compressor-state allocation lifetimes without changing their layouts or introducing a frame-time rent. Palette colors have their own current-block value and are copied only to the picture edges that later blocks can reference, so enabling palette mode does not add 50 bytes to every final-block entry. Construction and the explicit per-superblock reset initialize every syntax field, including the nonzero sentinel that disables filter-intra prediction; pooled quantizer, prediction, partition, and current-palette bytes cannot leak into the next decision pass. Roslynk reports zero compiler errors for the current one-owner refactor; runtime allocation verification remains pending. - [~] Finalized transform coefficients and packed EOB/type state now use raster-ordered, per-superblock plane segments matching current libaom's coefficient-pool geometry. One ImageSharp allocator owner replaces libaom's separate coefficient, EOB, and entropy-context allocations while preserving the full 1024 luma and 256-per-chroma 4x4 state capacity of a 128x128 4:2:0 superblock. The fixed 8x8 DC-intra traversal populates the owner's quantized coefficient and state slices while updating the caller-owned reconstruction plane directly, and a real tile-writer integration check proves that both sides consume identical luma and chroma areas. Complete mode decision still remains. - [~] Tile partition writing now follows current libaom's recursive `write_modes_sb` preorder traversal and `update_ext_partition_context` edge updates directly. Bottom-edge blocks use the horizontal-alike partition CDF and right-edge blocks use the vertical-alike CDF; byte-exact regressions cover both paths after the previous calls were found reversed. Lossless chroma-from-luma availability now uses the subsampled plane block size shared with the decoder instead of the lossy 32x32 limit, preserving the correct UV-mode alphabet for each segment. The obsolete SVT-derived global geometry catalog and its unimplemented lookup are removed; transform geometry is derived in libaom's bounded 64x64 residual order, fixed intra transform-size symbols use the reference depth and neighbor contexts, and each derived transform size is persisted to the frame-owned mode information before the entropy snapshot and coefficient traversal consume it. Frame-edge and segmentation syntax use mode-information units, and 128x128 CDEF units use libaom's 0-to-3 indexing and first-block strength ownership. The focused transform-state regression passes 3 of 3 direct net11 VSTest cases in Release. Writer, entropy, and OBU coverage passes 1,957 of 1,957 direct net11 VSTest cases in Release, with 20 of 20 focused encoder and decoder chroma-from-luma cases. Partition and mode analysis still need to populate these retained decisions; variable inter-transform syntax remains part of later inter-frame support. - [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions. -- [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform, defers both coefficient rents until the first nonzero transform block, and disposes all tile scratch independently from the detached encoded bytes. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. The single-tile production path finalizes in that existing allocation and transfers its owner plus the used byte length, removing the former second rent and full-tile copy; exact-length test callers retain the original overload. The ownership regression proves that writer disposal cannot return transferred storage and that the caller returns the original allocation exactly once. The internal frame operation now passes that payload directly to the OBU writer; AVIF container integration and public activation remain. +- [~] The coefficient symbol encoder allocates its bounded level and raster-context workspaces with the encoder state and reuses them for every transform and sequence sample, matching current libaom's fixed-geometry compressor lifetime instead of renting scratch during the first coded frame. Its range coder matches current libaom's 64-bit coding window, bulk big-endian byte flush, and backward carry propagation while using one byte of allocator scratch per estimated output byte instead of the former 16-bit pre-carry storage. Production tiles finalize consecutively inside one encoder-owned bounded output allocation; the OBU writer consumes non-owning tile slices synchronously before the encoder is reset or disposed, so no payload owner transfer, second rent, or full-tile copy occurs. Exact-length test callers retain the copying overload. In-memory allocation coverage proves reset-to-offset and reset-to-zero reuse the same output owner; runtime allocation verification of the complete production multi-tile and sequence paths remains pending. - [~] The planar conversion, DC intra prediction, residual construction, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Residual construction matches current libaom's exact source-minus-prediction arithmetic for 8-bit and high-bit-depth planes, preserves independent row strides and unaligned starts, and writes directly into caller-owned signed-short storage without allocation. Candidate distortion reuses that residual workspace and widens signed 12-bit lanes before vector squaring, accumulating exact full-block SSE in 64-bit scalar storage. The composed block path delegates arithmetic to those closed operators and adds no allocation. Apply the same rule to every later hot-path family. - [~] Residual tests verify misaligned planes, independent source, prediction, and destination strides, SIMD remainders, untouched padding, 8-bit, 10-bit, and 12-bit precision, every operator width independently of host acceleration, the scalar fallback, and zero per-transform allocations. - [~] The unused coefficient-shape transform facade and its unimplemented N2, N4, and DC-only branches are removed. Finalized block encoding now follows the complete-transform path that current libaom uses before fast quantization; later rate-distortion search may add proven coefficient optimization without exposing inactive runtime throws. @@ -855,9 +867,9 @@ Encoder verification contract: - [~] Palette entropy coding now mirrors current libaom's adaptive luma-mode, chroma-mode, palette-size, and spatial color-index distributions, together with its truncated-binary uniform code used by palette colors. The complete mutable palette probability graph is created once on first palette search or write, so the current palette-disabled frame path retains zero palette allocations. Three focused regressions cover every legal 2-through-8 color alphabet and every defined mode, size, and color-index context; all 1,928 entropy cases and all 8,978 HEIF/AV1 cases pass direct net11 Release VSTest. The exact Release rebuild remains at 1,005 warnings and zero errors. This checkpoint adds the exact entropy foundation only: palette candidate generation, retained color and index storage, mode decision, map tokenization, and production syntax remain incomplete, and no generated payload changed. - [~] Luma and chroma palette-color coding now matches current libaom's neighbor-cache flags, sorted delta representation, wrapped V-plane deltas, strict delta-versus-raw V selection, and fixed-point color-rate model at 8, 10, and 12 bits. Encoder costing and emission use only fixed stack spans, including explicitly initialized cache-membership state, and steady-state color costing allocates zero managed bytes. The decoder consumes the same bounded color-syntax primitive after the tile reader derives its neighbor cache, removing duplicated color parsing without changing retained palette ownership. Nine focused syntax, exact palette decode, constrained-allocation, truncation, presentation, and allocation cases pass; all 1,933 entropy cases and all 8,983 HEIF/AV1 cases pass direct net11 Release VSTest. The exact Release rebuild remains at 1,005 warnings and zero errors. Retained encoder palette colors, neighbor caches, color-index maps, candidate generation, and production palette selection remain incomplete, and the compact 8-byte frame mode entries were not enlarged. - [~] Palette color-index map coding now shares the exact current-libaom neighbor weights, stable color ordering, five context classes, first-index uniform code, and diagonal wavefront between encoder costing, encoder writing, and decoder parsing. The decoder's stack-allocated context scores are explicitly cleared before accumulation, removing an invalid dependency on uninitialized stack contents. Costing and writing use a closed generic operation while the shared driver owns traversal and context derivation, so the semantic operations remain independent of map layout and tail handling. The path adds no retained state or per-call managed allocation. Its allocation regression now runs one complete unmeasured hot-path window before measuring an independent 1,000-call steady-state window, so tiered-runtime transitions cannot make the full parallel suite report a one-time allocation as a recurring operation cost. Twelve focused map, exact palette decode, padding, trailing-bit, and allocation cases pass; all 1,941 entropy cases and all 8,991 HEIF/AV1 cases pass direct net11 Release VSTest. The exact Release rebuild remains at 1,005 warnings and zero errors. Production payloads remain unchanged because palette selection is still disabled; retained colors, neighbor caches, index-map storage, candidate generation, and production palette mode decision remain incomplete. -- [~] Retained encoder palette state and production palette writing now mirror current libaom's 50-byte palette-mode contents, separate luma and shared-chroma sizes, three eight-color planes, above-and-left sorted cache, 64-sample above-cache boundary, mode contexts, palette colors, color-index maps, and syntax order. The current block keeps one inline value in the reusable superblock workspace; only the 4x4-granularity top and left picture edges retain copies for later blocks. For a 3840x2160 tile these edges occupy about 73.4 KiB instead of about 6.2 MiB for a 50-byte palette value attached to every 8x8 mode allocation. Luma and chroma index maps share one lazily allocated 32 KiB owner containing two 128x128 maps, so the palette-disabled production path retains no map owner. The compact final-block workspace falls from about 10.3 KiB to about 8.3 KiB. The writer caps map traversal to the coded plane count, writes maps before transform syntax, and publishes palette edges only after the current block has consumed preceding contexts. Eight focused size, alignment, ownership, cache-boundary, round-trip, map-consumption, and edge-publication cases pass; all 114 palette cases, all 1,942 entropy cases, and all 8,996 HEIF/AV1 cases pass direct foreground net11 Release VSTest. The exact net11 Release rebuild reports 1,050 solution warnings and zero errors; Roslynk reports zero compiler errors and no diagnostics in the touched files. The current-main reference is `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727`. Production payloads remain unchanged because palette candidate generation is still disabled; that live rate-distortion search is the next checkpoint. +- [~] Retained encoder palette state and production palette writing now mirror current libaom's 50-byte palette-mode contents, separate luma and shared-chroma sizes, three eight-color planes, above-and-left sorted cache, 64-sample above-cache boundary, mode contexts, palette colors, color-index maps, and syntax order. The current block keeps one inline value in the reusable superblock workspace; only the 4x4-granularity top and left picture edges retain copies for later blocks. For a 3840x2160 tile these edges occupy about 73.4 KiB instead of about 6.2 MiB for a 50-byte palette value attached to every 8x8 mode allocation. Luma and chroma index maps occupy a fixed 32 KiB region of the single 40.3 KiB superblock-workspace owner. That owner is allocated with encoder state, matching libaom's compressor-state lifetime while removing libaom's separate palette allocation and cleanup path. The compact final-block decision region remains about 8.3 KiB. The writer caps map traversal to the coded plane count, writes maps before transform syntax, and publishes palette edges only after the current block has consumed preceding contexts. The previous eight focused size, alignment, ownership, cache-boundary, round-trip, map-consumption, and edge-publication cases passed with all 114 palette cases, all 1,942 entropy cases, and all 8,996 HEIF/AV1 cases through direct foreground net11 Release VSTest. The current one-owner refactor has zero Roslynk compiler errors; runtime verification remains pending. The current source reference is official libaom main at `d565eec60f084421fa34fc0534b760c6452b6a6c`. - [~] Luma palette clustering now follows current libaom's one-dimensional search primitive exactly: equal-interval midpoint initialization, first-color tie order, rounded centroid means, deterministic empty-cluster replacement, the 50-iteration limit, and retention of the preceding state when distortion increases. Nearest-color assignment improves on libaom's AVX2 implementation by dispatching Vector512, Vector256, Vector128, then scalar through ImageSharp's shared vector-count helpers. The primitive uses only bounded stack scratch and introduces no allocator rent, managed array, or per-row copy. Three independent tests cover exact centroid convergence, initialization order, 12-bit nearest-color distortion, destination bounds, and every hardware-intrinsic tier. The complete AVIF set passes 8,930 of 8,930 cases and the HEIF set passes 230 of 230 cases through direct foreground net11 Release VSTest. The exact net11 Release rebuild reports 1,050 solution warnings and zero errors, and Roslynk reports zero compiler errors. Candidate enumeration, palette-cache snapping, transform RD selection, and production activation remain in the open luma-palette checkpoint. -- [~] Live luma palette selection now follows current libaom's dominant-color and one-dimensional K-means candidate families, cache-bias threshold, sorted duplicate removal, active-edge map extension, and strict winner tie order. It improves on speed-configured libaom by evaluating both candidate families at every legal 2-through-8 size without early pruning, then exhaustively evaluates every legal transform using the existing SIMD prediction, residual, transform, quantization, and reconstruction operators. Candidate storage remains bounded stack memory; the reusable 32 KiB map owner is allocated only when an eligible block enters palette search. A production tile test proves that full 8x8 and clipped 5x3 blocks at 8 and 12 bits select exact colors and indices, extend the visible edges through coded padding, reconstruct every sample without coefficients, and emit a nonempty tile. The complete 57-case intra-superblock set, 8,931-case AVIF set, and 230-case HEIF set pass direct foreground net11 Release VSTest. The exact Release test-project build reports 1,992 baseline warnings and zero errors; Roslynk reports zero compiler errors and no touched-file analyzer warnings. Production frame activation remains gated until chroma palette mode and its rate accounting are complete. +- [~] Live luma palette selection now follows current libaom's dominant-color and one-dimensional K-means candidate families, cache-bias threshold, sorted duplicate removal, active-edge map extension, and strict winner tie order. It improves on speed-configured libaom by evaluating both candidate families at every legal 2-through-8 size without early pruning, then exhaustively evaluates every legal transform using the existing SIMD prediction, residual, transform, quantization, and reconstruction operators. Candidate storage remains bounded stack memory; the reusable maps come from the fixed encoder-lifetime superblock workspace, so palette search cannot introduce a first-use allocation. A production tile test proves that full 8x8 and clipped 5x3 blocks at 8 and 12 bits select exact colors and indices, extend the visible edges through coded padding, reconstruct every sample without coefficients, and emit a nonempty tile. The complete 57-case intra-superblock set, 8,931-case AVIF set, and 230-case HEIF set pass direct foreground net11 Release VSTest. The exact Release test-project build reports 1,992 baseline warnings and zero errors; Roslynk reports zero compiler errors and no touched-file analyzer warnings. Production frame activation remains gated until chroma palette mode and its rate accounting are complete. - [~] Paired chroma palette clustering now preserves current libaom's squared two-component distance, first-centroid tie order, independently rounded U/V means, paired deterministic empty-cluster replacement, preceding-state retention on increased distortion, and 50-iteration limit. Keeping the source planes separate avoids interleave/deinterleave copies and improves on libaom's AVX2 ceiling with Vector512, Vector256, Vector128, then scalar dispatch through ImageSharp's shared vector-count helpers. Three independent tests cover exact paired convergence, midpoint initialization, 12-bit distance and index parity, untouched destination bounds, and every intrinsic tier. The exact Release test-project build reports 1,992 baseline warnings and zero errors; the focused three-case set, complete 8,934-case AVIF set, and complete 230-case HEIF set pass direct foreground net11 Release VSTest. Roslynk reports zero compiler errors and no touched-file analyzer warnings. Candidate integration and production activation remain in the open chroma-palette checkpoint. - [~] Live paired chroma palette selection now follows current libaom's complete 2-through-8 color-size search, U-plane neighbor-cache snapping, stable U-ordered color pairs, shared U/V index map, implicit DCT-DCT transform, and strict rate-distortion winner replacement. It improves on speed-configured libaom by applying no early header-cost pruning, keeps planar U/V source data separate, and reuses the SIMD-first prediction, residual, transform, quantization, and reconstruction operators without allocator-backed candidate storage. The production tile regression proves both palette-mode probability branches, exact paired colors and indices, coefficient-free reconstruction, and nonempty syntax. The complete 58-case intra-superblock set, 8,935-case AVIF set, and 230-case HEIF set pass direct foreground net11 Release VSTest. The exact Release test-project build reports 1,992 baseline warnings and zero errors; Roslynk reports zero compiler errors and no touched-file analyzer warnings. Production frame activation remains the next checkpoint. - [x] Production screen-content activation now matches current libaom's default good-quality detector: it scans only complete 16x16 luma blocks, normalizes palette samples to eight bits, admits 2-through-4-color blocks, and uses the reference's strict greater-than-ten-percent frame-area threshold. The same pass accumulates centered sums and squared sums at native precision, applies libaom's exact 10-bit and 12-bit variance rounding, and enables intra-block copy only when positive rounded per-pixel variance exceeds its strict one-twelfth frame-area threshold. A 256-bit stack bitset and fifth-color early exit replace libaom's larger per-block histogram without a second source scan or allocation. The adaptive sequence flag remains enabled and both frame flags are fixed before picture-state allocation. Focused regressions prove strict palette-threshold equality, high-bit-depth normalization, the exact variance rounding boundary, five-color rejection, emitted frame-header activation, actual production IBC selection, and production decode. The exact Release test-project build reports 1,992 baseline warnings and zero errors; all 9,242 non-HEVC HEIF/AV1 cases pass direct foreground net11 Release VSTest. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts all 31 payloads regenerated by the current test tree, including an actual IBC-coded 328x16 stream with decoded MD5 `677435e5af39c930af1178f91c34af6a`. Roslynk reports zero compiler errors and no touched-file analyzer warnings. @@ -886,13 +898,14 @@ Encoder verification contract: - [~] The production path writes color and alpha payloads sequentially through allocator-backed chunked storage, supports non-seekable and prefixed destinations, and does not materialize a complete file or payload copy. Uniform encoder-side `pixi` depth is written directly without allocating per-item channel-depth arrays; decoder-side non-uniform channel depths remain supported. The Release test project builds with zero errors, all 39 encoder cases pass, the complete non-HEVC HEIF namespace passes 9,277 of 9,277, and current official libaom accepts all 47 generated payloads. - [x] Still-image AVIF metadata preservation now writes an unrestricted ICC `colr/prof` property before the independent `colr/nclx` property, Exif and XMP as separate `mdat` items, and one `cdsc` relationship from each metadata item to the primary color item. Exif stores the exact big-endian TIFF-header offset required by the HEIF item syntax; XMP uses the `mime` item type and `application/rdf+xml` content type. Existing ICC and XMP storage is read synchronously and copied once into final encoder storage rather than cloned into an intermediate array. `SkipMetadata` suppresses all three profile types while retaining the CICP values required to describe the encoded planes. The same option now reaches legacy JPEG payloads, whose encoder no longer writes application profiles or comments when metadata is disabled. - [x] Exact container tests verify every emitted item declaration, name, MIME content type, `cdsc` relationship, Exif offset and payload, XMP payload, ICC/CICP property order, compact association byte, propertyless metadata exclusion, decoded profile value, and both `SkipMetadata` branches. The final HEIF encoder set passes 44 of 44 and the complete JPEG encoder set passes 257 of 257 through direct foreground net11 Release VSTest. The complete non-HEVC HEIF namespace passes 9,282 of 9,282 with no failure, crash, or detached test host, and current official libaom accepts all 47 current generated AV1 payloads. -- [x] A code-wide production HEIF/AV1 stack-storage audit, excluding HEVC, removed every block-sized, variable-length, or repeatedly nested scratch buffer. Spatial luma and chroma, filter-intra, chroma-from-luma, luma and chroma palette selection, and K-means iteration now use typed views over 642 signed-integer elements, about 2.51 KiB, at the start of the existing 3.125 KiB IBC region. Those searches are sequential for one block, so the block-workspace owner does not grow and no rent, copy, or additional lifetime is introduced. CDEF directions, variances, and its 64-entry block list now append 1 KiB to the existing bounded operation owner instead of occupying hidden inline or explicit stack arrays. No remaining `stackalloc` depends on block dimensions, sample count, or runtime length; the largest remaining individual span is 128 bytes, and the remaining sites are fixed syntax, SIMD-lane, filter-tap, plane-metadata, or small candidate storage. The exact-owner test now proves the mode, palette, and IBC views share one allocation. Roslynk reports zero compiler errors and no diagnostics in the changed files, the Release test-project build completes with the established 1,992 warnings and zero errors, 81 of 81 focused cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,282 of 9,282 through one foreground net11 VSTest run. -- [~] Bounded public image-sequence output now emits an `avis` movie with version-one movie, track, and media headers; AV1 visual sample entries; exact run-length-compressed timing; per-sample sizes; 64-bit chunk offsets; and an explicit sync-sample table. Color and optional auxiliary alpha use independently configured AV1 tracks linked by `auxl`, while every current sample is an independently decodable all-intra picture. Frame payloads are written once into contiguous allocator-backed chunks per track. One compact managed table retains only offset, length, and duration for both tracks, and the bounded `moov` owner is patched once after its final size is known, so prefixed and non-seekable destinations require neither seeking nor a file-sized copy. The media timescale uses the exact representable least common multiple of frame-delay denominators and a documented microsecond fallback; zero delays become the smallest legal positive duration. Public lossless color-and-alpha round trips preserve all frames, distinct 24, 25, and 30 fps delays, and finite or infinite repetition, while a separate case proves prefixed non-seekable output. The net11 Release build completes with the established 1,005 warnings and zero errors, all 48 HEIF encoder cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,317 of 9,317 through foreground VSTest. Current official libaom `main` at `d565eec60f084421fa34fc0534b760c6452b6a6c` accepts all 66 raw AV1 payloads regenerated by that suite. Inter-picture reference search, sequence Exif/XMP, grids, and orientation remain open. +- [x] A code-wide production HEIF/AV1 stack-storage audit, excluding HEVC, removed every block-sized, variable-length, or repeatedly nested scratch buffer. Spatial luma and chroma, filter-intra, chroma-from-luma, luma and chroma palette selection, and K-means iteration now use typed views over 642 signed-integer elements, about 2.51 KiB, at the start of the shared inter-prediction region. Those searches are sequential for one block, so the block-workspace owner does not grow and no rent, copy, or additional lifetime is introduced. CDEF directions, variances, and its 64-entry block list now append 1 KiB to the existing bounded operation owner instead of occupying hidden inline or explicit stack arrays. No remaining `stackalloc` depends on block dimensions, sample count, or runtime length; the largest remaining individual span is 128 bytes, and the remaining sites are fixed syntax, SIMD-lane, filter-tap, plane-metadata, or small candidate storage. The exact-owner test now proves the mode, palette, and reference-prediction views share one allocation. Roslynk reports zero compiler errors and no diagnostics in the changed files, the Release test-project build completes with the established 1,992 warnings and zero errors, 81 of 81 focused cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,282 of 9,282 through one foreground net11 VSTest run. +- [~] Bounded public image-sequence output now emits an `avis` movie with version-one movie, track, and media headers; AV1 visual sample entries; exact run-length-compressed timing; per-sample sizes; 64-bit chunk offsets; and an explicit sync-sample table. The file type includes the required `miaf` compatibility brand, and every sequence now has the MIAF primary image item emitted by current libavif: normal sequences share the first sync-sample extent without another encode or copy, while separate-root sequences retain the still root as the primary image and begin timed samples at frame index one. The decoder allocates one final `Image`: the root is either the first timed sample or the separately decoded primary item, and each visible timed sample is decoded directly into a frame owned by that image. Exact quarter-turn presentation uses one frame-sized reusable pre-rotation buffer rather than a second image or a separately built frame collection. Color and optional auxiliary alpha use independently configured AV1 tracks linked by `auxl`. Lossless samples remain independently decodable key pictures and repeat the sequence header required for random access. Lossy continuation samples use LAST_FRAME inter prediction through the existing SIMD translational predictor; one track-scoped encoder session reuses its source allocation, packed-to-planar row storage and color converter, frame-sized coefficient storage, fixed-geometry picture and frame-header syntax state, tile/superblock/entropy cursors, block arithmetic workspace, complete probability graph, bounded tile-output owner, and OBU-header owner, and swaps two complete reconstruction buffers so the preceding decoded frame becomes the next reference without a plane copy. Sequence samples signal `still_picture=0` and use the complete non-reduced sequence and frame-header prefixes required for a multi-frame coded sequence. Frame payloads are written once into contiguous allocator-backed chunks per track. One compact managed table retains only offset, length, and duration for both tracks, and the bounded `moov` owner is patched once after its final size is known, so prefixed and non-seekable destinations require neither seeking nor a file-sized copy. The media timescale uses the exact representable least common multiple of animated frame-delay denominators and a documented microsecond fallback; zero delays become the smallest legal positive duration. Public lossless color-and-alpha round trips preserve all frames, distinct 24, 25, and 30 fps delays, finite or infinite repetition, ICC, Exif, and XMP metadata, while a separate case proves prefixed non-seekable output. Per-tile CDEF preset, preceding-quantizer state, and payload bounds now occupy one aligned region in the reusable allocator-owned picture buffer rather than separate managed arrays for every frame. The last verified net11 Release checkpoint completed with the established 1,005 warnings and zero errors, all 48 HEIF encoder cases passed, and the complete non-HEVC HEIF/AV1 namespace passed 9,317 of 9,317 through foreground VSTest. Current official libaom `main` at `d565eec60f084421fa34fc0534b760c6452b6a6c` accepted all 66 raw AV1 payloads regenerated by that suite. Verification of the current primary-item, separate-root, non-reduced sequence-header, retained-reference continuation, grid implementation, block-local motion search, conversion-row, probability, picture, frame-header, tile-cursor, tile-output, and OBU-header reuse, and multi-tile output is pending. Additional reference roles and compound prediction remain open. +- [~] Oversized still-image encoding now writes a derived AVIF grid when either source dimension exceeds the AV1 frame-header limit. Cells are encoded row-major from source rectangles without cropping to temporary images. Every coded cell uses the same at-most-65,536-sample extent for current-reader interoperability, with edge replication supplying the AVIF minimum 64-sample dimension and any final-row or final-column crop. Color and optional alpha grids use hidden AV1 items, ordered `dimg` references, one shared property set per plane, and independent descriptor payloads. The item-property length calculation counts every reused `ipma` association while retaining one `ipco` property definition. Explicit subsampled output rejects grid dimensions that MIAF cannot represent; an unspecified sampling choice promotes that exceptional odd-dimension grid to 4:4:4. Exact descriptor, hidden-flag, reference-order, property-reuse, cell-padding, and public round-trip coverage is present, and Roslynk reports zero compiler errors. Runtime verification remains pending. - [~] Write the correct AVIF file type, item information, locations, references, properties, AV1 configuration, dimensions, color, alpha, metadata, and media data. - [~] Support single images, alpha auxiliary images, grids, multiple extents, and bounded image sequences in the final public scope. - [x] Preserve ICC, Exif, and XMP according to encoder options. - [~] Write CICP, range, chroma position, bit depth, and subsampling values that match the encoded planes. -- [ ] Apply orientation and clean-aperture behavior consistently with ImageSharp encoder conventions. +- [x] Encode the current ImageSharp pixel lattice without inventing HEIF clean-aperture, rotation, or mirror properties. The HEIF decoder materializes those container transforms before returning an image, while ImageSharp encoders consistently preserve explicit Exif metadata without implicitly changing the pixels. - [~] Stream output through allocator-backed chunked storage without file-sized copies or ToArray materialization. Encoder exit gate: diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs index 51ffdb432d..782986cfeb 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs @@ -173,6 +173,84 @@ internal ref struct Av1BitStreamWriter } } + /// + /// Writes a finite subexponential value recentered around a signed reference value. + /// + /// The signed value to write. + /// One greater than the maximum absolute value in the signed domain. + /// The bit width of the first subexponential group. + /// The signed reference value around which smaller codewords are concentrated. + public void WriteSignedReferenceSubexponential(int value, int valueMagnitude, int groupBitCount, int reference) + { + int shiftedReference = reference + valueMagnitude - 1; + int shiftedValue = value + valueMagnitude - 1; + int scaledValueCount = (valueMagnitude << 1) - 1; + int recenteredValue = RecenterFiniteNonNegative(scaledValueCount, shiftedReference, shiftedValue); + + this.WriteSubexponential(recenteredValue, scaledValueCount, groupBitCount); + } + + /// + /// Writes one value using a finite sequence of exponentially growing code groups. + /// + private void WriteSubexponential(int value, int valueCount, int groupBitCount) + { + int groupIndex = 0; + int groupStart = 0; + while (true) + { + // The first two groups retain the initial width. Later groups grow one bit at a time until the + // finite tail is small enough for the exact non-symmetric alphabet. + int bitCount = groupIndex == 0 ? groupBitCount : groupBitCount + groupIndex - 1; + int groupSize = 1 << bitCount; + if (valueCount <= groupStart + (3 * groupSize)) + { + this.WriteNonSymmetric((uint)(value - groupStart), (uint)(valueCount - groupStart)); + + return; + } + + bool useLaterGroup = value >= groupStart + groupSize; + this.WriteBoolean(useLaterGroup); + if (!useLaterGroup) + { + this.WriteLiteral((uint)(value - groupStart), bitCount); + return; + } + + groupIndex++; + groupStart += groupSize; + } + } + + /// + /// Maps an unsigned value to increasing distance from a reference inside a finite domain. + /// + private static int RecenterFiniteNonNegative(int valueCount, int reference, int value) + { + if ((reference << 1) <= valueCount) + { + return RecenterNonNegative(reference, value); + } + + return RecenterNonNegative(valueCount - 1 - reference, valueCount - 1 - value); + } + + /// + /// Maps an unsigned value to alternating positions around a nonnegative reference. + /// + private static int RecenterNonNegative(int reference, int value) + { + if (value > (reference << 1)) + { + return value; + } + + return value >= reference + ? (value - reference) << 1 + : ((reference - value) << 1) - 1; + } + /// /// Appends one bit to the partially assembled output byte. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs index 15609fc2db..b168122c3c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs @@ -21,6 +21,41 @@ internal static class Av1Constants /// public const int LevelBits = 5; + /// + /// The number of bits used for the zero-based operating-point count. + /// + public const int OperatingPointCountBits = 5; + + /// + /// The number of bits used for an operating-point layer-selection mask. + /// + public const int OperatingPointIdcBits = 12; + + /// + /// The number of bits used for a frame type. + /// + public const int FrameTypeBits = 2; + + /// + /// The first sequence-level index that carries an explicit tier bit. + /// + public const int SequenceTierMinimumLevelIndex = 8; + + /// + /// The number of bits used to select a frame from the eight-slot reference map. + /// + public const int ReferenceFrameIndexBits = 3; + + /// + /// The sequence-header value that lets each frame choose whether to use screen-content tools. + /// + public const int SelectScreenContentTools = 2; + + /// + /// The sequence-header value that lets each applicable frame choose whether to require integer motion vectors. + /// + public const int SelectIntegerMotionVector = 2; + /// /// The maximum number of operating points declared by one AV1 sequence header. /// @@ -46,6 +81,11 @@ internal static class Av1Constants /// public const int ReferencesPerFrame = 7; + /// + /// The largest frame width or height representable by the 16-bit AV1 dimension syntax. + /// + public const int MaxFrameDimension = 1 << 16; + /// /// The maximum area of a tile in units of luma samples. /// @@ -66,6 +106,11 @@ internal static class Av1Constants /// public const int MaxTileRowCount = 64; + /// + /// The number of 64x64 CDEF filter units in a 128x128 superblock. + /// + public const int CdefUnitsPerSuperblock = 4; + /// /// The number of frames that can be stored for future reference. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index b755ee8a05..152df4509f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -261,6 +261,52 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable return this.ConvertToFrame(outputFrame.FrameBuffer, outputFrame.FrameHeader, effectiveColorProfile); } + /// + /// Decodes the next visible sample in a bounded AV1 image sequence directly into a caller-owned frame. + /// + /// The destination pixel type. + /// The complete AV1 sample payload. + /// The container color description. + /// The AV1 sample-entry configuration. + /// The coded dimensions declared by the visual sample entry. + /// The clean-aperture region mapped to the complete destination frame. + /// The caller-owned packed-pixel frame receiving the presented sample. + public void DecodeSequenceFrame( + Span buffer, + CicpProfile? containerColorProfile, + Av1CodecConfiguration? codecConfiguration, + Size expectedCodedSize, + Rectangle sourceRectangle, + ImageFrame destination) + where TPixel : unmanaged, IPixel + { + CicpProfile effectiveColorProfile = this.DecodePayload( + buffer, + containerColorProfile, + codecConfiguration, + null, + requireShownFrame: true); + + Av1ReferenceFrame outputFrame = this.referenceFrames.ResolveOutput(); + Size codedSize = new( + outputFrame.FrameHeader.FrameSize.SuperResolutionUpscaledWidth, + outputFrame.FrameHeader.FrameSize.FrameHeight); + + if (codedSize != expectedCodedSize) + { + throw new InvalidImageContentException( + "The decoded image-sequence sample dimensions do not match its visual sample entry."); + } + + Av1YuvConverter.ConvertRegionToRgb( + this.configuration, + outputFrame.FrameBuffer, + sourceRectangle, + destination); + + destination.Metadata.CicpProfile = effectiveColorProfile.DeepClone(); + } + /// /// Decodes one non-presented AV1 image-sequence sample while retaining its reference state. /// @@ -286,6 +332,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable /// The container color description. /// The AV1 sample-entry configuration. /// The required coded dimensions. + /// The clean-aperture luma region mapped to the destination. /// The packed color frame receiving alpha values. /// The complete presented size of the auxiliary image. /// The destination region receiving the alpha image. @@ -295,6 +342,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable CicpProfile? containerColorProfile, Av1CodecConfiguration? codecConfiguration, Size expectedCodedSize, + Rectangle sourceRectangle, ImageFrame destination, Size outputSize, Rectangle destinationRectangle, @@ -312,6 +360,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable this.ComposeAlpha( outputFrame.FrameBuffer, expectedCodedSize, + sourceRectangle, destination, outputSize, destinationRectangle, @@ -424,6 +473,35 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable Rectangle destinationRectangle, bool premultiplied) where TPixel : unmanaged, IPixel + => this.ComposeAlpha( + frameBuffer, + expectedCodedSize, + new Rectangle(0, 0, frameBuffer.Width, frameBuffer.Height), + destination, + outputSize, + destinationRectangle, + premultiplied); + + /// + /// Composes one decoded monochrome region into a packed color frame. + /// + /// The destination color pixel type. + /// The decoded monochrome planes. + /// The required coded dimensions. + /// The luma region mapped to the destination rectangle. + /// The packed color frame receiving alpha values. + /// The complete presented size of the auxiliary image. + /// The destination region receiving the alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + private void ComposeAlpha( + Av1FrameBuffer frameBuffer, + Size expectedCodedSize, + Rectangle sourceRectangle, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel { if (expectedCodedSize != default && (frameBuffer.Width != expectedCodedSize.Width || frameBuffer.Height != expectedCodedSize.Height)) { @@ -440,6 +518,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable Av1YuvConverter.ComposeAlpha( this.configuration, frameBuffer, + sourceRectangle, destination, outputSize, destinationRectangle, diff --git a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs index 18040bb514..5fe0b0cfc4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs @@ -92,6 +92,48 @@ internal static class Av1YuvConverter mode); } + /// + /// Converts a rectangular region of reconstructed component planes directly to packed pixels. + /// + /// The destination pixel type. + /// The configuration used for allocation and pixel conversion. + /// The reconstructed AV1 frame. + /// The luma-sample region mapped to the complete destination frame. + /// The destination image frame. + public static void ConvertRegionToRgb( + Configuration configuration, + Av1FrameBuffer frameBuffer, + Rectangle sourceRectangle, + ImageFrame image) + where TPixel : unmanaged, IPixel + { + HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out HeifColorConversionMode mode); + if (frameBuffer.BitDepth == Av1BitDepth.EightBit) + { + Av1PlanarSampleBuffer buffer = new(frameBuffer); + HeifPlanarColorConverter.ConvertToRgb, byte, HeifByteSampleConverter>( + configuration, + buffer, + image, + in parameters, + mode, + sourceRectangle.X, + sourceRectangle.Y); + + return; + } + + Av1PlanarSampleBuffer highBitDepthBuffer = new(frameBuffer); + HeifPlanarColorConverter.ConvertToRgb>( + configuration, + highBitDepthBuffer, + image, + in parameters, + mode, + sourceRectangle.X, + sourceRectangle.Y); + } + /// /// Composes the reconstructed luma plane into a packed color frame as auxiliary alpha. /// @@ -110,9 +152,37 @@ internal static class Av1YuvConverter Rectangle destinationRectangle, bool premultiplied) where TPixel : unmanaged, IPixel + => ComposeAlpha( + configuration, + frameBuffer, + new Rectangle(0, 0, frameBuffer.Width, frameBuffer.Height), + destination, + outputSize, + destinationRectangle, + premultiplied); + + /// + /// Composes a rectangular reconstructed luma region into a packed color frame as auxiliary alpha. + /// + /// The destination color pixel type. + /// The configuration used for allocation and pixel conversion. + /// The reconstructed AV1 frame containing the alpha luma plane. + /// The luma-sample region mapped to the destination rectangle. + /// The packed color frame receiving alpha values. + /// The complete presented size of the auxiliary image or grid tile. + /// The destination region receiving the presented alpha image. + /// Whether stored color samples must be converted to unassociated alpha. + public static void ComposeAlpha( + Configuration configuration, + Av1FrameBuffer frameBuffer, + Rectangle sourceRectangle, + ImageFrame destination, + Size outputSize, + Rectangle destinationRectangle, + bool premultiplied) + where TPixel : unmanaged, IPixel { HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out _); - Rectangle sourceRectangle = new(0, 0, frameBuffer.Width, frameBuffer.Height); if (frameBuffer.BitDepth == Av1BitDepth.EightBit) { Av1PlanarSampleBuffer buffer = new(frameBuffer); diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs index a8c3189607..b7e5cb17d8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs @@ -512,8 +512,8 @@ internal sealed class Av1FrameEntropyContext { int qContext = GetQContext(qIndex); - // The prototypes are never exposed to a range reader. Copying their state lets a decoder session reuse the - // same three mutable object graphs even when successive frames select different coefficient-model bands. + // The prototypes are never exposed to a range reader or writer. Copying their state lets each codec session + // reuse its mutable object graphs even when successive frames select different coefficient-model bands. this.CopyFrom(DefaultPrototypes[qContext]); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs index af51136b4b..5a28d06735 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs @@ -41,10 +41,7 @@ internal sealed class Av1MotionVectorContext /// The zero-based symbol. /// The live symbol distribution. /// The symbol cost in 1/512-bit units, or zero when writing. - public static abstract int ProcessSymbol( - Av1SymbolWriter writer, - int symbol, - Av1Distribution distribution); + public static abstract int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution); } /// @@ -103,13 +100,14 @@ internal sealed class Av1MotionVectorContext } /// - /// Writes an integer displacement vector relative to a spatially derived reference. + /// Writes a motion vector relative to a spatially derived reference. /// /// The tile range encoder. /// The displacement vector to encode. /// The spatially derived reference vector. - public void Write(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference) - => _ = this.Process(writer, value, reference); + /// The fractional precision selected by the frame header. + public void Write(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision) + => _ = this.Process(writer, value, reference, precision); /// /// Measures a motion-vector delta against the live distributions without changing them. @@ -117,17 +115,15 @@ internal sealed class Av1MotionVectorContext /// The tile range encoder associated with the live context. /// The motion vector to measure. /// The spatially derived reference vector. + /// The fractional precision selected by the frame header. /// The syntax cost in 1/512-bit units. - public int GetCost(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference) - => this.Process(writer, value, reference); + public int GetCost(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision) + => this.Process(writer, value, reference, precision); /// /// Processes one complete motion-vector delta through a closed symbol operation. /// - private int Process( - Av1SymbolWriter writer, - Av1MotionVector value, - Av1MotionVector reference) + private int Process(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision) where TOperation : struct, IMotionVectorSymbolOperation { int row = value.Row - reference.Row; @@ -140,12 +136,12 @@ internal sealed class Av1MotionVectorContext int rate = TOperation.ProcessSymbol(writer, jointType, this.Joint); if (row != 0) { - rate += this.Vertical.Process(writer, row); + rate += this.Vertical.Process(writer, row, precision); } if (column != 0) { - rate += this.Horizontal.Process(writer, column); + rate += this.Horizontal.Process(writer, column, precision); } return rate; @@ -157,10 +153,7 @@ internal sealed class Av1MotionVectorContext private readonly struct MotionVectorWriteOperation : IMotionVectorSymbolOperation { /// - public static int ProcessSymbol( - Av1SymbolWriter writer, - int symbol, - Av1Distribution distribution) + public static int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution) { writer.WriteSymbol(symbol, distribution); return 0; @@ -173,10 +166,7 @@ internal sealed class Av1MotionVectorContext private readonly struct MotionVectorCostOperation : IMotionVectorSymbolOperation { /// - public static int ProcessSymbol( - Av1SymbolWriter writer, - int symbol, - Av1Distribution distribution) + public static int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution) => Av1ProbabilityCost.GetSymbolCost(distribution, symbol); } @@ -350,48 +340,72 @@ internal sealed class Av1MotionVectorContext } /// - /// Writes one signed integer-precision component. + /// Writes one signed motion-vector component. /// /// The tile range encoder. /// The nonzero component in one-eighth-sample units. - public void Write(Av1SymbolWriter writer, int value) - => _ = this.Process(writer, value); + /// The fractional precision selected by the frame header. + public void Write(Av1SymbolWriter writer, int value, Av1MotionVectorPrecision precision) + => _ = this.Process(writer, value, precision); /// /// Processes one nonzero signed component through the shared motion-vector symbol operation. /// - public int Process(Av1SymbolWriter writer, int value) + public int Process(Av1SymbolWriter writer, int value, Av1MotionVectorPrecision precision) where TOperation : struct, IMotionVectorSymbolOperation { int magnitude = Math.Abs(value); - DebugGuard.IsTrue(magnitude > 0 && (magnitude & 7) == 0, "Displacement-vector components must use whole-sample precision."); - - // Class zero contains the two whole-sample magnitudes 8 and 16. Above it, the highest set bit of magnitude - // minus one selects the doubling range; subtracting three converts the eighth-sample bit index to the class. - int magnitudeClass = magnitude <= (ClassZeroSize << 3) ? 0 : Av1Math.MostSignificantBit((uint)(magnitude - 1)) - 3; + int precisionMask = precision == Av1MotionVectorPrecision.Integer + ? 7 + : precision == Av1MotionVectorPrecision.QuarterSample ? 1 : 0; + + DebugGuard.IsTrue( + magnitude > 0 && (magnitude & precisionMask) == 0, + "Motion-vector components must match the frame precision."); + + // The coded value is magnitude minus one. Its whole-sample portion selects the doubling class, + // while the remainder carries integer offset, fractional phase, and the high-precision bit. + int codedMagnitude = magnitude - 1; + uint classValue = (uint)(codedMagnitude >> 3); + int magnitudeClass = classValue == 0 ? 0 : Av1Math.MostSignificantBit(classValue); DebugGuard.MustBeLessThan(magnitudeClass, MagnitudeClassCount, nameof(magnitudeClass)); + int magnitudeBase = magnitudeClass == 0 ? 0 : ClassZeroSize << (magnitudeClass + 2); + int offset = codedMagnitude - magnitudeBase; + int integerOffset = offset >> 3; + int fractional = (offset >> 1) & 3; + int highPrecision = offset & 1; int rate = TOperation.ProcessSymbol(writer, value < 0 ? 1 : 0, this.Sign); rate += TOperation.ProcessSymbol(writer, magnitudeClass, this.MagnitudeClass); if (magnitudeClass == 0) { - rate += TOperation.ProcessSymbol(writer, (magnitude >> 3) - 1, this.ClassZero); - return rate; + rate += TOperation.ProcessSymbol(writer, integerOffset, this.ClassZero); + } + else + { + for (int bit = 0; bit < magnitudeClass; bit++) + { + // Integer offsets are transmitted least-significant bit first through independent models. + rate += TOperation.ProcessSymbol(writer, (integerOffset >> bit) & 1, this.OffsetBits[bit]); + } } - // Remove the class base and the implicit low-bit value 7 plus the final one before coding the remaining - // whole-sample offset least-significant bit first. - int magnitudeBase = ClassZeroSize << (magnitudeClass + 2); - int integerOffset = (magnitude - magnitudeBase - 8) >> 3; + if (precision != Av1MotionVectorPrecision.Integer) + { + Av1Distribution fractionalDistribution = magnitudeClass == 0 + ? this.ClassZeroFractional[integerOffset] + : this.Fractional; - for (int bit = 0; bit < magnitudeClass; bit++) + rate += TOperation.ProcessSymbol(writer, fractional, fractionalDistribution); + } + + if (precision == Av1MotionVectorPrecision.EighthSample) { - // The decoder reconstructs offsets least-significant bit first, so each adaptive bit model must be - // updated in the same order during encoding. - rate += TOperation.ProcessSymbol( - writer, - (integerOffset >> bit) & 1, - this.OffsetBits[bit]); + Av1Distribution highPrecisionDistribution = magnitudeClass == 0 + ? this.ClassZeroHighPrecision + : this.HighPrecision; + + rate += TOperation.ProcessSymbol(writer, highPrecision, highPrecisionDistribution); } return rate; diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs index 612a2597a6..fee03ebaa7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; @@ -10,6 +11,108 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; /// internal static class Av1RateDistortion { + /// + /// Each fitted curve contains 65 equally spaced samples, including the cubic interpolation endpoints. + /// + private const int ModelCurveLength = 65; + + /// + /// Gets the rate-curve category for each AV1 block geometry. + /// + private static ReadOnlySpan ModelRateCategories => [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 1, 1, 2, 2, 3, 3]; + + /// + /// Gets the four block-size rate curves in fixed-point bit-cost units per sample. + /// + private static ReadOnlySpan ModelRateCurves => + [ + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 118.257702, 120.210658, 121.434853, 122.100487, + 122.377758, 122.436865, 72.290102, 96.974289, 101.652727, + 126.830141, 140.417377, 157.644879, 184.315291, 215.823873, + 262.300169, 335.919859, 420.624173, 519.185032, 619.854243, + 726.053595, 827.663369, 933.127475, 1037.988755, 1138.839609, + 1233.342933, 1333.508064, 1428.760126, 1533.396364, 1616.952052, + 1744.539319, 1803.413586, 1951.466618, 1994.227838, 2086.031680, + 2148.635443, 2239.068450, 2222.590637, 2338.859809, 2402.929011, + 2418.727875, 2435.342670, 2471.159469, 2523.187446, 2591.183827, + 2674.905840, 2774.110714, 2888.555675, 3017.997952, 3162.194773, + 3320.903365, 3493.880956, 3680.884773, 3881.672045, 4096.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 13.087244, 15.919735, 25.930313, 24.412411, + 28.567417, 29.924194, 30.857010, 32.742979, 36.382570, + 39.210386, 42.265690, 47.378572, 57.014850, 82.740067, + 137.346562, 219.968084, 316.781856, 415.643773, 516.706538, + 614.914364, 714.303763, 815.512135, 911.210485, 1008.501528, + 1109.787854, 1213.772279, 1322.922561, 1414.752579, 1510.505641, + 1615.741888, 1697.989032, 1780.123933, 1847.453790, 1913.742309, + 1960.828122, 2047.500168, 2085.454095, 2129.230668, 2158.171824, + 2182.231724, 2217.684864, 2269.589211, 2337.264824, 2420.618694, + 2519.557814, 2633.989178, 2763.819779, 2908.956609, 3069.306660, + 3244.776927, 3435.274401, 3640.706076, 3860.978945, 4096.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 4.656893, 5.123633, 5.594132, 6.162376, + 6.918433, 7.768444, 8.739415, 10.105862, 11.477328, + 13.236604, 15.421030, 19.093623, 25.801871, 46.724612, + 98.841054, 181.113466, 272.586364, 359.499769, 445.546343, + 525.944439, 605.188743, 681.793483, 756.668359, 838.486885, + 926.950356, 1015.482542, 1113.353926, 1204.897193, 1288.871992, + 1373.464145, 1455.746628, 1527.796460, 1588.475066, 1658.144771, + 1710.302500, 1807.563351, 1863.197608, 1927.281616, 1964.450872, + 2022.719898, 2100.041145, 2185.205712, 2280.993936, 2387.616216, + 2505.282950, 2634.204540, 2774.591385, 2926.653884, 3090.602436, + 3266.647443, 3454.999303, 3655.868416, 3869.465182, 4096.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.000000, 0.000000, 0.000000, 0.000000, + 0.000000, 0.337370, 0.391916, 0.468839, 0.566334, + 0.762564, 1.069225, 1.384361, 1.787581, 2.293948, + 3.251909, 4.412991, 8.050068, 11.606073, 27.668092, + 65.227758, 128.463938, 202.097653, 262.715851, 312.464873, + 355.601398, 400.609054, 447.201352, 495.761568, 552.871938, + 619.067625, 691.984883, 773.753288, 860.628503, 946.262808, + 1019.805896, 1106.061360, 1178.422145, 1244.852258, 1302.173987, + 1399.650266, 1548.092912, 1545.928652, 1670.817500, 1694.523823, + 1779.195362, 1882.155494, 1990.662097, 2108.325181, 2235.456119, + 2372.366287, 2519.367059, 2676.769812, 2844.885918, 3024.026754, + 3214.503695, 3416.628115, 3630.711389, 3857.064892, 4096.000000, + ]; + + /// + /// Gets the low- and high-error distortion curves in sixteenth-sample-error units. + /// + private static ReadOnlySpan ModelDistortionCurves => + [ + 16.000000, 15.962891, 15.925174, 15.886888, 15.848074, + 15.808770, 15.769015, 15.728850, 15.688313, 15.647445, + 15.606284, 15.564870, 15.525918, 15.483820, 15.373330, + 15.126844, 14.637442, 14.184387, 13.560070, 12.880717, + 12.165995, 11.378144, 10.438769, 9.130790, 7.487633, + 5.688649, 4.267515, 3.196300, 2.434201, 1.834064, + 1.369920, 1.035921, 0.775279, 0.574895, 0.427232, + 0.314123, 0.233236, 0.171440, 0.128188, 0.092762, + 0.067569, 0.049324, 0.036330, 0.027008, 0.019853, + 0.015539, 0.011093, 0.008733, 0.007624, 0.008105, + 0.005427, 0.004065, 0.003427, 0.002848, 0.002328, + 0.001865, 0.001457, 0.001103, 0.000801, 0.000550, + 0.000348, 0.000193, 0.000085, 0.000021, 0.000000, + 16.000000, 15.996116, 15.984769, 15.966413, 15.941505, + 15.910501, 15.873856, 15.832026, 15.785466, 15.734633, + 15.679981, 15.621967, 15.560961, 15.460157, 15.288367, + 15.052462, 14.466922, 13.921212, 13.073692, 12.222005, + 11.237799, 9.985848, 8.898823, 7.423519, 5.995325, + 4.773152, 3.744032, 2.938217, 2.294526, 1.762412, + 1.327145, 1.020728, 0.765535, 0.570548, 0.425833, + 0.313825, 0.232959, 0.171324, 0.128174, 0.092750, + 0.067558, 0.049319, 0.036330, 0.027008, 0.019853, + 0.015539, 0.011093, 0.008733, 0.007624, 0.008105, + 0.005427, 0.004065, 0.003427, 0.002848, 0.002328, + 0.001865, 0.001457, 0.001103, 0.000801, 0.000550, + 0.000348, 0.000193, 0.000085, 0.000021, -0.000000, + ]; + /// /// Gets the key-frame rate multiplier for an AV1 quantizer and sample precision. /// @@ -32,6 +135,28 @@ internal static class Av1RateDistortion return (int)Math.Max(multiplier, 1); } + /// + /// Gets the inter-frame rate multiplier for an AV1 quantizer and sample precision. + /// + /// The segment quantizer index. + /// The coded sample bit depth. + /// The rate multiplier. + public static int GetInterFrameRateMultiplier(int qIndex, Av1BitDepth bitDepth) + { + int quantizer = Av1QuantizationLookup.GetDcQuant(qIndex, 0, bitDepth); + + // Ordinary inter frames use a slightly lower rate weight than key frames, preserving more residual detail. + // Distortion remains normalized to the eight-bit domain before it is combined with this value. + long multiplier = (long)((quantizer * (long)quantizer) * (3.2 + (0.0015 * quantizer))); + int shift = (bitDepth.GetBitCount() - 8) * 2; + if (shift > 0) + { + multiplier = (multiplier + (1L << (shift - 1))) >> shift; + } + + return (int)Math.Max(multiplier, 1); + } + /// /// Gets a rate-distortion cost using the encoder probability-cost precision. /// @@ -86,10 +211,7 @@ internal static class Av1RateDistortion /// The motion-vector syntax rate in 1/512-bit units. /// The unnormalized sample-domain absolute difference. /// The absolute difference plus the motion-vector search cost. - public static int GetMotionSearchSadCost( - int sadPerBit, - int motionVectorRate, - int sumOfAbsoluteDifferences) + public static int GetMotionSearchSadCost(int sadPerBit, int motionVectorRate, int sumOfAbsoluteDifferences) { const int MotionRateShift = 9; @@ -99,4 +221,99 @@ internal static class Av1RateDistortion int motionError = (int)((weightedRate + (1 << (MotionRateShift - 1))) >> MotionRateShift); return sumOfAbsoluteDifferences + motionError; } + + /// + /// Estimates residual rate and distortion from prediction error without running transforms or quantization. + /// + /// The plane block geometry selecting the fitted rate curve. + /// The visible prediction error normalized to eight-bit precision. + /// The number of visible samples contributing to the error. + /// The plane AC dequantization step at native sample precision. + /// The native sample precision. + /// The block's rate-distortion multiplier. + /// The estimated residual rate in 1/512-bit units. + /// The estimated residual distortion in sixteenth-sample-error units. + public static void ModelPredictionError( + Av1BlockSize blockSize, + long squaredError, + int sampleCount, + int acQuantizer, + Av1BitDepth bitDepth, + int rateMultiplier, + out int rate, + out long distortion) + { + if (squaredError == 0) + { + rate = 0; + distortion = 0; + return; + } + + const double CurveStart = -15.5; + const double CurveStep = 0.5; + const double EndpointMargin = 1E-6; + const double HighErrorThreshold = 16; + const int DistortionScaleShift = 4; + + // Transform dequantizers are scaled by eight. Normalize both their precision and the prediction error + // before taking the logarithmic feature, so the same fitted curves serve eight-, ten-, and twelve-bit input. + int quantizerStep = Math.Max(acQuantizer >> (bitDepth.GetBitCount() - 5), 1); + double normalizedError = (double)squaredError / sampleCount; + double feature = Math.Log2(normalizedError / ((double)quantizerStep * quantizerStep)); + double lastCurvePosition = CurveStart + ((ModelCurveLength - 1) * CurveStep); + feature = Math.Clamp(feature, CurveStart + CurveStep + EndpointMargin, lastCurvePosition - CurveStep - EndpointMargin); + double position = (feature - CurveStart) / CurveStep; + int index = (int)position; + double fraction = position - index; + int rateCategory = ModelRateCategories[(int)blockSize]; + int distortionCategory = normalizedError > HighErrorThreshold ? 1 : 0; + ReadOnlySpan ratePoints = ModelRateCurves.Slice((rateCategory * ModelCurveLength) + index - 1, 4); + ReadOnlySpan distortionPoints = ModelDistortionCurves.Slice((distortionCategory * ModelCurveLength) + index - 1, 4); + double rateEstimate; + double distortionEstimate; + if (Vector128.IsHardwareAccelerated) + { + // The two lanes evaluate rate and distortion together. Keep the cubic polynomial's operation order, + // including its separate multiplies and adds, so vector and scalar rounding agree at decision boundaries. + Vector128 p0 = Vector128.Create(ratePoints[0], distortionPoints[0]); + Vector128 p1 = Vector128.Create(ratePoints[1], distortionPoints[1]); + Vector128 p2 = Vector128.Create(ratePoints[2], distortionPoints[2]); + Vector128 p3 = Vector128.Create(ratePoints[3], distortionPoints[3]); + Vector128 x = Vector128.Create(fraction); + Vector128 cubic = (Vector128.Create(3.0) * (p1 - p2)) + p3 - p0; + Vector128 quadratic = (Vector128.Create(2.0) * p0) - (Vector128.Create(5.0) * p1) + (Vector128.Create(4.0) * p2) - p3; + Vector128 result = p1 + (Vector128.Create(0.5) * x * (p2 - p0 + (x * (quadratic + (x * cubic))))); + + rateEstimate = result.GetElement(0); + distortionEstimate = result.GetElement(1); + } + else + { + rateEstimate = InterpolateModelCurve(ratePoints, fraction); + distortionEstimate = InterpolateModelCurve(distortionPoints, fraction); + } + + rate = (int)(Math.Max(0, rateEstimate * sampleCount) + 0.5); + distortion = (long)(Math.Max(0, (distortionEstimate * normalizedError) * sampleCount) + 0.5); + long skipDistortion = squaredError << DistortionScaleShift; + + // A modeled coded residual is useful only if it beats leaving the prediction unchanged. Preserve the + // reference model's zero-rate rule instead of returning an artificially low distortion for a skipped block. + if (rate == 0 || GetCost(rateMultiplier, rate, distortion) >= GetCost(rateMultiplier, 0, skipDistortion)) + { + rate = 0; + distortion = skipDistortion; + } + } + + /// + /// Evaluates one fitted curve's cubic segment without fusing arithmetic operations. + /// + private static double InterpolateModelCurve(ReadOnlySpan points, double fraction) + { + double cubic = (3.0 * (points[1] - points[2])) + points[3] - points[0]; + double quadratic = (2.0 * points[0]) - (5.0 * points[1]) + (4.0 * points[2]) - points[3]; + return points[1] + (0.5 * fraction * (points[2] - points[0] + (fraction * (quadratic + (fraction * cubic))))); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs index 1b7ddd9706..d7d56d3249 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs @@ -46,7 +46,7 @@ internal static class Av1SymbolContextHelper /// /// The number of interpolation filters selectable by per-block switchable syntax. /// - private const int SwitchableInterpolationFilterCount = 3; + public const int SwitchableInterpolationFilterCount = 3; /// /// The number of transform types represented by each flattened transform-set row. @@ -961,25 +961,70 @@ internal static class Av1SymbolContextHelper Av1BlockModeInfo? left, int direction) { - const int filterContextCount = SwitchableInterpolationFilterCount + 1; - const int horizontalContextOffset = filterContextCount * 2; ReadOnlySpan referenceFrames = modeInfo.ReferenceFrames; Av1ReferenceFrameType primaryReference = referenceFrames[0]; bool isCompound = referenceFrames[1] > Av1ReferenceFrameType.Intra; - // The sixteen rows are laid out as single vertical, compound vertical, single horizontal, then compound - // horizontal, with four neighbor states in each group. - int context = (isCompound ? filterContextCount : 0) + (direction * horizontalContextOffset); int leftFilter = GetReferenceInterpolationFilterContext(left, primaryReference, direction); int aboveFilter = GetReferenceInterpolationFilterContext(above, primaryReference, direction); + return GetSwitchableInterpolationContext(aboveFilter, leftFilter, isCompound, direction); + } + + /// + /// Gets the switchable interpolation-filter context from the encoder's packed single-reference neighbors. + /// + /// The current inter block. + /// The current block's available spatial neighbors. + /// Zero for the vertical filter or one for the horizontal filter. + /// The single-reference context for the selected direction. + public static int GetSwitchableInterpolationContext( + Av1EncoderBlockModeInfo modeInfo, + Av1MacroBlockD macroBlock, + int direction) + { + int aboveFilter = SwitchableInterpolationFilterCount; + int leftFilter = SwitchableInterpolationFilterCount; + if (macroBlock.IsUpAvailable) + { + Av1EncoderBlockModeInfo above = macroBlock.GetRelativeModeInfo(-macroBlock.ModeInfoStride).Block; + if (above.ReferenceFrame == modeInfo.ReferenceFrame) + { + aboveFilter = (int)(direction == 0 ? above.VerticalInterpolationFilter : above.HorizontalInterpolationFilter); + } + } + + if (macroBlock.IsLeftAvailable) + { + Av1EncoderBlockModeInfo left = macroBlock.GetRelativeModeInfo(-1).Block; + if (left.ReferenceFrame == modeInfo.ReferenceFrame) + { + leftFilter = (int)(direction == 0 ? left.VerticalInterpolationFilter : left.HorizontalInterpolationFilter); + } + } + + return GetSwitchableInterpolationContext(aboveFilter, leftFilter, isCompound: false, direction); + } + + /// + /// Combines the two neighboring filter states into the shared encoder and decoder context layout. + /// + private static int GetSwitchableInterpolationContext(int aboveFilter, int leftFilter, bool isCompound, int direction) + { + const int filterContextCount = SwitchableInterpolationFilterCount + 1; + const int horizontalContextOffset = filterContextCount * 2; + + // The sixteen rows are single vertical, compound vertical, single horizontal, then compound horizontal, + // with four neighbor states in each group. Both storage representations must use this same mapping. + int context = (isCompound ? filterContextCount : 0) + (direction * horizontalContextOffset); + if (leftFilter == aboveFilter) { return context + leftFilter; } - // The fourth neighbor state is not a selectable Bilinear filter. It is the value the reference decoder uses when a neighbor - // does not share the current primary reference, and when two contributing neighbors selected different filters. + // The fourth neighbor state means no matching primary reference or disagreement between contributing + // neighbors. It is not the Bilinear filter, which is absent from the switchable alphabet. if (leftFilter == SwitchableInterpolationFilterCount) { return context + aboveFilter; diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs index 87a3170933..52d2b50067 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs @@ -6,6 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; @@ -22,6 +23,11 @@ internal sealed class Av1SymbolEncoder : IDisposable /// private const int MaximumCoefficientContextCount = (Av1Constants.MaxTransformSize / 2) * (Av1Constants.MaxTransformSize / 2); + /// + /// Owns every mutable tile distribution and restores normative defaults without rebuilding the object graph. + /// + private readonly Av1FrameEntropyContext entropyContext; + /// /// The tile-adaptive intra-block-copy distribution. /// @@ -30,7 +36,32 @@ internal sealed class Av1SymbolEncoder : IDisposable /// /// The tile-adaptive integer displacement-vector context. /// - private readonly Av1MotionVectorContext displacementVector = new(); + private readonly Av1MotionVectorContext displacementVector; + + /// + /// The tile-adaptive normal inter motion-vector context. + /// + private readonly Av1MotionVectorContext motionVector; + + /// + /// The tile-adaptive NEWMV branch distributions. + /// + private readonly Av1Distribution[] newMotionVector; + + /// + /// The tile-adaptive GLOBALMV branch distributions. + /// + private readonly Av1Distribution[] zeroMotionVector; + + /// + /// The tile-adaptive NEARESTMV branch distributions. + /// + private readonly Av1Distribution[] referenceMotionVector; + + /// + /// The tile-adaptive dynamic-reference-list distributions. + /// + private readonly Av1Distribution[] dynamicReferenceList; /// /// The tile-adaptive partition-type distributions. @@ -42,6 +73,21 @@ internal sealed class Av1SymbolEncoder : IDisposable /// private readonly Av1Distribution[][] keyFrameYMode; + /// + /// The tile-adaptive inter-frame intra luma-mode distributions. + /// + private readonly Av1Distribution[] frameYMode; + + /// + /// The tile-adaptive intra-versus-inter distributions. + /// + private readonly Av1Distribution[] intraInter; + + /// + /// The tile-adaptive single-reference branch distributions. + /// + private readonly Av1Distribution[][] singleReference; + /// /// The tile-adaptive chroma intra-mode distributions. /// @@ -82,11 +128,6 @@ internal sealed class Av1SymbolEncoder : IDisposable /// private readonly Av1Distribution filterIntraMode; - /// - /// The palette probability state, created only when screen-content coding uses it. - /// - private PaletteEntropyContext? paletteEntropyContext; - /// /// The tile-adaptive absolute quantizer delta distribution. /// @@ -157,20 +198,15 @@ internal sealed class Av1SymbolEncoder : IDisposable /// private bool isDisposed; - /// - /// The configuration providing lazily allocated coefficient scratch. - /// - private readonly Configuration configuration; - /// /// The reusable padded coefficient levels used to derive entropy contexts. /// - private Av1LevelBuffer? levels; + private readonly Av1LevelBuffer levels; /// /// The reusable raster-order coefficient contexts for one transform. /// - private IMemoryOwner? coefficientContexts; + private readonly IMemoryOwner coefficientContexts; /// /// The range writer producing the current tile payload. @@ -183,7 +219,7 @@ internal sealed class Av1SymbolEncoder : IDisposable private readonly int baseQIndex; /// - /// Initializes a new instance of the class for one AV1 tile. + /// Initializes a new instance of the class with reusable tile state. /// /// The configuration providing output and temporary memory. /// The complete fixed output allocation length in bytes. @@ -191,34 +227,50 @@ internal sealed class Av1SymbolEncoder : IDisposable /// A value indicating whether encoded symbols adapt their tile distributions. public Av1SymbolEncoder(Configuration configuration, int bufferLength, int qIndex, bool updateCdf) { - this.configuration = configuration; - - // Every default accessor creates independently mutable state. Encoding and decoding therefore begin from - // equivalent tile-local models without constructing and immediately deep-copying a second object graph. - this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy; - this.tilePartitionTypes = Av1DefaultDistributions.PartitionTypes; - this.keyFrameYMode = Av1DefaultDistributions.KeyFrameYMode; - this.uvMode = Av1DefaultDistributions.UvMode; - this.filterIntra = Av1DefaultDistributions.FilterIntra; - this.filterIntraMode = Av1DefaultDistributions.FilterIntraMode; - this.deltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute; - this.intraExtendedTransform = Av1DefaultDistributions.IntraExtendedTransform; - this.interExtendedTransform = Av1DefaultDistributions.InterExtendedTransform; - this.transformSize = Av1DefaultDistributions.TransformSize; - this.transformPartition = Av1DefaultDistributions.TransformPartition; - this.segmentId = Av1DefaultDistributions.SegmentId; - this.angleDelta = Av1DefaultDistributions.AngleDelta; - this.skip = Av1DefaultDistributions.Skip; - this.skipMode = Av1DefaultDistributions.SkipMode; - this.chromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign; - this.chromaFromLumaAlpha = Av1DefaultDistributions.ChromaFromLumaAlpha; - this.transformBlockSkip = Av1DefaultDistributions.GetTransformBlockSkip(qIndex); - this.endOfBlockFlag = Av1DefaultDistributions.GetEndOfBlockFlag(qIndex); - this.coefficientsBaseRange = Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex); - this.coefficientsBase = Av1DefaultDistributions.GetCoefficientsBase(qIndex); - this.coefficientsBaseEndOfBlock = Av1DefaultDistributions.GetBaseEndOfBlock(qIndex); - this.dcSign = Av1DefaultDistributions.GetDcSign(qIndex); - this.endOfBlockExtra = Av1DefaultDistributions.GetEndOfBlockExtra(qIndex); + this.entropyContext = new Av1FrameEntropyContext(qIndex); + + // Encoder and decoder now share the same mutable context shape. Every field aliases that single graph so + // sequence samples can restore normative defaults without replacing any distribution or array. + this.tileIntraBlockCopy = this.entropyContext.IntraBlockCopy; + this.motionVector = this.entropyContext.MotionVector; + this.displacementVector = this.entropyContext.DisplacementVector; + this.tilePartitionTypes = this.entropyContext.PartitionTypes; + this.keyFrameYMode = this.entropyContext.KeyFrameYMode; + this.frameYMode = this.entropyContext.FrameYMode; + this.intraInter = this.entropyContext.IntraInter; + this.singleReference = this.entropyContext.SingleReference; + this.newMotionVector = this.entropyContext.NewMv; + this.zeroMotionVector = this.entropyContext.ZeroMv; + this.referenceMotionVector = this.entropyContext.RefMv; + this.dynamicReferenceList = this.entropyContext.Drl; + this.uvMode = this.entropyContext.UvMode; + this.filterIntra = this.entropyContext.FilterIntra; + this.filterIntraMode = this.entropyContext.FilterIntraMode; + this.deltaQuantizerAbsolute = this.entropyContext.DeltaQuantizerAbsolute; + this.intraExtendedTransform = this.entropyContext.IntraExtendedTransform; + this.interExtendedTransform = this.entropyContext.InterExtendedTransform; + this.transformSize = this.entropyContext.TransformSize; + this.transformPartition = this.entropyContext.TransformPartition; + this.segmentId = this.entropyContext.SegmentId; + this.angleDelta = this.entropyContext.AngleDelta; + this.skip = this.entropyContext.Skip; + this.skipMode = this.entropyContext.SkipMode; + this.chromaFromLumaSign = this.entropyContext.ChromaFromLumaSign; + this.chromaFromLumaAlpha = this.entropyContext.ChromaFromLumaAlpha; + this.transformBlockSkip = this.entropyContext.TransformBlockSkip; + this.endOfBlockFlag = this.entropyContext.EndOfBlockFlag; + this.coefficientsBaseRange = this.entropyContext.CoefficientsBaseRange; + this.coefficientsBase = this.entropyContext.CoefficientsBase; + this.coefficientsBaseEndOfBlock = this.entropyContext.BaseEndOfBlock; + this.dcSign = this.entropyContext.DcSign; + this.endOfBlockExtra = this.entropyContext.EndOfBlockExtra; + + // Transform dimensions are bounded by the AV1 coefficient-coding rules, so the complete entropy scratch + // is known with the tile output capacity and remains valid for every transform in every sequence sample. + this.levels = new Av1LevelBuffer(configuration); + this.coefficientContexts = + configuration.MemoryAllocator.Allocate(MaximumCoefficientContextCount); + this.writer = new(configuration, bufferLength, updateCdf); this.baseQIndex = qIndex; } @@ -287,6 +339,25 @@ internal sealed class Av1SymbolEncoder : IDisposable int colorOrderIndex); } + /// + /// Restores the initial tile distributions and range coder while retaining their complete object graph and buffers. + /// + public void Reset() + { + this.entropyContext.ResetToDefaults(this.baseQIndex); + this.writer.Reset(); + } + + /// + /// Restores the initial tile distributions and begins the next tile at an offset in the retained output buffer. + /// + /// The first output byte available to the next tile. + public void Reset(int outputOffset) + { + this.entropyContext.ResetToDefaults(this.baseQIndex); + this.writer.Reset(outputOffset); + } + /// /// Writes an unsigned fixed-width literal to the tile entropy stream. /// @@ -342,9 +413,8 @@ internal sealed class Av1SymbolEncoder : IDisposable /// The rate cost in 1/512-bit units. public int GetPaletteYModeCost(bool usePalette, int blockSizeContext, int neighborContext) { - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); return Av1ProbabilityCost.GetSymbolCost( - context.YMode[blockSizeContext][neighborContext], + this.entropyContext.PaletteYMode[blockSizeContext][neighborContext], usePalette ? 1 : 0); } @@ -357,8 +427,7 @@ internal sealed class Av1SymbolEncoder : IDisposable public void WritePaletteYMode(bool usePalette, int blockSizeContext, int neighborContext) { ref Av1SymbolWriter w = ref this.writer; - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); - w.WriteSymbol(usePalette, context.YMode[blockSizeContext][neighborContext]); + w.WriteSymbol(usePalette, this.entropyContext.PaletteYMode[blockSizeContext][neighborContext]); } /// @@ -369,9 +438,8 @@ internal sealed class Av1SymbolEncoder : IDisposable /// The rate cost in 1/512-bit units. public int GetPaletteUvModeCost(bool usePalette, bool hasLumaPalette) { - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); return Av1ProbabilityCost.GetSymbolCost( - context.UvMode[hasLumaPalette ? 1 : 0], + this.entropyContext.PaletteUvMode[hasLumaPalette ? 1 : 0], usePalette ? 1 : 0); } @@ -383,8 +451,7 @@ internal sealed class Av1SymbolEncoder : IDisposable public void WritePaletteUvMode(bool usePalette, bool hasLumaPalette) { ref Av1SymbolWriter w = ref this.writer; - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); - w.WriteSymbol(usePalette, context.UvMode[hasLumaPalette ? 1 : 0]); + w.WriteSymbol(usePalette, this.entropyContext.PaletteUvMode[hasLumaPalette ? 1 : 0]); } /// @@ -396,10 +463,9 @@ internal sealed class Av1SymbolEncoder : IDisposable /// The rate cost in 1/512-bit units. public int GetPaletteSizeCost(int paletteSize, int blockSizeContext, Av1PlaneType planeType) { - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); Av1Distribution distribution = planeType == Av1PlaneType.Y - ? context.YSize[blockSizeContext] - : context.UvSize[blockSizeContext]; + ? this.entropyContext.PaletteYSize[blockSizeContext] + : this.entropyContext.PaletteUvSize[blockSizeContext]; return Av1ProbabilityCost.GetSymbolCost(distribution, paletteSize - 2); } @@ -413,10 +479,9 @@ internal sealed class Av1SymbolEncoder : IDisposable public void WritePaletteSize(int paletteSize, int blockSizeContext, Av1PlaneType planeType) { ref Av1SymbolWriter w = ref this.writer; - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); Av1Distribution distribution = planeType == Av1PlaneType.Y - ? context.YSize[blockSizeContext] - : context.UvSize[blockSizeContext]; + ? this.entropyContext.PaletteYSize[blockSizeContext] + : this.entropyContext.PaletteUvSize[blockSizeContext]; w.WriteSymbol(paletteSize - 2, distribution); } @@ -435,10 +500,9 @@ internal sealed class Av1SymbolEncoder : IDisposable int colorContext, Av1PlaneType planeType) { - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); Av1Distribution distribution = planeType == Av1PlaneType.Y - ? context.YColorIndex[paletteSize - 2][colorContext] - : context.UvColorIndex[paletteSize - 2][colorContext]; + ? this.entropyContext.PaletteYColorIndex[paletteSize - 2][colorContext] + : this.entropyContext.PaletteUvColorIndex[paletteSize - 2][colorContext]; return Av1ProbabilityCost.GetSymbolCost(distribution, colorOrderIndex); } @@ -457,10 +521,9 @@ internal sealed class Av1SymbolEncoder : IDisposable Av1PlaneType planeType) { ref Av1SymbolWriter w = ref this.writer; - PaletteEntropyContext context = this.paletteEntropyContext ??= new(); Av1Distribution distribution = planeType == Av1PlaneType.Y - ? context.YColorIndex[paletteSize - 2][colorContext] - : context.UvColorIndex[paletteSize - 2][colorContext]; + ? this.entropyContext.PaletteYColorIndex[paletteSize - 2][colorContext] + : this.entropyContext.PaletteUvColorIndex[paletteSize - 2][colorContext]; w.WriteSymbol(colorOrderIndex, distribution); } @@ -695,7 +758,7 @@ internal sealed class Av1SymbolEncoder : IDisposable /// The displacement vector to encode. /// The spatially derived reference vector. public void WriteDisplacementVector(Av1MotionVector value, Av1MotionVector reference) - => this.displacementVector.Write(this.writer, value, reference); + => this.displacementVector.Write(this.writer, value, reference, Av1MotionVectorPrecision.Integer); /// /// Measures an integer intra-block-copy displacement vector against the live distributions. @@ -707,7 +770,11 @@ internal sealed class Av1SymbolEncoder : IDisposable { const int DisplacementVectorCostWeight = 120; const int WeightShift = 7; - int rate = this.displacementVector.GetCost(this.writer, value, reference); + int rate = this.displacementVector.GetCost( + this.writer, + value, + reference, + Av1MotionVectorPrecision.Integer); // Displacement syntax uses a 120/128 discount during mode search; adding half the divisor rounds to nearest. return ((rate * DisplacementVectorCostWeight) + (1 << (WeightShift - 1))) >> WeightShift; @@ -720,7 +787,133 @@ internal sealed class Av1SymbolEncoder : IDisposable /// The spatially derived reference vector. /// The syntax cost in 1/512-bit units. public int GetDisplacementVectorSearchCost(Av1MotionVector value, Av1MotionVector reference) - => this.displacementVector.GetCost(this.writer, value, reference); + => this.displacementVector.GetCost( + this.writer, + value, + reference, + Av1MotionVectorPrecision.Integer); + + /// + /// Measures one switchable interpolation filter against its live tile distribution. + /// + /// The regular, smooth, or sharp filter. + /// The spatial filter context for the selected direction. + /// The syntax cost in 1/512-bit units. + public int GetSwitchableInterpolationFilterCost(Av1InterpolationFilter filter, int context) + => Av1ProbabilityCost.GetSymbolCost(this.entropyContext.SwitchableInterpolation[context], (int)filter); + + /// + /// Writes one switchable interpolation filter and updates its live tile distribution. + /// + /// The regular, smooth, or sharp filter. + /// The spatial filter context for the selected direction. + public void WriteSwitchableInterpolationFilter(Av1InterpolationFilter filter, int context) + => this.writer.WriteSymbol((int)filter, this.entropyContext.SwitchableInterpolation[context]); + + /// + /// Measures a single-reference inter mode against the live branch distributions. + /// + /// The new, global, nearest, or near motion-vector mode. + /// The packed context derived from the reference-vector stack. + /// The syntax cost in 1/512-bit units. + public int GetInterModeCost(Av1PredictionMode mode, int modeContext) + { + bool isNotNew = mode != Av1PredictionMode.NewMotionVector; + int rate = Av1ProbabilityCost.GetSymbolCost( + this.newMotionVector[Av1SymbolContextHelper.GetNewMvContext(modeContext)], + isNotNew ? 1 : 0); + + if (!isNotNew) + { + return rate; + } + + bool isNotGlobal = mode != Av1PredictionMode.GlobalMotionVector; + rate += Av1ProbabilityCost.GetSymbolCost( + this.zeroMotionVector[Av1SymbolContextHelper.GetZeroMvContext(modeContext)], + isNotGlobal ? 1 : 0); + + if (!isNotGlobal) + { + return rate; + } + + return rate + Av1ProbabilityCost.GetSymbolCost( + this.referenceMotionVector[Av1SymbolContextHelper.GetRefMvContext(modeContext)], + mode == Av1PredictionMode.NearMotionVector ? 1 : 0); + } + + /// + /// Writes a single-reference inter mode through the NEWMV, GLOBALMV, and NEARESTMV branch tree. + /// + /// The new, global, nearest, or near motion-vector mode. + /// The packed context derived from the reference-vector stack. + public void WriteInterMode(Av1PredictionMode mode, int modeContext) + { + ref Av1SymbolWriter w = ref this.writer; + bool isNotNew = mode != Av1PredictionMode.NewMotionVector; + w.WriteSymbol(isNotNew, this.newMotionVector[Av1SymbolContextHelper.GetNewMvContext(modeContext)]); + if (!isNotNew) + { + return; + } + + bool isNotGlobal = mode != Av1PredictionMode.GlobalMotionVector; + w.WriteSymbol(isNotGlobal, this.zeroMotionVector[Av1SymbolContextHelper.GetZeroMvContext(modeContext)]); + if (!isNotGlobal) + { + return; + } + + w.WriteSymbol( + mode == Av1PredictionMode.NearMotionVector, + this.referenceMotionVector[Av1SymbolContextHelper.GetRefMvContext(modeContext)]); + } + + /// + /// Measures one dynamic-reference-list advance decision. + /// + /// Whether selection advances to the next candidate. + /// The candidate-weight context. + /// The syntax cost in 1/512-bit units. + public int GetDynamicReferenceListCost(bool advance, int context) + => Av1ProbabilityCost.GetSymbolCost(this.dynamicReferenceList[context], advance ? 1 : 0); + + /// + /// Writes one dynamic-reference-list advance decision. + /// + /// Whether selection advances to the next candidate. + /// The candidate-weight context. + public void WriteDynamicReferenceList(bool advance, int context) + { + ref Av1SymbolWriter w = ref this.writer; + w.WriteSymbol(advance, this.dynamicReferenceList[context]); + } + + /// + /// Measures an inter motion vector relative to its selected stack reference. + /// + /// The selected motion vector. + /// The differential reference from the candidate stack. + /// The fractional precision selected by the frame header. + /// The syntax cost in 1/512-bit units. + public int GetMotionVectorCost( + Av1MotionVector value, + Av1MotionVector reference, + Av1MotionVectorPrecision precision) + => this.motionVector.GetCost(this.writer, value, reference, precision); + + /// + /// Writes an inter motion vector relative to its selected stack reference. + /// + /// The selected motion vector. + /// The differential reference from the candidate stack. + /// The fractional precision selected by the frame header. + public void WriteMotionVector( + Av1MotionVector value, + Av1MotionVector reference, + Av1MotionVectorPrecision precision) + => this.motionVector.Write(this.writer, value, reference, precision); /// /// Gets the current fixed-point cost of a complete block partition symbol. @@ -1136,15 +1329,11 @@ internal sealed class Av1SymbolEncoder : IDisposable bool clearLevels, out Span coefficientContexts) { - Av1LevelBuffer levels = this.levels ??= new(this.configuration); - IMemoryOwner coefficientContextOwner = this.coefficientContexts ??= - this.configuration.MemoryAllocator.Allocate(MaximumCoefficientContextCount); - // AV1 omits high-frequency coefficients beyond 32 samples on every 64-point transform dimension. The tile // creates maximum-sized workspaces once, then changes only the active views for subsequent transform blocks. - levels.Reset(new Size(width, height), clearLevels); - coefficientContexts = coefficientContextOwner.Memory.Span[..(width * height)]; - return levels; + this.levels.Reset(new Size(width, height), clearLevels); + coefficientContexts = this.coefficientContexts.Memory.Span[..(width * height)]; + return this.levels; } /// @@ -1314,21 +1503,23 @@ internal sealed class Av1SymbolEncoder : IDisposable /// /// The memory owner containing the encoded tile bytes. public IMemoryOwner Exit() - { - ref Av1SymbolWriter w = ref this.writer; - return w.Exit(); - } + => this.writer.Exit(); /// /// Finalizes the range-coded tile payload and exposes its encoded prefix without copying. /// /// The number of encoded bytes in the returned memory. - /// The encoded prefix, valid until this encoder is disposed. + /// The encoded prefix, valid until this encoder is reset or disposed. public ReadOnlyMemory Exit(out int length) - { - ref Av1SymbolWriter w = ref this.writer; - return w.Exit(out length); - } + => this.writer.Exit(out length); + + /// + /// Exposes a prefix containing every consecutively encoded tile without copying their bytes. + /// + /// The number of bytes in the prefix. + /// The encoded prefix, valid until this encoder is reset to offset zero or disposed. + public ReadOnlyMemory GetOutput(int length) + => this.writer.GetOutput(length); /// /// Releases the range-coder output buffer and coefficient scratch memory. @@ -1337,8 +1528,8 @@ internal sealed class Av1SymbolEncoder : IDisposable { if (!this.isDisposed) { - this.coefficientContexts?.Dispose(); - this.levels?.Dispose(); + this.coefficientContexts.Dispose(); + this.levels.Dispose(); this.writer.Dispose(); this.isDisposed = true; } @@ -1657,6 +1848,142 @@ internal sealed class Av1SymbolEncoder : IDisposable w.WriteSymbol((int)lumaMode, this.keyFrameYMode[topContext][leftContext]); } + /// + /// Gets the cost of an intra luma mode coded inside an inter frame. + /// + /// The intra luma mode. + /// The coding block size selecting the size group. + /// The syntax cost in 1/512-bit units. + public int GetInterFrameLumaModeCost(Av1PredictionMode lumaMode, Av1BlockSize blockSize) + => Av1ProbabilityCost.GetSymbolCost(this.frameYMode[blockSize.GetSizeGroup()], (int)lumaMode); + + /// + /// Writes an intra luma mode coded inside an inter frame. + /// + /// The intra luma mode. + /// The coding block size selecting the size group. + public void WriteInterFrameLumaMode(Av1PredictionMode lumaMode, Av1BlockSize blockSize) + { + ref Av1SymbolWriter w = ref this.writer; + w.WriteSymbol((int)lumaMode, this.frameYMode[blockSize.GetSizeGroup()]); + } + + /// + /// Gets the cost of the prediction-domain decision for an inter-frame block. + /// + /// Whether the block uses a retained reference frame. + /// The neighboring prediction-domain context. + /// The syntax cost in 1/512-bit units. + public int GetIsInterCost(bool isInter, int context) + => Av1ProbabilityCost.GetSymbolCost(this.intraInter[context], isInter ? 1 : 0); + + /// + /// Writes the prediction-domain decision for an inter-frame block. + /// + /// Whether the block uses a retained reference frame. + /// The neighboring prediction-domain context. + public void WriteIsInter(bool isInter, int context) + { + ref Av1SymbolWriter w = ref this.writer; + w.WriteSymbol(isInter, this.intraInter[context]); + } + + /// + /// Gets the cost of selecting one reference from the single-reference branch tree. + /// + /// The selected reference-frame label. + /// The neighboring reference counts indexed by reference-frame label. + /// The syntax cost in 1/512-bit units. + public int GetSingleReferenceCost( + Av1ReferenceFrameType referenceFrame, + ReadOnlySpan referenceCounts) + { + bool isBackward = referenceFrame >= Av1ReferenceFrameType.Backward; + int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts); + int rate = Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][0], isBackward ? 1 : 0); + if (isBackward) + { + bool isAlternate = referenceFrame == Av1ReferenceFrameType.Alternate; + context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts); + rate += Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][1], isAlternate ? 1 : 0); + if (isAlternate) + { + return rate; + } + + context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts); + return rate + Av1ProbabilityCost.GetSymbolCost( + this.singleReference[context][5], + referenceFrame == Av1ReferenceFrameType.Alternate2 ? 1 : 0); + } + + bool isLast3OrGolden = referenceFrame is Av1ReferenceFrameType.Last3 or Av1ReferenceFrameType.Golden; + context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts); + rate += Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][2], isLast3OrGolden ? 1 : 0); + if (isLast3OrGolden) + { + context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts); + return rate + Av1ProbabilityCost.GetSymbolCost( + this.singleReference[context][4], + referenceFrame == Av1ReferenceFrameType.Golden ? 1 : 0); + } + + context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts); + return rate + Av1ProbabilityCost.GetSymbolCost( + this.singleReference[context][3], + referenceFrame == Av1ReferenceFrameType.Last2 ? 1 : 0); + } + + /// + /// Writes one reference through the single-reference branch tree. + /// + /// The selected reference-frame label. + /// The neighboring reference counts indexed by reference-frame label. + public void WriteSingleReference( + Av1ReferenceFrameType referenceFrame, + ReadOnlySpan referenceCounts) + { + ref Av1SymbolWriter w = ref this.writer; + bool isBackward = referenceFrame >= Av1ReferenceFrameType.Backward; + int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts); + w.WriteSymbol(isBackward, this.singleReference[context][0]); + if (isBackward) + { + bool isAlternate = referenceFrame == Av1ReferenceFrameType.Alternate; + context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts); + w.WriteSymbol(isAlternate, this.singleReference[context][1]); + if (isAlternate) + { + return; + } + + context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts); + w.WriteSymbol( + referenceFrame == Av1ReferenceFrameType.Alternate2, + this.singleReference[context][5]); + + return; + } + + bool isLast3OrGolden = referenceFrame is Av1ReferenceFrameType.Last3 or Av1ReferenceFrameType.Golden; + context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts); + w.WriteSymbol(isLast3OrGolden, this.singleReference[context][2]); + if (isLast3OrGolden) + { + context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts); + w.WriteSymbol( + referenceFrame == Av1ReferenceFrameType.Golden, + this.singleReference[context][4]); + + return; + } + + context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts); + w.WriteSymbol( + referenceFrame == Av1ReferenceFrameType.Last2, + this.singleReference[context][3]); + } + /// /// Gets the current fixed-point cost of a directional angle-delta symbol. /// @@ -2087,40 +2414,4 @@ internal sealed class Av1SymbolEncoder : IDisposable colorContext, planeType); } - - /// - /// Owns the adaptive distributions used only by AV1 palette syntax. - /// - private sealed class PaletteEntropyContext - { - /// - /// Gets the luma palette-mode distributions. - /// - public Av1Distribution[][] YMode { get; } = Av1DefaultDistributions.PaletteYMode; - - /// - /// Gets the chroma palette-mode distributions. - /// - public Av1Distribution[] UvMode { get; } = Av1DefaultDistributions.PaletteUvMode; - - /// - /// Gets the luma palette-size distributions. - /// - public Av1Distribution[] YSize { get; } = Av1DefaultDistributions.PaletteYSize; - - /// - /// Gets the chroma palette-size distributions. - /// - public Av1Distribution[] UvSize { get; } = Av1DefaultDistributions.PaletteUvSize; - - /// - /// Gets the luma palette color-index distributions. - /// - public Av1Distribution[][] YColorIndex { get; } = Av1DefaultDistributions.PaletteYColorIndex; - - /// - /// Gets the chroma palette color-index distributions. - /// - public Av1Distribution[][] UvColorIndex { get; } = Av1DefaultDistributions.PaletteUvColorIndex; - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs index fda8597fa2..a8110cd755 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs @@ -12,6 +12,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; /// internal sealed class Av1SymbolWriter : IDisposable { + /// + /// The normalized range before the first symbol narrows the coding interval. + /// + private const uint InitialRange = 0x8000U; + + /// + /// The initial bit count that crosses the first byte-and-carry flush boundary after one output byte. + /// + private const int InitialCount = -9; + /// /// The lower endpoint of the current coding interval. /// @@ -20,7 +30,7 @@ internal sealed class Av1SymbolWriter : IDisposable /// /// The width of the current normalized coding interval. /// - private uint rng = 0x8000U; + private uint rng = InitialRange; /// /// The number of accumulated bits relative to the next byte-and-carry flush boundary. @@ -28,7 +38,7 @@ internal sealed class Av1SymbolWriter : IDisposable /// /// The initial value of -9 crosses zero after one output byte and its carry bit have accumulated. /// - private int cnt = -9; + private int cnt = InitialCount; /// /// The configuration that supplies output allocation. @@ -36,14 +46,19 @@ internal sealed class Av1SymbolWriter : IDisposable private readonly Configuration configuration; /// - /// The owner of the fixed output buffer supplied for this tile. + /// The owner of the fixed output buffer shared by consecutively encoded tiles. /// private readonly IMemoryOwner bufferOwner; + /// + /// The complete requested output allocation, including every consecutively encoded tile. + /// + private readonly Memory outputBuffer; + /// /// The requested output range, excluding any excess capacity returned by a pooling allocator. /// - private readonly Memory buffer; + private Memory buffer; /// /// Indicates whether encoded symbols adapt their distributions. @@ -65,10 +80,29 @@ internal sealed class Av1SymbolWriter : IDisposable { this.configuration = configuration; this.bufferOwner = configuration.MemoryAllocator.Allocate(bufferLength); - this.buffer = this.bufferOwner.Memory[..bufferLength]; + this.outputBuffer = this.bufferOwner.Memory[..bufferLength]; + this.buffer = this.outputBuffer; this.updateCdf = updateCdf; } + /// + /// Restores the initial range-coder state while retaining the bounded output allocation. + /// + public void Reset() => this.Reset(0); + + /// + /// Restores the initial range-coder state and begins writing at an offset in the retained output allocation. + /// + /// The first byte available to the next range-coded tile. + public void Reset(int outputOffset) + { + this.buffer = this.outputBuffer[outputOffset..]; + this.low = 0; + this.rng = InitialRange; + this.cnt = InitialCount; + this.position = 0; + } + /// /// Releases the tile output buffer. /// @@ -147,13 +181,20 @@ internal sealed class Av1SymbolWriter : IDisposable /// Finalizes the range-coded sequence and exposes its encoded prefix without copying. /// /// The number of encoded bytes in the returned memory. - /// The encoded prefix, valid until this writer is disposed. + /// The encoded prefix, valid until this writer is reset or disposed. public ReadOnlyMemory Exit(out int length) { length = this.FinalizeRange(); return this.buffer[..length]; } + /// + /// Exposes a prefix containing consecutively encoded tiles without copying their bytes. + /// + /// The number of bytes in the prefix. + /// The encoded prefix, valid until this writer is reset to offset zero or disposed. + public ReadOnlyMemory GetOutput(int length) => this.outputBuffer[..length]; + /// /// Terminates the range-coded sequence in the current output allocation. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs index 532b6f2e67..5e9d7f6030 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs @@ -20,6 +20,51 @@ internal struct Av1GlobalMotionParameters /// public const int ModelScale = 1 << ModelPrecisionBits; + /// + /// The initial finite-subexponential group width used by every global-motion parameter. + /// + public const int SubexponentialGroupBitCount = 3; + + /// + /// The finite signed-domain size parameter for coded affine coefficients. + /// + public const int AlphaValueMagnitude = (1 << 12) + 1; + + /// + /// The number of fractional bits carried by coded affine coefficients. + /// + public const int AlphaPrecisionBits = 15; + + /// + /// The precision increase from a coded affine coefficient to the stored matrix. + /// + public const int AlphaPrecisionDifference = ModelPrecisionBits - AlphaPrecisionBits; + + /// + /// The scale factor that restores a coded affine coefficient to the stored matrix precision. + /// + public const int AlphaDecodeFactor = 1 << AlphaPrecisionDifference; + + /// + /// The signed magnitude bit count of a general affine model's translation components. + /// + public const int AbsoluteTranslationBits = 12; + + /// + /// The signed magnitude bit count of a translation-only model before precision adjustment. + /// + public const int AbsoluteTranslationOnlyBits = 9; + + /// + /// The number of fractional bits carried by general affine translation components. + /// + public const int TranslationPrecisionBits = 6; + + /// + /// The number of fractional bits carried by translation-only components. + /// + public const int TranslationOnlyPrecisionBits = 3; + /// /// The number of low-order bits removed from the derived shear parameters. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs index 92d6009faf..2854d1562f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs @@ -13,6 +13,16 @@ internal readonly struct Av1MotionVector : IEquatable /// public const int MaximumTemporalDistance = 31; + /// + /// The number of fractional bits used by AV1 motion-vector components. + /// + public const int SubpixelBits = 3; + + /// + /// The number of motion-vector units in one full pixel. + /// + public const int SubpixelScale = 1 << SubpixelBits; + /// /// The reserved lower endpoint of the signed AV1 motion-vector domain. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs index 20209cc36b..758d3563e8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Diagnostics.CodeAnalysis; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -10,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Motion; /// /// Derives the weighted AV1 reference-motion-vector candidates for one inter block. /// -internal sealed class Av1ReferenceMotionVectors +internal struct Av1ReferenceMotionVectors { /// /// The number of surrounding mode-information rows and columns examined by the spatial search. @@ -80,16 +81,19 @@ internal sealed class Av1ReferenceMotionVectors /// /// Gets the derived candidates in normative nearest-region then outer-region order. /// + [UnscopedRef] public ReadOnlySpan Candidates => this.candidates[..this.Count]; /// /// Gets the secondary vectors corresponding to for a compound block. /// + [UnscopedRef] public ReadOnlySpan CompoundCandidates => this.compoundCandidates[..this.Count]; /// /// Gets the accumulated weight corresponding to each entry in . /// + [UnscopedRef] public ReadOnlySpan Weights => this.weights[..this.Count]; /// @@ -116,11 +120,67 @@ internal sealed class Av1ReferenceMotionVectors Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame = Av1ReferenceFrameType.None) { - Av1BlockSize blockSize = partitionInfo.ModeInfo.BlockSize; + ReferenceContext context = new(ref partitionInfo, sequenceHeader.SuperblockModeInfoSize, frameInfo); + this.Build( + in context, + tileInfo, + sequenceHeader, + frameHeader, + referenceFrame, + secondaryReferenceFrame); + } + + /// + /// Derives the spatial single-reference motion-vector candidates for an encoder block. + /// + /// The encoder picture state containing previously coded neighbors. + /// The current block geometry and tile availability. + /// The current block origin in 4x4 mode-information units. + /// The current coding-block size. + /// The partition that produced the current block. + /// The sequence-level superblock configuration. + /// The frame-level global-motion and motion-vector precision configuration. + /// The canonical inter reference selected for the current block. + public void Build( + Av1PictureControlSet picture, + Av1MacroBlockD macroBlock, + Point modeInfoPosition, + Av1BlockSize blockSize, + Av1PartitionType partitionType, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1ReferenceFrameType referenceFrame) + { + ReferenceContext context = new( + picture, + macroBlock, + modeInfoPosition, + blockSize, + partitionType, + sequenceHeader.SuperblockModeInfoSize); + + this.Build( + in context, + macroBlock.Tile, + sequenceHeader, + frameHeader, + referenceFrame, + Av1ReferenceFrameType.None); + } + + private void Build( + in ReferenceContext context, + Av1TileInfo tileInfo, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1ReferenceFrameType referenceFrame, + Av1ReferenceFrameType secondaryReferenceFrame) + { + Av1BlockSize blockSize = context.BlockSize; int width = blockSize.Get4x4WideCount(); int height = blockSize.Get4x4HighCount(); - int row = partitionInfo.RowIndex; - int column = partitionInfo.ColumnIndex; + int row = context.RowIndex; + int column = context.ColumnIndex; int rowAdjustment = height < 2 && (row & 1) != 0 ? 1 : 0; int columnAdjustment = width < 2 && (column & 1) != 0 ? 1 : 0; int maximumRowOffset = 0; @@ -129,13 +189,13 @@ internal sealed class Av1ReferenceMotionVectors this.Count = 0; this.ModeContext = 0; - if (partitionInfo.AvailableAbove) + if (context.AvailableAbove) { maximumRowOffset = height < 2 ? -4 + rowAdjustment : -(ReferenceSearchDistance << 1) + rowAdjustment; maximumRowOffset = Math.Clamp(maximumRowOffset, tileInfo.ModeInfoRowStart - row, tileInfo.ModeInfoRowEnd - row - 1); } - if (partitionInfo.AvailableLeft) + if (context.AvailableLeft) { maximumColumnOffset = width < 2 ? -4 + columnAdjustment : -(ReferenceSearchDistance << 1) + columnAdjustment; maximumColumnOffset = Math.Clamp(maximumColumnOffset, tileInfo.ModeInfoColumnStart - column, tileInfo.ModeInfoColumnEnd - column - 1); @@ -171,7 +231,7 @@ internal sealed class Av1ReferenceMotionVectors if (Math.Abs(maximumRowOffset) >= 1) { this.ScanRow( - ref partitionInfo, + in context, referenceFrame, secondaryReferenceFrame, in globalMotion, @@ -188,7 +248,7 @@ internal sealed class Av1ReferenceMotionVectors if (Math.Abs(maximumColumnOffset) >= 1) { this.ScanColumn( - ref partitionInfo, + in context, referenceFrame, secondaryReferenceFrame, in globalMotion, @@ -202,10 +262,10 @@ internal sealed class Av1ReferenceMotionVectors ref processedColumns); } - if (partitionInfo.HasTopRight(sequenceHeader.SuperblockModeInfoSize)) + if (context.HasTopRight) { this.AddSpatialBlock( - ref partitionInfo, + in context, tileInfo, referenceFrame, secondaryReferenceFrame, @@ -229,9 +289,8 @@ internal sealed class Av1ReferenceMotionVectors if (frameHeader.UseReferenceFrameMotionVectors) { this.AddTemporalCandidates( - ref partitionInfo, + in context, tileInfo, - frameInfo, sequenceHeader.OrderHintInfo, frameHeader, referenceFrame, @@ -245,7 +304,7 @@ internal sealed class Av1ReferenceMotionVectors // The top-left block begins the lower-priority outer region. Candidate deduplication still spans both // regions, while the two independent stable sorts below preserve the normative nearest-before-outer order. this.AddSpatialBlock( - ref partitionInfo, + in context, tileInfo, referenceFrame, secondaryReferenceFrame, @@ -265,7 +324,7 @@ internal sealed class Av1ReferenceMotionVectors if (Math.Abs(rowOffset) <= Math.Abs(maximumRowOffset) && Math.Abs(rowOffset) > processedRows) { this.ScanRow( - ref partitionInfo, + in context, referenceFrame, secondaryReferenceFrame, in globalMotion, @@ -282,7 +341,7 @@ internal sealed class Av1ReferenceMotionVectors if (Math.Abs(columnOffset) <= Math.Abs(maximumColumnOffset) && Math.Abs(columnOffset) > processedColumns) { this.ScanColumn( - ref partitionInfo, + in context, referenceFrame, secondaryReferenceFrame, in globalMotion, @@ -323,8 +382,7 @@ internal sealed class Av1ReferenceMotionVectors if (this.Count < 2) { this.ExtendCompoundStack( - ref partitionInfo, - frameInfo, + in context, referenceFrame, secondaryReferenceFrame, globalMotionVector, @@ -340,15 +398,15 @@ internal sealed class Av1ReferenceMotionVectors // Differing reference sign biases are reversed before either candidate enters that pair. for (int index = 0; Math.Abs(maximumRowOffset) >= 1 && index < extensionLength && this.Count < 2;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column + index, row - 1)); - this.AddExtensionCandidate(candidate, frameInfo, referenceFrame); + ReferenceBlock candidate = context.GetModeInfoAt(new Point(column + index, row - 1)); + this.AddExtensionCandidate(candidate, in context, referenceFrame); index += candidate.BlockSize.Get4x4WideCount(); } for (int index = 0; Math.Abs(maximumColumnOffset) >= 1 && index < extensionLength && this.Count < 2;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column - 1, row + index)); - this.AddExtensionCandidate(candidate, frameInfo, referenceFrame); + ReferenceBlock candidate = context.GetModeInfoAt(new Point(column - 1, row + index)); + this.AddExtensionCandidate(candidate, in context, referenceFrame); index += candidate.BlockSize.Get4x4HighCount(); } } @@ -358,20 +416,20 @@ internal sealed class Av1ReferenceMotionVectors this.candidates[index] = this.candidates[index].ClampReference( blockSize.GetWidth(), blockSize.GetHeight(), - partitionInfo.ModeBlockToLeftEdge, - partitionInfo.ModeBlockToRightEdge, - partitionInfo.ModeBlockToTopEdge, - partitionInfo.ModeBlockToBottomEdge); + context.ModeBlockToLeftEdge, + context.ModeBlockToRightEdge, + context.ModeBlockToTopEdge, + context.ModeBlockToBottomEdge); if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra) { this.compoundCandidates[index] = this.compoundCandidates[index].ClampReference( blockSize.GetWidth(), blockSize.GetHeight(), - partitionInfo.ModeBlockToLeftEdge, - partitionInfo.ModeBlockToRightEdge, - partitionInfo.ModeBlockToTopEdge, - partitionInfo.ModeBlockToBottomEdge); + context.ModeBlockToLeftEdge, + context.ModeBlockToRightEdge, + context.ModeBlockToTopEdge, + context.ModeBlockToBottomEdge); } } @@ -452,7 +510,7 @@ internal sealed class Av1ReferenceMotionVectors /// /// Scans one spatial row using AV1's block-size-dependent steps and weights. /// - /// The current block geometry and frame-wide mode map. + /// The current block geometry and frame-wide mode map. /// The canonical inter reference selected for the current block. /// The secondary compound reference, or . /// The selected reference's global-motion model. @@ -465,7 +523,7 @@ internal sealed class Av1ReferenceMotionVectors /// Accumulates matching neighbors whose inter mode contains a new vector. /// Receives the spatial depth covered by block-height weighting. private void ScanRow( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, in Av1GlobalMotionParameters globalMotion, @@ -478,13 +536,13 @@ internal sealed class Av1ReferenceMotionVectors ref int newMotionVectorCount, ref int processedRows) { - int width = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); - int end = Math.Min(partitionInfo.GetMaxBlockWide(partitionInfo.ModeInfo.BlockSize, false), MaximumSearchBlockSize); + int width = context.BlockSize.Get4x4WideCount(); + int end = Math.Min(context.GetMaxBlockWide(), MaximumSearchBlockSize); int columnOffset = 0; if (Math.Abs(rowOffset) > 1) { columnOffset = 1; - if ((partitionInfo.ColumnIndex & 1) != 0 && width < 2) + if ((context.ColumnIndex & 1) != 0 && width < 2) { columnOffset--; } @@ -495,8 +553,8 @@ internal sealed class Av1ReferenceMotionVectors bool useFourUnitStep = width >= MaximumSearchBlockSize; for (int index = 0; index < end;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt( - new Point(partitionInfo.ColumnIndex + columnOffset + index, partitionInfo.RowIndex + rowOffset)); + ReferenceBlock candidate = context.GetModeInfoAt( + new Point(context.ColumnIndex + columnOffset + index, context.RowIndex + rowOffset)); int candidateWidth = candidate.BlockSize.Get4x4WideCount(); int length = Math.Min(width, candidateWidth); @@ -536,7 +594,7 @@ internal sealed class Av1ReferenceMotionVectors /// /// Scans one spatial column using AV1's block-size-dependent steps and weights. /// - /// The current block geometry and frame-wide mode map. + /// The current block geometry and frame-wide mode map. /// The canonical inter reference selected for the current block. /// The secondary compound reference, or . /// The selected reference's global-motion model. @@ -549,7 +607,7 @@ internal sealed class Av1ReferenceMotionVectors /// Accumulates matching neighbors whose inter mode contains a new vector. /// Receives the spatial depth covered by block-width weighting. private void ScanColumn( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, in Av1GlobalMotionParameters globalMotion, @@ -562,13 +620,13 @@ internal sealed class Av1ReferenceMotionVectors ref int newMotionVectorCount, ref int processedColumns) { - int height = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); - int end = Math.Min(partitionInfo.GetMaxBlockHigh(partitionInfo.ModeInfo.BlockSize, false), MaximumSearchBlockSize); + int height = context.BlockSize.Get4x4HighCount(); + int end = Math.Min(context.GetMaxBlockHigh(), MaximumSearchBlockSize); int rowOffset = 0; if (Math.Abs(columnOffset) > 1) { rowOffset = 1; - if ((partitionInfo.RowIndex & 1) != 0 && height < 2) + if ((context.RowIndex & 1) != 0 && height < 2) { rowOffset--; } @@ -579,8 +637,8 @@ internal sealed class Av1ReferenceMotionVectors bool useFourUnitStep = height >= MaximumSearchBlockSize; for (int index = 0; index < end;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt( - new Point(partitionInfo.ColumnIndex + columnOffset, partitionInfo.RowIndex + rowOffset + index)); + ReferenceBlock candidate = context.GetModeInfoAt( + new Point(context.ColumnIndex + columnOffset, context.RowIndex + rowOffset + index)); int candidateHeight = candidate.BlockSize.Get4x4HighCount(); int length = Math.Min(height, candidateHeight); @@ -620,7 +678,7 @@ internal sealed class Av1ReferenceMotionVectors /// /// Adds the candidate at one tile-relative spatial search position. /// - /// The current block geometry and frame-wide mode map. + /// The current block geometry and frame-wide mode map. /// The active tile boundaries. /// The canonical inter reference selected for the current block. /// The secondary compound reference, or . @@ -633,7 +691,7 @@ internal sealed class Av1ReferenceMotionVectors /// Accumulates matching reference labels at the search position. /// Accumulates matching neighbors whose inter mode contains a new vector. private void AddSpatialBlock( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1TileInfo tileInfo, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, @@ -646,15 +704,15 @@ internal sealed class Av1ReferenceMotionVectors ref int referenceMatchCount, ref int newMotionVectorCount) { - int row = partitionInfo.RowIndex + rowOffset; - int column = partitionInfo.ColumnIndex + columnOffset; + int row = context.RowIndex + rowOffset; + int column = context.ColumnIndex + columnOffset; if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd || column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd) { return; } - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column, row)); + ReferenceBlock candidate = context.GetModeInfoAt(new Point(column, row)); this.AddCandidate( candidate, referenceFrame, @@ -682,7 +740,7 @@ internal sealed class Av1ReferenceMotionVectors /// Accumulates matching reference labels in the active scan direction. /// Accumulates matching neighbors whose inter mode contains a new vector. private void AddCandidate( - Av1BlockModeInfo candidate, + ReferenceBlock candidate, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, in Av1GlobalMotionParameters globalMotion, @@ -693,16 +751,15 @@ internal sealed class Av1ReferenceMotionVectors ref int referenceMatchCount, ref int newMotionVectorCount) { - Span candidateReferences = candidate.ReferenceFrames; - if (candidateReferences[0] <= Av1ReferenceFrameType.Intra) + if (candidate.GetReferenceFrame(0) <= Av1ReferenceFrameType.Intra) { return; } - Span candidateMotionVectors = candidate.MotionVectors; if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra) { - if (candidateReferences[0] != referenceFrame || candidateReferences[1] != secondaryReferenceFrame) + if (candidate.GetReferenceFrame(0) != referenceFrame || + candidate.GetReferenceFrame(1) != secondaryReferenceFrame) { return; } @@ -719,11 +776,11 @@ internal sealed class Av1ReferenceMotionVectors Av1MotionVector primaryMotionVector = usePrimaryGlobalMotion ? globalMotionVector - : candidateMotionVectors[0]; + : candidate.GetMotionVector(0); Av1MotionVector secondaryMotionVector = useSecondaryGlobalMotion ? secondaryGlobalMotionVector - : candidateMotionVectors[1]; + : candidate.GetMotionVector(1); this.AddUnique(primaryMotionVector, secondaryMotionVector, weight); if (UsesNewMotionVector(candidate.YMode)) @@ -737,7 +794,7 @@ internal sealed class Av1ReferenceMotionVectors for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++) { - if (candidateReferences[referenceIndex] != referenceFrame) + if (candidate.GetReferenceFrame(referenceIndex) != referenceFrame) { continue; } @@ -752,7 +809,7 @@ internal sealed class Av1ReferenceMotionVectors Av1MotionVector motionVector = useGlobalMotion ? globalMotionVector - : candidateMotionVectors[referenceIndex]; + : candidate.GetMotionVector(referenceIndex); this.AddUnique(motionVector, weight); @@ -770,9 +827,8 @@ internal sealed class Av1ReferenceMotionVectors /// /// Adds projected temporal candidates over the current block and its permitted extension positions. /// - /// The current block geometry. + /// The current block geometry and decoder temporal state. /// The active tile boundaries. - /// The projected temporal motion field. /// The sequence modulo order-hint configuration. /// The frame-level motion-vector precision configuration. /// The canonical inter reference selected for the current block. @@ -780,9 +836,8 @@ internal sealed class Av1ReferenceMotionVectors /// The selected reference's global-motion vector at the current block. /// The secondary reference's global-motion vector at the current block. private void AddTemporalCandidates( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1TileInfo tileInfo, - Av1FrameInfo frameInfo, ObuOrderHintInfo orderHintInfo, ObuFrameHeader frameHeader, Av1ReferenceFrameType referenceFrame, @@ -790,8 +845,8 @@ internal sealed class Av1ReferenceMotionVectors Av1MotionVector globalMotionVector, Av1MotionVector secondaryGlobalMotionVector) { - int width = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); - int height = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); + int width = context.BlockSize.Get4x4WideCount(); + int height = context.BlockSize.Get4x4HighCount(); int verticalOffset = Math.Max(2, height); int horizontalOffset = Math.Max(2, width); int blockRowEnd = Math.Min(height, MaximumSearchBlockSize); @@ -805,9 +860,8 @@ internal sealed class Av1ReferenceMotionVectors for (int blockColumn = 0; blockColumn < blockColumnEnd; blockColumn += columnStep) { bool available = this.AddTemporalCandidate( - ref partitionInfo, + in context, tileInfo, - frameInfo, orderHintInfo, frameHeader, referenceFrame, @@ -838,9 +892,8 @@ internal sealed class Av1ReferenceMotionVectors // These three positions extend the temporal search below-left, below-right, and above-right. The 64x64 // boundary test is normative even when the sequence uses 128x128 superblocks. this.AddTemporalExtension( - ref partitionInfo, + in context, tileInfo, - frameInfo, orderHintInfo, frameHeader, referenceFrame, @@ -851,9 +904,8 @@ internal sealed class Av1ReferenceMotionVectors -2); this.AddTemporalExtension( - ref partitionInfo, + in context, tileInfo, - frameInfo, orderHintInfo, frameHeader, referenceFrame, @@ -864,9 +916,8 @@ internal sealed class Av1ReferenceMotionVectors horizontalOffset); this.AddTemporalExtension( - ref partitionInfo, + in context, tileInfo, - frameInfo, orderHintInfo, frameHeader, referenceFrame, @@ -880,9 +931,8 @@ internal sealed class Av1ReferenceMotionVectors /// /// Adds one optional temporal extension candidate after applying the normative 64x64 boundary rule. /// - /// The current block geometry. + /// The current block geometry and decoder temporal state. /// The active tile boundaries. - /// The projected temporal motion field. /// The sequence modulo order-hint configuration. /// The frame-level motion-vector precision configuration. /// The canonical inter reference selected for the current block. @@ -892,9 +942,8 @@ internal sealed class Av1ReferenceMotionVectors /// The temporal sample row relative to the current block in 4x4 units. /// The temporal sample column relative to the current block in 4x4 units. private void AddTemporalExtension( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1TileInfo tileInfo, - Av1FrameInfo frameInfo, ObuOrderHintInfo orderHintInfo, ObuFrameHeader frameHeader, Av1ReferenceFrameType referenceFrame, @@ -904,8 +953,8 @@ internal sealed class Av1ReferenceMotionVectors int blockRow, int blockColumn) { - int rowWithinBlock64 = partitionInfo.RowIndex & (MaximumSearchBlockSize - 1); - int columnWithinBlock64 = partitionInfo.ColumnIndex & (MaximumSearchBlockSize - 1); + int rowWithinBlock64 = context.RowIndex & (MaximumSearchBlockSize - 1); + int columnWithinBlock64 = context.ColumnIndex & (MaximumSearchBlockSize - 1); if (rowWithinBlock64 + blockRow < 0 || rowWithinBlock64 + blockRow >= MaximumSearchBlockSize || columnWithinBlock64 + blockColumn < 0 || columnWithinBlock64 + blockColumn >= MaximumSearchBlockSize) { @@ -913,9 +962,8 @@ internal sealed class Av1ReferenceMotionVectors } _ = this.AddTemporalCandidate( - ref partitionInfo, + in context, tileInfo, - frameInfo, orderHintInfo, frameHeader, referenceFrame, @@ -929,9 +977,8 @@ internal sealed class Av1ReferenceMotionVectors /// /// Projects and accumulates one temporal motion-field sample. /// - /// The current block geometry. + /// The current block geometry and decoder temporal state. /// The active tile boundaries. - /// The projected temporal motion field. /// The sequence modulo order-hint configuration. /// The frame-level motion-vector precision configuration. /// The canonical inter reference selected for the current block. @@ -942,9 +989,8 @@ internal sealed class Av1ReferenceMotionVectors /// The temporal sample column relative to the current block in 4x4 units. /// when the projected motion field covers the requested position. private bool AddTemporalCandidate( - ref Av1PartitionInfo partitionInfo, + in ReferenceContext context, Av1TileInfo tileInfo, - Av1FrameInfo frameInfo, ObuOrderHintInfo orderHintInfo, ObuFrameHeader frameHeader, Av1ReferenceFrameType referenceFrame, @@ -954,17 +1000,17 @@ internal sealed class Av1ReferenceMotionVectors int blockRow, int blockColumn) { - int rowOffset = (partitionInfo.RowIndex & 1) != 0 ? blockRow : blockRow + 1; - int columnOffset = (partitionInfo.ColumnIndex & 1) != 0 ? blockColumn : blockColumn + 1; - int row = partitionInfo.RowIndex + rowOffset; - int column = partitionInfo.ColumnIndex + columnOffset; + int rowOffset = (context.RowIndex & 1) != 0 ? blockRow : blockRow + 1; + int columnOffset = (context.ColumnIndex & 1) != 0 ? blockColumn : blockColumn + 1; + int row = context.RowIndex + rowOffset; + int column = context.ColumnIndex + columnOffset; if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd || column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd) { return false; } - if (!frameInfo.TryGetProjectedTemporalMotionVector( + if (!context.TryGetProjectedTemporalMotionVector( row, column, referenceFrame, @@ -978,7 +1024,7 @@ internal sealed class Av1ReferenceMotionVectors Av1MotionVector secondaryMotionVector = default; if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra && - !frameInfo.TryGetProjectedTemporalMotionVector( + !context.TryGetProjectedTemporalMotionVector( row, column, secondaryReferenceFrame, @@ -1018,27 +1064,25 @@ internal sealed class Av1ReferenceMotionVectors /// Extends a short stack with inter vectors from a neighboring block, correcting their temporal direction. /// /// The decoded neighboring block. - /// The reference-side classification for the current frame. + /// The current block geometry and decoder reference classification. /// The canonical inter reference selected for the current block. private void AddExtensionCandidate( - Av1BlockModeInfo candidate, - Av1FrameInfo frameInfo, + ReferenceBlock candidate, + in ReferenceContext context, Av1ReferenceFrameType referenceFrame) { - Span candidateReferences = candidate.ReferenceFrames; - Span candidateMotionVectors = candidate.MotionVectors; - bool targetSignBias = frameInfo.IsReferenceSignBiased(referenceFrame); + bool targetSignBias = context.IsReferenceSignBiased(referenceFrame); for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++) { - Av1ReferenceFrameType candidateReference = candidateReferences[referenceIndex]; + Av1ReferenceFrameType candidateReference = candidate.GetReferenceFrame(referenceIndex); if (candidateReference <= Av1ReferenceFrameType.Intra) { continue; } - Av1MotionVector motionVector = candidateMotionVectors[referenceIndex]; - if (frameInfo.IsReferenceSignBiased(candidateReference) != targetSignBias) + Av1MotionVector motionVector = candidate.GetMotionVector(referenceIndex); + if (context.IsReferenceSignBiased(candidateReference) != targetSignBias) { motionVector = new Av1MotionVector(-motionVector.Row, -motionVector.Column); } @@ -1067,8 +1111,7 @@ internal sealed class Av1ReferenceMotionVectors /// Extends a short compound stack from the immediate above and left blocks. /// private void ExtendCompoundStack( - ref Av1PartitionInfo partitionInfo, - Av1FrameInfo frameInfo, + in ReferenceContext context, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, Av1MotionVector globalMotionVector, @@ -1085,15 +1128,15 @@ internal sealed class Av1ReferenceMotionVectors int secondaryExactCount = 0; int primaryDifferentCount = 0; int secondaryDifferentCount = 0; - int row = partitionInfo.RowIndex; - int column = partitionInfo.ColumnIndex; + int row = context.RowIndex; + int column = context.ColumnIndex; for (int index = 0; hasAbove && index < extensionLength;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column + index, row - 1)); + ReferenceBlock candidate = context.GetModeInfoAt(new Point(column + index, row - 1)); CollectCompoundExtensionCandidate( candidate, - frameInfo, + in context, referenceFrame, secondaryReferenceFrame, ref primaryExact, @@ -1110,10 +1153,10 @@ internal sealed class Av1ReferenceMotionVectors for (int index = 0; hasLeft && index < extensionLength;) { - Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column - 1, row + index)); + ReferenceBlock candidate = context.GetModeInfoAt(new Point(column - 1, row + index)); CollectCompoundExtensionCandidate( candidate, - frameInfo, + in context, referenceFrame, secondaryReferenceFrame, ref primaryExact, @@ -1168,8 +1211,8 @@ internal sealed class Av1ReferenceMotionVectors /// Collects exact-reference and temporal-direction-corrected fallback vectors from one neighboring block. /// private static void CollectCompoundExtensionCandidate( - Av1BlockModeInfo candidate, - Av1FrameInfo frameInfo, + ReferenceBlock candidate, + in ReferenceContext context, Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType secondaryReferenceFrame, ref InlineArray2 primaryExact, @@ -1181,13 +1224,10 @@ internal sealed class Av1ReferenceMotionVectors ref InlineArray2 secondaryDifferent, ref int secondaryDifferentCount) { - Span candidateReferences = candidate.ReferenceFrames; - Span candidateMotionVectors = candidate.MotionVectors; - for (int candidateIndex = 0; candidateIndex < 2; candidateIndex++) { - Av1ReferenceFrameType candidateReference = candidateReferences[candidateIndex]; - Av1MotionVector candidateMotionVector = candidateMotionVectors[candidateIndex]; + Av1ReferenceFrameType candidateReference = candidate.GetReferenceFrame(candidateIndex); + Av1MotionVector candidateMotionVector = candidate.GetMotionVector(candidateIndex); for (int targetIndex = 0; targetIndex < 2; targetIndex++) { @@ -1223,7 +1263,7 @@ internal sealed class Av1ReferenceMotionVectors } Av1MotionVector differentMotionVector = candidateMotionVector; - if (frameInfo.IsReferenceSignBiased(candidateReference) != frameInfo.IsReferenceSignBiased(targetReference)) + if (context.IsReferenceSignBiased(candidateReference) != context.IsReferenceSignBiased(targetReference)) { differentMotionVector = new Av1MotionVector(-differentMotionVector.Row, -differentMotionVector.Column); } @@ -1364,4 +1404,195 @@ internal sealed class Av1ReferenceMotionVectors Av1PredictionMode.NewNearestMotionVector or Av1PredictionMode.NearNewMotionVector or Av1PredictionMode.NewNearMotionVector; + + /// + /// Provides one allocation-free view over decoder or encoder mode-information storage. + /// + private readonly struct ReferenceContext + { + private readonly Av1SuperblockInfo decodedSuperblock; + private readonly Av1PictureControlSet? encodedPicture; + private readonly Av1FrameInfo? decodedFrame; + + public ReferenceContext(ref Av1PartitionInfo partitionInfo, int superblockModeInfoSize, Av1FrameInfo frameInfo) + { + this.decodedSuperblock = partitionInfo.SuperblockInfo; + this.encodedPicture = null; + this.decodedFrame = frameInfo; + this.BlockSize = partitionInfo.ModeInfo.BlockSize; + this.RowIndex = partitionInfo.RowIndex; + this.ColumnIndex = partitionInfo.ColumnIndex; + this.AvailableAbove = partitionInfo.AvailableAbove; + this.AvailableLeft = partitionInfo.AvailableLeft; + this.ModeBlockToLeftEdge = partitionInfo.ModeBlockToLeftEdge; + this.ModeBlockToRightEdge = partitionInfo.ModeBlockToRightEdge; + this.ModeBlockToTopEdge = partitionInfo.ModeBlockToTopEdge; + this.ModeBlockToBottomEdge = partitionInfo.ModeBlockToBottomEdge; + this.HasTopRight = partitionInfo.HasTopRight(superblockModeInfoSize); + } + + public ReferenceContext( + Av1PictureControlSet picture, + Av1MacroBlockD macroBlock, + Point modeInfoPosition, + Av1BlockSize blockSize, + Av1PartitionType partitionType, + int superblockModeInfoSize) + { + this.decodedSuperblock = default; + this.encodedPicture = picture; + this.decodedFrame = null; + this.BlockSize = blockSize; + this.RowIndex = modeInfoPosition.Y; + this.ColumnIndex = modeInfoPosition.X; + this.AvailableAbove = macroBlock.IsUpAvailable; + this.AvailableLeft = macroBlock.IsLeftAvailable; + this.ModeBlockToLeftEdge = macroBlock.ToLeftEdge; + this.ModeBlockToRightEdge = macroBlock.ToRightEdge; + this.ModeBlockToTopEdge = macroBlock.ToTopEdge; + this.ModeBlockToBottomEdge = macroBlock.ToBottomEdge; + this.HasTopRight = Av1PartitionInfo.HasTopRight( + blockSize, + partitionType, + modeInfoPosition.Y, + modeInfoPosition.X, + superblockModeInfoSize); + } + + public Av1BlockSize BlockSize { get; } + + public int RowIndex { get; } + + public int ColumnIndex { get; } + + public bool AvailableAbove { get; } + + public bool AvailableLeft { get; } + + public int ModeBlockToLeftEdge { get; } + + public int ModeBlockToRightEdge { get; } + + public int ModeBlockToTopEdge { get; } + + public int ModeBlockToBottomEdge { get; } + + public bool HasTopRight { get; } + + public int GetMaxBlockWide() + { + int width = this.BlockSize.GetWidth(); + if (this.ModeBlockToRightEdge < 0) + { + width += this.ModeBlockToRightEdge >> 3; + } + + return width >> Av1Constants.ModeInfoSizeLog2; + } + + public int GetMaxBlockHigh() + { + int height = this.BlockSize.GetHeight(); + if (this.ModeBlockToBottomEdge < 0) + { + height += this.ModeBlockToBottomEdge >> 3; + } + + return height >> Av1Constants.ModeInfoSizeLog2; + } + + public ReferenceBlock GetModeInfoAt(Point position) + { + Av1PictureControlSet? picture = this.encodedPicture; + if (picture is not null) + { + Av1MacroBlockModeInfo encodedModeInfo = picture.GetFromModeInfoGrid(position); + return new ReferenceBlock( + encodedModeInfo.Block.BlockSize, + encodedModeInfo.Block.Mode, + encodedModeInfo.Block.ReferenceFrame, + Av1ReferenceFrameType.None, + picture.GetDisplacementVector(position), + default); + } + + Av1BlockModeInfo decodedModeInfo = this.decodedSuperblock.GetModeInfoAt(position); + return new ReferenceBlock( + decodedModeInfo.BlockSize, + decodedModeInfo.YMode, + decodedModeInfo.ReferenceFrames[0], + decodedModeInfo.ReferenceFrames[1], + decodedModeInfo.MotionVectors[0], + decodedModeInfo.MotionVectors[1]); + } + + public bool IsReferenceSignBiased(Av1ReferenceFrameType referenceFrame) + { + Av1FrameInfo? frameInfo = this.decodedFrame; + return frameInfo is not null && frameInfo.IsReferenceSignBiased(referenceFrame); + } + + public bool TryGetProjectedTemporalMotionVector( + int row, + int column, + Av1ReferenceFrameType referenceFrame, + ObuOrderHintInfo orderHintInfo, + bool allowHighPrecisionMotionVector, + bool forceIntegerMotionVector, + out Av1MotionVector motionVector) + { + Av1FrameInfo? frameInfo = this.decodedFrame; + if (frameInfo is null) + { + motionVector = default; + return false; + } + + return frameInfo.TryGetProjectedTemporalMotionVector( + row, + column, + referenceFrame, + orderHintInfo, + allowHighPrecisionMotionVector, + forceIntegerMotionVector, + out motionVector); + } + } + + /// + /// Carries the neighboring mode fields consumed by reference-vector ranking. + /// + private readonly struct ReferenceBlock + { + private readonly Av1ReferenceFrameType primaryReferenceFrame; + private readonly Av1ReferenceFrameType secondaryReferenceFrame; + private readonly Av1MotionVector primaryMotionVector; + private readonly Av1MotionVector secondaryMotionVector; + + public ReferenceBlock( + Av1BlockSize blockSize, + Av1PredictionMode mode, + Av1ReferenceFrameType primaryReferenceFrame, + Av1ReferenceFrameType secondaryReferenceFrame, + Av1MotionVector primaryMotionVector, + Av1MotionVector secondaryMotionVector) + { + this.BlockSize = blockSize; + this.YMode = mode; + this.primaryReferenceFrame = primaryReferenceFrame; + this.secondaryReferenceFrame = secondaryReferenceFrame; + this.primaryMotionVector = primaryMotionVector; + this.secondaryMotionVector = secondaryMotionVector; + } + + public Av1BlockSize BlockSize { get; } + + public Av1PredictionMode YMode { get; } + + public Av1ReferenceFrameType GetReferenceFrame(int index) + => index == 0 ? this.primaryReferenceFrame : this.secondaryReferenceFrame; + + public Av1MotionVector GetMotionVector(int index) + => index == 0 ? this.primaryMotionVector : this.secondaryMotionVector; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index ca496c9903..b18e367858 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -72,6 +72,15 @@ internal sealed class ObuFrameHeader /// public bool AllowHighPrecisionMotionVector { get; set; } + /// + /// Gets the component precision selected by the integer and high-precision frame flags. + /// + public Av1MotionVectorPrecision MotionVectorPrecision => this.ForceIntegerMotionVector + ? Av1MotionVectorPrecision.Integer + : this.AllowHighPrecisionMotionVector + ? Av1MotionVectorPrecision.EighthSample + : Av1MotionVectorPrecision.QuarterSample; + /// /// Gets or sets the frame-level interpolation filter used for inter prediction. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs index b861c1a58b..ced173c3fb 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs @@ -28,6 +28,21 @@ internal sealed class ObuOperatingPoint /// public bool IsDecoderModelInfoPresent { get; set; } + /// + /// Gets or sets the decoder-buffer delay measured in decoding ticks. + /// + public uint DecoderBufferDelay { get; set; } + + /// + /// Gets or sets the encoder-buffer delay measured in decoding ticks. + /// + public uint EncoderBufferDelay { get; set; } + + /// + /// Gets or sets a value indicating whether the operating point uses the low-delay decoding model. + /// + public bool LowDelayMode { get; set; } + /// /// Gets or sets a value indicating whether an initial display delay is present for this operating point. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 0931f8ac4e..de876efbaa 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -14,57 +14,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// internal sealed class ObuReader { - /// - /// The number of bits used to address one of AV1's eight reference-map slots. - /// - private const int ReferenceFrameIndexBits = 3; - - /// - /// The initial finite-subexponential group width used by every global-motion parameter. - /// - private const int GlobalMotionSubexponentialGroupBitCount = 3; - - /// - /// The finite signed-domain size parameter for coded global-motion affine coefficients. - /// - private const int GlobalMotionAlphaValueMagnitude = (1 << 12) + 1; - - /// - /// The number of fractional bits carried by coded global-motion affine coefficients. - /// - private const int GlobalMotionAlphaPrecisionBits = 15; - - /// - /// The precision increase from a coded affine coefficient to the stored global-motion matrix. - /// - private const int GlobalMotionAlphaPrecisionDifference = - Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionAlphaPrecisionBits; - - /// - /// The scale factor that restores a coded affine coefficient to the global-motion matrix precision. - /// - private const int GlobalMotionAlphaDecodeFactor = 1 << GlobalMotionAlphaPrecisionDifference; - - /// - /// The signed magnitude bit count of a general affine model's translation components. - /// - private const int GlobalMotionAbsoluteTranslationBits = 12; - - /// - /// The signed magnitude bit count of a translation-only model before precision adjustment. - /// - private const int GlobalMotionAbsoluteTranslationOnlyBits = 9; - - /// - /// The number of fractional bits carried by general affine translation components. - /// - private const int GlobalMotionTranslationPrecisionBits = 6; - - /// - /// The number of fractional bits carried by translation-only components. - /// - private const int GlobalMotionTranslationOnlyPrecisionBits = 3; - /// /// The zero-based sequence-header operating-point index selected by the container. /// @@ -951,7 +900,7 @@ internal sealed class ObuReader } sequenceHeader.InitialDisplayDelayPresentFlag = reader.ReadBoolean(); - int operatingPointsCnt = (int)reader.ReadLiteral(5) + 1; + int operatingPointsCnt = (int)reader.ReadLiteral(Av1Constants.OperatingPointCountBits) + 1; if (sequenceHeader.OperatingPoint.Length != operatingPointsCnt) { sequenceHeader.OperatingPoint = new ObuOperatingPoint[operatingPointsCnt]; @@ -961,15 +910,15 @@ internal sealed class ObuReader { sequenceHeader.OperatingPoint[i] = new ObuOperatingPoint { - Idc = reader.ReadLiteral(12), - SequenceLevelIndex = (int)reader.ReadLiteral(5) + Idc = reader.ReadLiteral(Av1Constants.OperatingPointIdcBits), + SequenceLevelIndex = (int)reader.ReadLiteral(Av1Constants.LevelBits) }; if (!IsValidSequenceLevel(sequenceHeader.OperatingPoint[i].SequenceLevelIndex)) { throw new InvalidImageContentException("The AV1 sequence header contains an undefined sequence-level index."); } - if (sequenceHeader.OperatingPoint[i].SequenceLevelIndex > 7) + if (sequenceHeader.OperatingPoint[i].SequenceLevelIndex >= Av1Constants.SequenceTierMinimumLevelIndex) { sequenceHeader.OperatingPoint[i].SequenceTier = (int)reader.ReadLiteral(1); } @@ -983,10 +932,13 @@ internal sealed class ObuReader sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent = reader.ReadBoolean(); if (sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent) { - // Operating-point delays affect scheduling rather than still-image reconstruction, but their - // syntax must be consumed so the following image dimensions remain bit aligned. + // Retain the scheduling values so the parsed sequence header can be written again without + // losing decoder-model state that is independent from pixel reconstruction. ObuDecoderModelInfo decoderModelInfo = sequenceHeader.GetDecoderModelInfo(); - ReadOperatingParametersInfo(ref reader, (int)decoderModelInfo.BufferDelayLength); + ReadOperatingParametersInfo( + ref reader, + (int)decoderModelInfo.BufferDelayLength, + sequenceHeader.OperatingPoint[i]); } } else @@ -1047,8 +999,8 @@ internal sealed class ObuReader sequenceHeader.EnableDualFilter = false; sequenceHeader.OrderHintInfo.EnableJointCompound = false; sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors = false; - sequenceHeader.ForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS - sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV + sequenceHeader.ForceScreenContentTools = Av1Constants.SelectScreenContentTools; + sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector; sequenceHeader.OrderHintInfo.OrderHintBits = 0; } else @@ -1072,7 +1024,7 @@ internal sealed class ObuReader bool seqChooseScreenContentTools = reader.ReadBoolean(); if (seqChooseScreenContentTools) { - sequenceHeader.ForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS + sequenceHeader.ForceScreenContentTools = Av1Constants.SelectScreenContentTools; } else { @@ -1084,7 +1036,7 @@ internal sealed class ObuReader bool seqChooseIntegerMv = reader.ReadBoolean(); if (seqChooseIntegerMv) { - sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV + sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector; } else { @@ -1093,7 +1045,7 @@ internal sealed class ObuReader } else { - sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV + sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector; } if (sequenceHeader.EnableOrderHint) @@ -1233,15 +1185,19 @@ internal sealed class ObuReader }; /// - /// Consumes operating-point buffer parameters that do not affect still-image reconstruction. + /// Reads the decoder-model parameters for one operating point. /// /// The reader positioned at the operating-point parameters. /// The bit width of each encoded buffer delay. - private static void ReadOperatingParametersInfo(ref Av1BitStreamReader reader, int bufferDelayLength) + /// The operating point that receives the decoded parameters. + private static void ReadOperatingParametersInfo( + ref Av1BitStreamReader reader, + int bufferDelayLength, + ObuOperatingPoint operatingPoint) { - _ = reader.ReadLiteral(bufferDelayLength); - _ = reader.ReadLiteral(bufferDelayLength); - _ = reader.ReadBoolean(); + operatingPoint.DecoderBufferDelay = reader.ReadLiteral(bufferDelayLength); + operatingPoint.EncoderBufferDelay = reader.ReadLiteral(bufferDelayLength); + operatingPoint.LowDelayMode = reader.ReadBoolean(); } /// @@ -1716,7 +1672,7 @@ internal sealed class ObuReader throw new InvalidImageContentException("An AV1 still picture cannot display a previously decoded frame."); } - frameHeader.FrameToShowMapIdx = reader.ReadLiteral(3); + frameHeader.FrameToShowMapIdx = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits); if (sequenceHeader.DecoderModelInfoPresentFlag && sequenceHeader.TimingInfo?.EqualPictureInterval == false) { @@ -1781,7 +1737,7 @@ internal sealed class ObuReader return; } - frameHeader.FrameType = (ObuFrameType)reader.ReadLiteral(2); + frameHeader.FrameType = (ObuFrameType)reader.ReadLiteral(Av1Constants.FrameTypeBits); frameHeader.ShowFrame = reader.ReadBoolean(); if (sequenceHeader.IsStillPicture && (frameHeader.FrameType != ObuFrameType.KeyFrame || !frameHeader.ShowFrame)) { @@ -2109,8 +2065,8 @@ internal sealed class ObuReader if (usesShortSignaling) { - uint lastFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); - uint goldenFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); + uint lastFrameIndex = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits); + uint goldenFrameIndex = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits); InlineArray8 slotOccupancyStorage = default; Span slotOccupancy = slotOccupancyStorage; @@ -2136,7 +2092,7 @@ internal sealed class ObuReader uint slot = referenceFrameIndices[reference]; if (!usesShortSignaling) { - slot = reader.ReadLiteral(ReferenceFrameIndexBits); + slot = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits); referenceFrameIndices[reference] = slot; } @@ -2778,31 +2734,43 @@ internal sealed class ObuReader { // Diagonal terms are coded as a delta from the identity scale, whereas off-diagonal terms are centered // directly around zero. Both are restored to the common sixteen-bit matrix precision after decoding. + int referenceHorizontalScale = + (referenceParameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + parameters[2] = (reader.ReadSignedReferenceSubexponential( - GlobalMotionAlphaValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, - (referenceParameters[2] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceHorizontalScale) * + Av1GlobalMotionParameters.AlphaDecodeFactor) + Av1GlobalMotionParameters.ModelScale; parameters[3] = reader.ReadSignedReferenceSubexponential( - GlobalMotionAlphaValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, - referenceParameters[3] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) * + Av1GlobalMotionParameters.AlphaDecodeFactor; } if (type >= Av1GlobalMotionType.Affine) { + int referenceVerticalScale = + (referenceParameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + parameters[4] = reader.ReadSignedReferenceSubexponential( - GlobalMotionAlphaValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, - referenceParameters[4] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) * + Av1GlobalMotionParameters.AlphaDecodeFactor; parameters[5] = (reader.ReadSignedReferenceSubexponential( - GlobalMotionAlphaValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, - (referenceParameters[5] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceVerticalScale) * + Av1GlobalMotionParameters.AlphaDecodeFactor) + Av1GlobalMotionParameters.ModelScale; } else @@ -2820,23 +2788,26 @@ internal sealed class ObuReader // remains in quarter-sample units. Affine translation retains the fixed model-to-translation precision gap. int precisionAdjustment = type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0; int translationBits = type == Av1GlobalMotionType.Translation - ? GlobalMotionAbsoluteTranslationOnlyBits - precisionAdjustment - : GlobalMotionAbsoluteTranslationBits; + ? Av1GlobalMotionParameters.AbsoluteTranslationOnlyBits - precisionAdjustment + : Av1GlobalMotionParameters.AbsoluteTranslationBits; int translationPrecisionDifference = type == Av1GlobalMotionType.Translation - ? Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationOnlyPrecisionBits + precisionAdjustment - : Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationPrecisionBits; + ? Av1GlobalMotionParameters.ModelPrecisionBits - + Av1GlobalMotionParameters.TranslationOnlyPrecisionBits + + precisionAdjustment + : Av1GlobalMotionParameters.ModelPrecisionBits - + Av1GlobalMotionParameters.TranslationPrecisionBits; int translationDecodeFactor = 1 << translationPrecisionDifference; int translationValueMagnitude = (1 << translationBits) + 1; parameters[0] = reader.ReadSignedReferenceSubexponential( translationValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, referenceParameters[0] >> translationPrecisionDifference) * translationDecodeFactor; parameters[1] = reader.ReadSignedReferenceSubexponential( translationValueMagnitude, - GlobalMotionSubexponentialGroupBitCount, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, referenceParameters[1] >> translationPrecisionDifference) * translationDecodeFactor; } diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs index 167cebbf3d..f694837976 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs @@ -219,6 +219,14 @@ internal sealed class ObuSequenceHeader /// public uint AdditionalFrameIdLength { get; set; } + /// + /// Gets the timing information required by syntax whose presence flag is set. + /// + /// The sequence timing information. + public ObuTimingInfo GetTimingInfo() => + this.TimingInfo + ?? throw new InvalidOperationException("The AV1 sequence has no timing information."); + /// /// Gets the decoder-buffer model information required by syntax whose presence flag is set. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs index 5b3e3ba5af..7d06c4e3d2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs @@ -3,82 +3,142 @@ using System.Buffers; using System.Buffers.Binary; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// -/// Writes the AV1 open bitstream units required for a single still-image frame. +/// Writes the AV1 open bitstream units for one coded frame. /// -internal sealed class ObuWriter +internal sealed class ObuWriter : IDisposable { // Sequence and uncompressed-frame syntax have fixed field and array limits. A 512-byte owner covers their // maximum supported representation without retaining any entropy-coded tile bytes in the header scratch. private const int MaximumHeaderLength = 512; + private readonly IMemoryOwner headerOwner; + + /// + /// Initializes a new instance of the class. + /// + /// The configuration providing reusable header memory. + public ObuWriter(Configuration configuration) + => this.headerOwner = configuration.MemoryAllocator.Allocate(MaximumHeaderLength); + + /// + /// Writes a temporal delimiter, sequence header, and coded frame for the first sample in a sequence. + /// + /// The non-boxed tile source type. + /// The destination stream. + /// The sequence header. + /// The frame header. + /// The encoded tile source. + public void WriteSequenceFrame( + Stream stream, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + TTileWriter tileWriter) + where TTileWriter : IAv1TileWriter + { + Span headerBuffer = this.headerOwner.Memory.Span[..MaximumHeaderLength]; + Av1BitStreamWriter writer = new(headerBuffer); + WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []); + WriteSequenceHeader(ref writer, sequenceHeader); + int bytesWritten = (writer.BitPosition + 7) >> 3; + writer.Flush(); + WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]); + WriteFrameObu(stream, sequenceHeader, frameHeader, tileWriter, headerBuffer, ref writer); + } + /// - /// Writes a temporal delimiter and the supplied sequence and frame OBUs. + /// Writes an empty temporal-delimiter OBU. + /// + /// The destination stream. + public static void WriteTemporalDelimiter(Stream stream) + => WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []); + + /// + /// Writes a temporal delimiter followed by one sequence-header OBU. /// /// The configuration used to allocate temporary encoding memory. /// The destination stream. - /// The optional still-picture sequence header. - /// The optional intra-frame header. - /// The tile writer used when a frame header is supplied. - [System.Diagnostics.CodeAnalysis.SuppressMessage( - "Performance", - "CA1822:Mark members as static", - Justification = "Preserves the existing writer instance contract.")] - public void WriteAll(Configuration configuration, Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, IAv1TileWriter tileWriter) - { - // The reusable scratch only contains headers. Entropy-coded tiles remain in their owning buffers and are - // streamed directly so the complete compressed frame is never duplicated. + /// The sequence header. + public static void WriteSequenceHeader(Configuration configuration, Stream stream, ObuSequenceHeader sequenceHeader) + { using IMemoryOwner headerOwner = configuration.MemoryAllocator.Allocate(MaximumHeaderLength); Span headerBuffer = headerOwner.Memory.Span[..MaximumHeaderLength]; Av1BitStreamWriter writer = new(headerBuffer); WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []); + WriteSequenceHeader(ref writer, sequenceHeader); + int bytesWritten = (writer.BitPosition + 7) >> 3; + writer.Flush(); + WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]); + } - if (sequenceHeader != null) - { - WriteSequenceHeader(ref writer, sequenceHeader); - int bytesWritten = (writer.BitPosition + 7) >> 3; - writer.Flush(); - WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]); - } - - if (frameHeader != null && sequenceHeader != null) - { - WriteFrameHeader(ref writer, sequenceHeader, frameHeader); - ObuTileGroupHeader tileInfo = frameHeader.TilesInfo; - if (tileInfo != null) - { - WriteTileGroupHeader(ref writer, tileInfo); - } - - int frameHeaderBytes = (writer.BitPosition + 7) >> 3; - writer.Flush(); + /// + /// Writes a temporal delimiter and coded frame that continues an established sequence. + /// + /// The non-boxed tile source type. + /// The destination stream. + /// The sequence header established by an earlier sample. + /// The frame header. + /// The encoded tile source. + public void WriteFrame( + Stream stream, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + TTileWriter tileWriter) + where TTileWriter : IAv1TileWriter + { + Span headerBuffer = this.headerOwner.Memory.Span[..MaximumHeaderLength]; + Av1BitStreamWriter writer = new(headerBuffer); + WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []); + WriteFrameObu(stream, sequenceHeader, frameHeader, tileWriter, headerBuffer, ref writer); + } - uint framePayloadSize = (uint)frameHeaderBytes; - if (tileInfo != null) - { - int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; - framePayloadSize += (uint)((tileCount - 1) * tileInfo.TileSizeBytes); + /// + public void Dispose() => this.headerOwner.Dispose(); - for (int tileNum = 0; tileNum < tileCount; tileNum++) - { - framePayloadSize += (uint)tileWriter.GetTileData(tileNum).Length; - } - } + /// + /// Writes the combined frame OBU header followed by each retained tile payload. + /// + /// The non-boxed tile source type. + /// The destination stream. + /// The sequence header governing frame syntax. + /// The uncompressed frame header. + /// The encoded tile source. + /// The reusable OBU header scratch. + /// The bit writer over . + private static void WriteFrameObu( + Stream stream, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + TTileWriter tileWriter, + Span headerBuffer, + ref Av1BitStreamWriter writer) + where TTileWriter : IAv1TileWriter + { + WriteFrameHeader(ref writer, sequenceHeader, frameHeader); + ObuTileGroupHeader tileInfo = frameHeader.TilesInfo; + WriteTileGroupHeader(ref writer, tileInfo); - WriteObuHeaderAndSize(stream, ObuType.Frame, framePayloadSize); - stream.Write(headerBuffer[..frameHeaderBytes]); + int frameHeaderBytes = (writer.BitPosition + 7) >> 3; + writer.Flush(); - if (tileInfo != null) - { - WriteTileData(stream, tileInfo, tileWriter); - } + int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; + uint framePayloadSize = (uint)(frameHeaderBytes + ((tileCount - 1) * tileInfo.TileSizeBytes)); + for (int tileNum = 0; tileNum < tileCount; tileNum++) + { + framePayloadSize += (uint)tileWriter.GetTileData(tileNum).Length; } + + WriteObuHeaderAndSize(stream, ObuType.Frame, framePayloadSize); + stream.Write(headerBuffer[..frameHeaderBytes]); + WriteTileData(stream, tileInfo, tileWriter); } /// @@ -145,29 +205,123 @@ internal sealed class ObuWriter } /// - /// Writes a reduced still-picture sequence header. + /// Writes an AV1 sequence header. /// /// The bit writer receiving the sequence header. /// The sequence header to encode. private static void WriteSequenceHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader) { writer.WriteLiteral((uint)sequenceHeader.SequenceProfile, 3); - writer.WriteBoolean(true); // IsStillPicture - writer.WriteBoolean(true); // IsReducedStillPicture - writer.WriteLiteral((uint)sequenceHeader.OperatingPoint[0].SequenceLevelIndex, Av1Constants.LevelBits); + writer.WriteBoolean(sequenceHeader.IsStillPicture); + writer.WriteBoolean(sequenceHeader.IsReducedStillPictureHeader); + if (sequenceHeader.IsReducedStillPictureHeader) + { + writer.WriteLiteral((uint)sequenceHeader.OperatingPoint[0].SequenceLevelIndex, Av1Constants.LevelBits); + } + else + { + writer.WriteBoolean(sequenceHeader.TimingInfoPresentFlag); + if (sequenceHeader.TimingInfoPresentFlag) + { + WriteTimingInfo(ref writer, sequenceHeader.GetTimingInfo()); + writer.WriteBoolean(sequenceHeader.DecoderModelInfoPresentFlag); + if (sequenceHeader.DecoderModelInfoPresentFlag) + { + WriteDecoderModelInfo(ref writer, sequenceHeader.GetDecoderModelInfo()); + } + } + + writer.WriteBoolean(sequenceHeader.InitialDisplayDelayPresentFlag); + writer.WriteLiteral( + (uint)(sequenceHeader.OperatingPoint.Length - 1), + Av1Constants.OperatingPointCountBits); + + foreach (ObuOperatingPoint operatingPoint in sequenceHeader.OperatingPoint) + { + writer.WriteLiteral(operatingPoint.Idc, Av1Constants.OperatingPointIdcBits); + writer.WriteLiteral((uint)operatingPoint.SequenceLevelIndex, Av1Constants.LevelBits); + if (operatingPoint.SequenceLevelIndex >= Av1Constants.SequenceTierMinimumLevelIndex) + { + writer.WriteBoolean(operatingPoint.SequenceTier != 0); + } + + if (sequenceHeader.DecoderModelInfoPresentFlag) + { + writer.WriteBoolean(operatingPoint.IsDecoderModelInfoPresent); + if (operatingPoint.IsDecoderModelInfoPresent) + { + WriteOperatingParametersInfo( + ref writer, + sequenceHeader.GetDecoderModelInfo(), + operatingPoint); + } + } + + if (sequenceHeader.InitialDisplayDelayPresentFlag) + { + writer.WriteBoolean(operatingPoint.IsInitialDisplayDelayPresent); + if (operatingPoint.IsInitialDisplayDelayPresent) + { + writer.WriteLiteral(operatingPoint.InitialDisplayDelay - 1, 4); + } + } + } + } - // Frame width and Height + // The maximum dimensions determine the fixed-width fields used by every frame in the sequence. writer.WriteLiteral((uint)sequenceHeader.FrameWidthBits - 1, 4); writer.WriteLiteral((uint)sequenceHeader.FrameHeightBits - 1, 4); writer.WriteLiteral((uint)sequenceHeader.MaxFrameWidth - 1, sequenceHeader.FrameWidthBits); writer.WriteLiteral((uint)sequenceHeader.MaxFrameHeight - 1, sequenceHeader.FrameHeightBits); + if (!sequenceHeader.IsReducedStillPictureHeader) + { + writer.WriteBoolean(sequenceHeader.IsFrameIdNumbersPresent); + if (sequenceHeader.IsFrameIdNumbersPresent) + { + writer.WriteLiteral((uint)sequenceHeader.DeltaFrameIdLength - 2, 4); + writer.WriteLiteral(sequenceHeader.AdditionalFrameIdLength - 1, 3); + } + } - // Video related flags removed writer.WriteBoolean(sequenceHeader.Use128x128Superblock); writer.WriteBoolean(sequenceHeader.EnableFilterIntra); writer.WriteBoolean(sequenceHeader.EnableIntraEdgeFilter); + if (!sequenceHeader.IsReducedStillPictureHeader) + { + writer.WriteBoolean(sequenceHeader.EnableInterIntraCompound); + writer.WriteBoolean(sequenceHeader.EnableMaskedCompound); + writer.WriteBoolean(sequenceHeader.EnableWarpedMotion); + writer.WriteBoolean(sequenceHeader.EnableDualFilter); + writer.WriteBoolean(sequenceHeader.EnableOrderHint); + if (sequenceHeader.EnableOrderHint) + { + writer.WriteBoolean(sequenceHeader.OrderHintInfo.EnableJointCompound); + writer.WriteBoolean(sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors); + } + + bool chooseScreenContentTools = sequenceHeader.ForceScreenContentTools == Av1Constants.SelectScreenContentTools; + writer.WriteBoolean(chooseScreenContentTools); + if (!chooseScreenContentTools) + { + writer.WriteBoolean(sequenceHeader.ForceScreenContentTools != 0); + } + + if (sequenceHeader.ForceScreenContentTools > 0) + { + bool chooseIntegerMotionVector = sequenceHeader.ForceIntegerMotionVector == Av1Constants.SelectIntegerMotionVector; + writer.WriteBoolean(chooseIntegerMotionVector); + if (!chooseIntegerMotionVector) + { + writer.WriteBoolean(sequenceHeader.ForceIntegerMotionVector != 0); + } + } + + if (sequenceHeader.EnableOrderHint) + { + writer.WriteLiteral((uint)sequenceHeader.OrderHintInfo.OrderHintBits - 1, 3); + } + } - // Video related flags removed writer.WriteBoolean(sequenceHeader.EnableSuperResolution); writer.WriteBoolean(sequenceHeader.EnableCdef); writer.WriteBoolean(sequenceHeader.EnableRestoration); @@ -176,6 +330,65 @@ internal sealed class ObuWriter WriteTrailingBits(ref writer); } + /// + /// Writes sequence timing in the fixed-width and unsigned-variable-length forms required by AV1. + /// + /// The bit writer receiving the timing information. + /// The timing values to encode. + private static void WriteTimingInfo(ref Av1BitStreamWriter writer, ObuTimingInfo timingInfo) + { + writer.WriteLiteral(timingInfo.NumUnitsInDisplayTick, 32); + writer.WriteLiteral(timingInfo.TimeScale, 32); + writer.WriteBoolean(timingInfo.EqualPictureInterval); + if (timingInfo.EqualPictureInterval) + { + WriteUnsignedVariableLength(ref writer, timingInfo.NumTicksPerPicture - 1); + } + } + + /// + /// Writes decoder-buffer field widths and decoding-clock units. + /// + /// The bit writer receiving the decoder-model information. + /// The decoder-model values to encode. + private static void WriteDecoderModelInfo(ref Av1BitStreamWriter writer, ObuDecoderModelInfo decoderModelInfo) + { + writer.WriteLiteral(decoderModelInfo.BufferDelayLength - 1, 5); + writer.WriteLiteral(decoderModelInfo.NumUnitsInDecodingTick, 32); + writer.WriteLiteral(decoderModelInfo.BufferRemovalTimeLength - 1, 5); + writer.WriteLiteral(decoderModelInfo.FramePresentationTimeLength - 1, 5); + } + + /// + /// Writes the decoder-model parameters for one operating point. + /// + /// The bit writer receiving the operating-point parameters. + /// The decoder model defining the delay field width. + /// The operating-point values to encode. + private static void WriteOperatingParametersInfo( + ref Av1BitStreamWriter writer, + ObuDecoderModelInfo decoderModelInfo, + ObuOperatingPoint operatingPoint) + { + int bufferDelayLength = (int)decoderModelInfo.BufferDelayLength; + writer.WriteLiteral(operatingPoint.DecoderBufferDelay, bufferDelayLength); + writer.WriteLiteral(operatingPoint.EncoderBufferDelay, bufferDelayLength); + writer.WriteBoolean(operatingPoint.LowDelayMode); + } + + /// + /// Writes an AV1 unsigned variable-length value. + /// + /// The bit writer receiving the value. + /// The value to encode. + private static void WriteUnsignedVariableLength(ref Av1BitStreamWriter writer, uint value) + { + uint encodedValue = value + 1; + int leadingZeroCount = Av1Math.MostSignificantBit(encodedValue); + writer.WriteLiteral(0, leadingZeroCount); + writer.WriteLiteral(encodedValue, leadingZeroCount + 1); + } + /// /// Writes the sequence color configuration. /// @@ -418,15 +631,60 @@ internal sealed class ObuWriter } /// - /// Writes the reduced uncompressed header for an intra still-image frame. + /// Writes the uncompressed header for an AV1 frame. /// /// The bit writer receiving the uncompressed frame header. /// The sequence header controlling available coding tools. /// The frame header to encode. private static void WriteUncompressedFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) { + bool frameSizeOverrideFlag = false; + if (!sequenceHeader.IsReducedStillPictureHeader) + { + writer.WriteBoolean(frameHeader.ShowExistingFrame); + if (frameHeader.ShowExistingFrame) + { + writer.WriteLiteral(frameHeader.FrameToShowMapIdx, Av1Constants.ReferenceFrameIndexBits); + if (sequenceHeader.DecoderModelInfoPresentFlag && !sequenceHeader.GetTimingInfo().EqualPictureInterval) + { + writer.WriteLiteral( + frameHeader.FramePresentationTime, + (int)sequenceHeader.GetDecoderModelInfo().FramePresentationTimeLength); + } + + if (sequenceHeader.IsFrameIdNumbersPresent) + { + writer.WriteLiteral(frameHeader.DisplayFrameId, sequenceHeader.FrameIdLength); + } + + return; + } + + writer.WriteLiteral((uint)frameHeader.FrameType, Av1Constants.FrameTypeBits); + writer.WriteBoolean(frameHeader.ShowFrame); + if (frameHeader.ShowFrame && + sequenceHeader.DecoderModelInfoPresentFlag && + !sequenceHeader.GetTimingInfo().EqualPictureInterval) + { + writer.WriteLiteral( + frameHeader.FramePresentationTime, + (int)sequenceHeader.GetDecoderModelInfo().FramePresentationTimeLength); + } + + if (!frameHeader.ShowFrame) + { + writer.WriteBoolean(frameHeader.ShowableFrame); + } + + if (frameHeader.FrameType != ObuFrameType.SwitchFrame && + (frameHeader.FrameType != ObuFrameType.KeyFrame || !frameHeader.ShowFrame)) + { + writer.WriteBoolean(frameHeader.ErrorResilientMode); + } + } + writer.WriteBoolean(frameHeader.DisableCdfUpdate); - if (sequenceHeader.ForceScreenContentTools == 2) + if (sequenceHeader.ForceScreenContentTools == Av1Constants.SelectScreenContentTools) { writer.WriteBoolean(frameHeader.AllowScreenContentTools); } @@ -437,7 +695,7 @@ internal sealed class ObuWriter if (frameHeader.AllowScreenContentTools) { - if (sequenceHeader.ForceIntegerMotionVector == 2) + if (sequenceHeader.ForceIntegerMotionVector == Av1Constants.SelectIntegerMotionVector) { writer.WriteBoolean(frameHeader.ForceIntegerMotionVector); } @@ -447,6 +705,41 @@ internal sealed class ObuWriter } } + if (!sequenceHeader.IsReducedStillPictureHeader) + { + if (sequenceHeader.IsFrameIdNumbersPresent) + { + writer.WriteLiteral(frameHeader.CurrentFrameId, sequenceHeader.FrameIdLength); + } + + frameSizeOverrideFlag = frameHeader.FrameType == ObuFrameType.SwitchFrame || + frameHeader.FrameSize.SuperResolutionUpscaledWidth != sequenceHeader.MaxFrameWidth || + frameHeader.FrameSize.FrameHeight != sequenceHeader.MaxFrameHeight; + + if (frameHeader.FrameType != ObuFrameType.SwitchFrame) + { + writer.WriteBoolean(frameSizeOverrideFlag); + } + + writer.WriteLiteral(frameHeader.OrderHint, sequenceHeader.OrderHintInfo.OrderHintBits); + if (!frameHeader.ErrorResilientMode && !frameHeader.IsIntra) + { + writer.WriteLiteral(frameHeader.PrimaryReferenceFrame, Av1Constants.PrimaryReferenceBits); + } + } + + if (sequenceHeader.DecoderModelInfoPresentFlag) + { + // Image-sequence timing is carried by the container track, so encoded samples do not signal decoder-buffer removal times. + writer.WriteBoolean(false); + } + + if ((frameHeader.FrameType == ObuFrameType.KeyFrame && !frameHeader.ShowFrame) || + frameHeader.FrameType is ObuFrameType.InterFrame or ObuFrameType.IntraOnlyFrame) + { + writer.WriteLiteral(frameHeader.RefreshFrameFlags, Av1Constants.ReferenceFrameCount); + } + if (frameHeader.FrameType == ObuFrameType.KeyFrame) { if (!frameHeader.ShowFrame) @@ -461,7 +754,7 @@ internal sealed class ObuWriter if (frameHeader.FrameType == ObuFrameType.KeyFrame) { - WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); + WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag); WriteRenderSize(ref writer, frameHeader); if (frameHeader.AllowScreenContentTools) { @@ -470,7 +763,7 @@ internal sealed class ObuWriter } else if (frameHeader.FrameType == ObuFrameType.IntraOnlyFrame) { - WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); + WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag); WriteRenderSize(ref writer, frameHeader); if (frameHeader.AllowScreenContentTools) { @@ -479,7 +772,32 @@ internal sealed class ObuWriter } else { - throw new NotImplementedException("Inter frames not applicable for AVIF."); + WriteReferenceFrameIndices(ref writer, sequenceHeader, frameHeader); + WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag); + WriteRenderSize(ref writer, frameHeader); + if (!frameHeader.ForceIntegerMotionVector) + { + writer.WriteBoolean(frameHeader.AllowHighPrecisionMotionVector); + } + + WriteFrameInterpolationFilter(ref writer, frameHeader.InterpolationFilter); + writer.WriteBoolean(frameHeader.IsMotionModeSwitchable); + } + + bool mightAllowReferenceFrameMotionVectors = + !frameHeader.ErrorResilientMode && + sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors && + sequenceHeader.EnableOrderHint && + !frameHeader.IsIntra; + + if (mightAllowReferenceFrameMotionVectors) + { + writer.WriteBoolean(frameHeader.UseReferenceFrameMotionVectors); + } + + if (!sequenceHeader.IsReducedStillPictureHeader && !frameHeader.DisableCdfUpdate) + { + writer.WriteBoolean(frameHeader.DisableFrameEndUpdateCdf); } WriteTileInfo(ref writer, sequenceHeader, frameHeader); @@ -531,20 +849,68 @@ internal sealed class ObuWriter } } - // No Frame Reference mode selection for AVIF WriteTransformMode(ref writer, frameHeader); - // No compound INTER-INTER for AVIF. WriteFrameReferenceMode(ref writer, frameHeader); WriteSkipModeParameters(ref writer, frameHeader); + if (!frameHeader.IsIntra && !frameHeader.ErrorResilientMode && sequenceHeader.EnableWarpedMotion) + { + writer.WriteBoolean(frameHeader.AllowWarpedMotion); + } - // No warp motion for AVIF. writer.WriteBoolean(frameHeader.UseReducedTransformSet); WriteGlobalMotionParameters(ref writer, frameHeader); WriteFilmGrainFilterParameters(ref writer, sequenceHeader, frameHeader); } + /// + /// Writes the seven reference-map slots selected by an inter frame. + /// + /// The bit writer receiving the reference indices. + /// The sequence header defining frame-ID and order-hint syntax. + /// The frame header containing the selected reference slots. + private static void WriteReferenceFrameIndices( + ref Av1BitStreamWriter writer, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader) + { + // Long signaling is deterministic and permits every reference role to select the same retained slot. + if (sequenceHeader.EnableOrderHint) + { + writer.WriteBoolean(false); + } + + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + Span referenceFrameIds = frameHeader.GetReferenceFrameIds(); + uint frameIdModulus = 1U << sequenceHeader.FrameIdLength; + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + uint slot = referenceFrameIndices[reference]; + writer.WriteLiteral(slot, Av1Constants.ReferenceFrameIndexBits); + if (sequenceHeader.IsFrameIdNumbersPresent) + { + uint deltaFrameId = (frameHeader.CurrentFrameId + frameIdModulus - referenceFrameIds[(int)slot]) % frameIdModulus; + writer.WriteLiteral(deltaFrameId - 1, sequenceHeader.DeltaFrameIdLength); + } + } + } + + /// + /// Writes the frame-level interpolation-filter selection. + /// + /// The bit writer receiving the filter selection. + /// The fixed filter family or per-block selection. + private static void WriteFrameInterpolationFilter(ref Av1BitStreamWriter writer, Av1InterpolationFilter filter) + { + bool isSwitchable = filter == Av1InterpolationFilter.Switchable; + writer.WriteBoolean(isSwitchable); + if (!isSwitchable) + { + writer.WriteLiteral((uint)filter, 2); + } + } + /// /// Writes the frame-header portion of a combined frame OBU. /// @@ -571,8 +937,8 @@ internal sealed class ObuWriter if (tileCount > 1) { - // A combined OBU_FRAME has implicit complete-frame tile bounds. The reference decoder still - // writes the presence bit for a multi-tile frame, but requires that bit to remain zero. + // A combined frame always carries the complete raster tile group. The zero bit selects those implicit + // full-frame bounds instead of adding explicit start and end tile indices. writer.WriteBoolean(false); } @@ -582,10 +948,15 @@ internal sealed class ObuWriter /// /// Writes the size-prefixed tile payloads in raster order. /// + /// The non-boxed tile source type. /// The destination stream receiving tile data. /// The frame tile layout and tile-size field width. /// The writer that produces each tile payload. - private static void WriteTileData(Stream stream, ObuTileGroupHeader tileInfo, IAv1TileWriter tileWriter) + private static void WriteTileData( + Stream stream, + ObuTileGroupHeader tileInfo, + TTileWriter tileWriter) + where TTileWriter : IAv1TileWriter { int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; Span tileSizeBuffer = stackalloc byte[sizeof(uint)]; @@ -659,7 +1030,7 @@ internal sealed class ObuWriter } /// - /// Writes segmentation feature data for an independently decoded still-image frame. + /// Writes segmentation feature data for one coded frame. /// /// The bit writer receiving the segmentation parameters. /// The frame header containing segmentation feature data. @@ -672,9 +1043,8 @@ internal sealed class ObuWriter return; } - // The still-image writer emits independent intra frames with no primary reference. - // AV1 therefore infers update-map and update-data as enabled and carries feature data - // directly, without the inter-frame update flags. + // A frame with no primary reference starts a new segmentation domain. AV1 therefore infers + // update-map and update-data as enabled and carries the complete feature state directly. for (int segmentId = 0; segmentId < Av1Constants.MaxSegmentCount; segmentId++) { for (int featureId = 0; featureId < Av1Constants.SegmentationLevelMax; featureId++) @@ -839,15 +1209,140 @@ internal sealed class ObuWriter /// The current frame header. private static void WriteGlobalMotionParameters(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader) { - _ = writer; - if (frameHeader.IsIntra) { - // Nothing to be written for INTRA frames. return; } - throw new InvalidImageContentException("AVIF files can only contain INTRA frames."); + ReadOnlySpan parameters = frameHeader.GetGlobalMotionParameters(); + Av1GlobalMotionParameters referenceParameters = Av1GlobalMotionParameters.Identity; + for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++) + { + WriteGlobalMotionModel( + ref writer, + parameters[reference], + referenceParameters, + frameHeader.AllowHighPrecisionMotionVector); + } + } + + /// + /// Writes one global-motion model relative to the same-role model in the primary reference frame. + /// + private static void WriteGlobalMotionModel( + ref Av1BitStreamWriter writer, + Av1GlobalMotionParameters parameters, + Av1GlobalMotionParameters referenceParameters, + bool allowHighPrecisionMotionVector) + { + Av1GlobalMotionType type = parameters.Type; + writer.WriteBoolean(type != Av1GlobalMotionType.Identity); + if (type != Av1GlobalMotionType.Identity) + { + writer.WriteBoolean(type == Av1GlobalMotionType.RotationZoom); + if (type != Av1GlobalMotionType.RotationZoom) + { + writer.WriteBoolean(type == Av1GlobalMotionType.Translation); + } + } + + if (type >= Av1GlobalMotionType.RotationZoom) + { + int horizontalScale = + (parameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + + int referenceHorizontalScale = + (referenceParameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + + writer.WriteSignedReferenceSubexponential( + horizontalScale, + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceHorizontalScale); + + writer.WriteSignedReferenceSubexponential( + parameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference, + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference); + } + + if (type >= Av1GlobalMotionType.Affine) + { + int verticalScale = + (parameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + + int referenceVerticalScale = + (referenceParameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) - + (1 << Av1GlobalMotionParameters.AlphaPrecisionBits); + + writer.WriteSignedReferenceSubexponential( + parameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference, + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference); + + writer.WriteSignedReferenceSubexponential( + verticalScale, + Av1GlobalMotionParameters.AlphaValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceVerticalScale); + } + + if (type >= Av1GlobalMotionType.Translation) + { + int precisionAdjustment = + type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0; + + int translationBits = type == Av1GlobalMotionType.Translation + ? Av1GlobalMotionParameters.AbsoluteTranslationOnlyBits - precisionAdjustment + : Av1GlobalMotionParameters.AbsoluteTranslationBits; + + int translationPrecisionDifference = type == Av1GlobalMotionType.Translation + ? Av1GlobalMotionParameters.ModelPrecisionBits - + Av1GlobalMotionParameters.TranslationOnlyPrecisionBits + + precisionAdjustment + : Av1GlobalMotionParameters.ModelPrecisionBits - + Av1GlobalMotionParameters.TranslationPrecisionBits; + + int translationValueMagnitude = (1 << translationBits) + 1; + writer.WriteSignedReferenceSubexponential( + parameters[0] >> translationPrecisionDifference, + translationValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[0] >> translationPrecisionDifference); + + writer.WriteSignedReferenceSubexponential( + parameters[1] >> translationPrecisionDifference, + translationValueMagnitude, + Av1GlobalMotionParameters.SubexponentialGroupBitCount, + referenceParameters[1] >> translationPrecisionDifference); + } + } + + /// + /// Gets the exact number of uncompressed-header bits required by one global-motion model. + /// + /// The model to measure. + /// Whether translation may retain one-eighth-sample precision. + /// The encoded model length in bits. + internal static int GetGlobalMotionModelBitCount( + Av1GlobalMotionParameters parameters, + bool allowHighPrecisionMotionVector) + { + InlineArray16 storage = default; + Span buffer = storage; + Av1BitStreamWriter writer = new(buffer); + WriteGlobalMotionModel( + ref writer, + parameters, + Av1GlobalMotionParameters.Identity, + allowHighPrecisionMotionVector); + + return writer.BitPosition; } /// @@ -857,15 +1352,12 @@ internal sealed class ObuWriter /// The current frame header. private static void WriteFrameReferenceMode(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader) { - _ = writer; - if (frameHeader.IsIntra) { - // Nothing to be written for INTRA frames. return; } - throw new InvalidImageContentException("AVIF files can only contain INTRA frames."); + writer.WriteBoolean(frameHeader.ReferenceMode == ObuReferenceMode.ReferenceModeSelect); } /// @@ -882,7 +1374,7 @@ internal sealed class ObuWriter } /// - /// Writes film-grain synthesis parameters for a displayed still-image frame. + /// Writes film-grain synthesis parameters for a displayed frame. /// /// The bit writer receiving the film-grain parameters. /// The sequence header defining film-grain availability and color sampling. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs index 9386e92e7d..62e982c752 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; @@ -39,40 +40,49 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable private const int TransformCoefficientOffset = ResidualStorageLength; private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount; private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount; - private const int IntraBlockCopySampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength; - private const int IntraBlockCopySampleStorageLength = - Av1EncoderIntraBlockCopyWorkspace.SampleBufferCount * - Av1EncoderIntraBlockCopyWorkspace.MaximumSampleCount * + private const int InterPredictionSampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength; + private const int InterPredictionSampleStorageLength = + Av1EncoderInterPredictionWorkspace.SampleBufferCount * + Av1EncoderInterPredictionWorkspace.MaximumSampleCount * sizeof(ushort) / sizeof(int); - private const int IntraBlockCopyResidualStorageOffset = - IntraBlockCopySampleStorageOffset + IntraBlockCopySampleStorageLength; + private const int InterPredictionResidualStorageOffset = + InterPredictionSampleStorageOffset + InterPredictionSampleStorageLength; - private const int IntraBlockCopyResidualStorageLength = - Av1EncoderIntraBlockCopyWorkspace.MaximumSampleCount * + private const int InterPredictionResidualStorageLength = + Av1EncoderInterPredictionWorkspace.MaximumSampleCount * sizeof(short) / sizeof(int); - private const int IntraBlockCopyCoefficientStorageOffset = - IntraBlockCopyResidualStorageOffset + IntraBlockCopyResidualStorageLength; + private const int InterPredictionScratchStorageOffset = + InterPredictionResidualStorageOffset + InterPredictionResidualStorageLength; - private const int IntraBlockCopyCoefficientStorageLength = - Av1EncoderIntraBlockCopyWorkspace.CoefficientBufferCount * - Av1EncoderIntraBlockCopyWorkspace.MaximumSampleCount; + private const int InterPredictionScratchStorageLength = + Av1EncoderInterPredictionWorkspace.PredictionScratchCount * + sizeof(short) / + sizeof(int); + + private const int InterPredictionCoefficientStorageOffset = + InterPredictionScratchStorageOffset + InterPredictionScratchStorageLength; - private const int IntraBlockCopyStorageLength = - IntraBlockCopySampleStorageLength + - IntraBlockCopyResidualStorageLength + - IntraBlockCopyCoefficientStorageLength; + private const int InterPredictionCoefficientStorageLength = + Av1EncoderInterPredictionWorkspace.CoefficientBufferCount * + Av1EncoderInterPredictionWorkspace.MaximumSampleCount; + + private const int InterPredictionStorageLength = + InterPredictionSampleStorageLength + + InterPredictionResidualStorageLength + + InterPredictionScratchStorageLength + + InterPredictionCoefficientStorageLength; private const int ModeDecisionStorageLength = Av1EncoderModeDecisionWorkspace.StorageLength; - private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > IntraBlockCopyStorageLength + private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > InterPredictionStorageLength ? ModeDecisionStorageLength - : IntraBlockCopyStorageLength; + : InterPredictionStorageLength; private const int PartitionContextStorageOffset = - IntraBlockCopySampleStorageOffset + SharedModeDecisionStorageLength; + InterPredictionSampleStorageOffset + SharedModeDecisionStorageLength; private const int MaximumPartitionEdgeUnitCount = 2 * (1 << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2)); @@ -97,6 +107,11 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable /// private readonly IMemoryOwner owner; + /// + /// Reuses the fixed-capacity reference-vector stack for every inter block in the frame. + /// + private Av1ReferenceMotionVectors referenceMotionVectors; + /// /// Initializes a new instance of the class. /// @@ -128,6 +143,11 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable public Span TransformWorkspace => this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength); + /// + /// Gets the reusable reference-vector stack used by inter mode decision and syntax writing. + /// + public ref Av1ReferenceMotionVectors ReferenceMotionVectors => ref this.referenceMotionVectors; + /// /// Gets the disjoint edge snapshot used to restore one square partition-search level. /// @@ -152,43 +172,49 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable public Av1EncoderModeDecisionWorkspace GetModeDecisionWorkspace() where TSample : unmanaged { - // Conventional intra search finishes before intra-block-copy search begins for the same block. + // Conventional intra search finishes before reference prediction begins for the same block. // Both phases can therefore reuse this aligned region without extending the owner or preserving stale scratch. Span storage = this.owner.Memory.Span.Slice( - IntraBlockCopySampleStorageOffset, + InterPredictionSampleStorageOffset, SharedModeDecisionStorageLength); return new(storage[..Av1EncoderModeDecisionWorkspace.StorageLength]); } /// - /// Gets the reusable storage used while comparing intra-block-copy candidates. + /// Gets the reusable storage used while comparing single-reference or intra-block-copy candidates. /// /// The native sample type selected by the encoder pipeline. - /// The typed intra-block-copy workspace. - public Av1EncoderIntraBlockCopyWorkspace GetIntraBlockCopyWorkspace() + /// The typed inter-prediction workspace. + public Av1EncoderInterPredictionWorkspace GetInterPredictionWorkspace() where TSample : unmanaged { Span storage = this.owner.Memory.Span; Span sampleStorage = MemoryMarshal - .Cast(storage.Slice(IntraBlockCopySampleStorageOffset, IntraBlockCopySampleStorageLength)); + .Cast(storage.Slice(InterPredictionSampleStorageOffset, InterPredictionSampleStorageLength)); sampleStorage = sampleStorage[ - ..(Av1EncoderIntraBlockCopyWorkspace.SampleBufferCount * - Av1EncoderIntraBlockCopyWorkspace.MaximumSampleCount)]; + ..(Av1EncoderInterPredictionWorkspace.SampleBufferCount * + Av1EncoderInterPredictionWorkspace.MaximumSampleCount)]; Span residualStorage = MemoryMarshal - .Cast(storage.Slice(IntraBlockCopyResidualStorageOffset, IntraBlockCopyResidualStorageLength)); + .Cast(storage.Slice(InterPredictionResidualStorageOffset, InterPredictionResidualStorageLength)); + + residualStorage = residualStorage[..Av1EncoderInterPredictionWorkspace.MaximumSampleCount]; + + Span predictionScratch = MemoryMarshal + .Cast(storage.Slice(InterPredictionScratchStorageOffset, InterPredictionScratchStorageLength)); - residualStorage = residualStorage[..Av1EncoderIntraBlockCopyWorkspace.MaximumSampleCount]; + predictionScratch = predictionScratch[..Av1EncoderInterPredictionWorkspace.PredictionScratchCount]; Span coefficientStorage = storage.Slice( - IntraBlockCopyCoefficientStorageOffset, - IntraBlockCopyCoefficientStorageLength); + InterPredictionCoefficientStorageOffset, + InterPredictionCoefficientStorageLength); - return new Av1EncoderIntraBlockCopyWorkspace( + return new Av1EncoderInterPredictionWorkspace( sampleStorage, residualStorage, + predictionScratch, coefficientStorage); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs index 69bd3974b2..3f63afff57 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs @@ -345,6 +345,44 @@ internal readonly struct Av1EncoderFrame _ => this.chromaRed }; + /// + /// Gets a top-left view whose visible dimensions can be smaller than the backing coded planes. + /// + /// The visible luma width. + /// The visible luma height. + /// The requested non-owning planar view. + public PlanarView GetSubView(int width, int height) + { + Av1ColorFormat colorFormat = this.IsMonochrome + ? Av1ColorFormat.Yuv400 + : this.ChromaSubsamplingX == 0 + ? Av1ColorFormat.Yuv444 + : this.ChromaSubsamplingY == 0 + ? Av1ColorFormat.Yuv422 + : Av1ColorFormat.Yuv420; + + int chromaWidth = (width + this.ChromaSubsamplingX) >> this.ChromaSubsamplingX; + int chromaHeight = (height + this.ChromaSubsamplingY) >> this.ChromaSubsamplingY; + Buffer2DRegion blue = this.IsMonochrome + ? default + : this.chromaBlue.GetSubRegion(0, 0, chromaWidth, chromaHeight); + + Buffer2DRegion red = this.IsMonochrome + ? default + : this.chromaRed.GetSubRegion(0, 0, chromaWidth, chromaHeight); + + return new PlanarView( + this.luma.GetSubRegion(0, 0, width, height), + blue, + red, + width, + height, + this.LumaBitDepth, + colorFormat, + this.ChromaPositionX, + this.ChromaPositionY); + } + /// public Span GetLumaRowSpan(int row) => this.luma.DangerousGetRowSpan(row); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs index 9c429bd2a6..1742b94ed0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs @@ -14,6 +14,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; internal sealed class Av1EncoderFrameBuffer : IDisposable where TSample : unmanaged { + /// + /// The byte boundary used by libaom for SIMD-accessible component planes. + /// + private const int PlaneAlignmentBytes = 32; + /// /// The complete frame owner, or after disposal. /// @@ -48,7 +53,7 @@ internal sealed class Av1EncoderFrameBuffer : IDisposable : Av1EncoderFrame.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY); int chromaElementCount = checked(chromaSize.Width * chromaSize.Height); - int planeAlignment = Math.Max(32 / Unsafe.SizeOf(), 1); + int planeAlignment = Math.Max(PlaneAlignmentBytes / Unsafe.SizeOf(), 1); int chromaBlueOffset = Align(lumaElementCount, planeAlignment); int chromaRedOffset = Align(checked(chromaBlueOffset + chromaElementCount), planeAlignment); int storageLength = colorFormat == Av1ColorFormat.Yuv400 diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderInterPredictionWorkspace.cs similarity index 76% rename from src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs rename to src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderInterPredictionWorkspace.cs index 565a7bc008..9d26114870 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderInterPredictionWorkspace.cs @@ -1,19 +1,33 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Provides disjoint reusable buffers for intra-block-copy mode decisions. +/// Provides disjoint reusable buffers for single-reference and intra-block-copy mode decisions. /// /// The native sample type selected by the encoder pipeline. -internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace +internal readonly ref struct Av1EncoderInterPredictionWorkspace where TSample : unmanaged { /// - /// The number of samples in the fixed 8x8 intra-block-copy transform. + /// The width and height of the fixed prediction block handled by the current inter search. + /// + private const int MaximumBlockDimension = 8; + + /// + /// The number of samples in the fixed prediction block. + /// + public const int MaximumSampleCount = MaximumBlockDimension * MaximumBlockDimension; + + /// + /// The signed intermediate capacity needed when both translational interpolation axes are filtered. /// - public const int MaximumSampleCount = 8 * 8; + public const int PredictionScratchCount = + Av1TranslationalInterPredictor.MinimumScratchStride * + (MaximumBlockDimension + Av1TranslationalInterPredictor.MaximumExtraRows); /// /// The number of sample buffers retained by one mode decision. @@ -27,21 +41,25 @@ internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace private readonly Span samples; private readonly Span residual; + private readonly Span predictionScratch; private readonly Span coefficients; /// - /// Initializes a new instance of the struct. + /// Initializes a new instance of the struct. /// /// The sample storage. /// The residual storage shared by sequential plane evaluations. + /// The intermediate storage used by two-dimensional interpolation. /// The coefficient storage. - public Av1EncoderIntraBlockCopyWorkspace( + public Av1EncoderInterPredictionWorkspace( Span samples, Span residual, + Span predictionScratch, Span coefficients) { this.samples = samples; this.residual = residual; + this.predictionScratch = predictionScratch; this.coefficients = coefficients; } @@ -100,6 +118,11 @@ internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace /// public Span Residual => this.residual; + /// + /// Gets the intermediate scratch used when both translational interpolation axes are filtered. + /// + public Span PredictionScratch => this.predictionScratch; + /// /// Gets the selected luma coefficients. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs index c8586d9d8e..6cc6151393 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs @@ -1,20 +1,26 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.Color; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Components; using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; +using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// -/// Performs operation-scoped AV1 still-image frame encoding. +/// Performs operation-scoped AV1 frame encoding. /// internal static class Av1FrameEncoder { @@ -41,6 +47,88 @@ internal static class Av1FrameEncoder /// private const int AllIntraBufferScaleDenominator = 2; + /// + /// The sequence-level value that leaves the operating point unconstrained for decoder capability signaling. + /// + private const int UnconstrainedSequenceLevelIndex = 31; + + /// + /// The highest public effort value, which enables 128x128 superblocks for sufficiently large images. + /// + private const int MaximumEffort = 10; + + /// + /// The native component precision used by the byte pipeline. + /// + private const int ByteSampleBitDepth = 8; + + /// + /// The centered chroma position expressed in AV1 half-luma-sample units. + /// + private const int CenteredChromaSamplePosition = 1; + + /// + /// The first effort tier that searches frame-level translation between sequence samples. + /// + private const int MinimumGlobalMotionSearchEffort = 6; + + /// + /// The first effort tier that compares the three interpolation families for reference-frame prediction. + /// + private const int MinimumSwitchableInterpolationEffort = 8; + + /// + /// The first effort tier that searches independent vertical and horizontal interpolation families. + /// + private const int MinimumDualInterpolationEffort = 9; + + /// + /// The smallest full-pixel radius searched when frame-level motion analysis is enabled. + /// + private const int MinimumGlobalMotionSearchRadius = 4; + + /// + /// The largest dimension of the central luma window used during candidate discovery. + /// + private const int MaximumGlobalMotionAnalysisDimension = 512; + + /// + /// The number of cardinal and diagonal candidates evaluated at each motion-search step. + /// + private const int GlobalMotionSearchDirectionCount = 8; + + private enum FrameEncodingKind + { + StillColor, + StillAlpha + } + + /// + /// Defines the sample-specific SIMD squared-error operation used by frame-level motion search. + /// + /// The component sample type. + private interface IGlobalMotionSearchOperator + where TSample : unmanaged + { + /// + /// Calculates squared error between two equally sized strided sample regions. + /// + /// The first sample of the source region. + /// The source distance, in samples, between adjacent rows. + /// The first sample of the prediction region. + /// The prediction distance, in samples, between adjacent rows. + /// The number of samples compared in each row. + /// The number of rows compared. + /// The sum of squared component differences. + public static abstract long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height); + } + /// /// Encodes one reduced-still-picture AV1 frame into a low-overhead OBU stream. /// @@ -60,7 +148,53 @@ internal static class Av1FrameEncoder int qIndex, int effort) where TPixel : unmanaged, IPixel - => Encode(configuration, image, stream, colorConfig, qIndex, effort, false); + { + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + return Encode( + configuration, + image, + sourceRectangle, + sourceRectangle.Size, + stream, + colorConfig, + qIndex, + effort, + FrameEncodingKind.StillColor); + } + + /// + /// Encodes one grid cell as a reduced-still-picture AV1 frame in a low-overhead OBU stream. + /// + /// The packed source pixel type. + /// The configuration providing every operation-scoped allocation. + /// The packed source frame. + /// The source region copied into the top-left of the encoded cell. + /// The encoded cell dimensions, including any required edge padding. + /// The destination receiving the complete AV1 item payload. + /// The resolved native color and precision configuration. + /// The frame quantizer index. + /// The mode-search effort in the inclusive range zero through ten. + /// The sequence header describing the encoded payload. + public static ObuSequenceHeader EncodeGridCell( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Size cellSize, + Stream stream, + ObuColorConfig colorConfig, + int qIndex, + int effort) + where TPixel : unmanaged, IPixel + => Encode( + configuration, + image, + sourceRectangle, + cellSize, + stream, + colorConfig, + qIndex, + effort, + FrameEncodingKind.StillColor); /// /// Encodes one packed alpha channel as a reduced-still-picture monochrome AV1 frame. @@ -81,20 +215,167 @@ internal static class Av1FrameEncoder int qIndex, int effort) where TPixel : unmanaged, IPixel - => Encode(configuration, image, stream, colorConfig, qIndex, effort, true); + { + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + return Encode( + configuration, + image, + sourceRectangle, + sourceRectangle.Size, + stream, + colorConfig, + qIndex, + effort, + FrameEncodingKind.StillAlpha); + } + + /// + /// Encodes one packed alpha grid cell as a reduced-still-picture monochrome AV1 frame. + /// + /// The packed source pixel type. + /// The configuration providing every operation-scoped allocation. + /// The packed source frame. + /// The source region copied into the top-left of the encoded cell. + /// The encoded cell dimensions, including any required edge padding. + /// The destination receiving the complete AV1 item payload. + /// The resolved monochrome precision configuration. + /// The frame quantizer index. + /// The mode-search effort in the inclusive range zero through ten. + /// The sequence header describing the encoded payload. + public static ObuSequenceHeader EncodeAlphaGridCell( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Size cellSize, + Stream stream, + ObuColorConfig colorConfig, + int qIndex, + int effort) + where TPixel : unmanaged, IPixel + => Encode( + configuration, + image, + sourceRectangle, + cellSize, + stream, + colorConfig, + qIndex, + effort, + FrameEncodingKind.StillAlpha); + + /// + /// Creates an encoder that retains reconstructed color frames for prediction by later samples in the sequence. + /// + public static SequenceEncoder CreateColorSequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort) + => CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, false); + + /// + /// Creates an encoder that retains reconstructed alpha frames for prediction by later samples in the sequence. + /// + public static SequenceEncoder CreateAlphaSequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort) + => CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, true); private static ObuSequenceHeader Encode( Configuration configuration, ImageFrame image, + Rectangle sourceRectangle, + Size frameSize, Stream stream, ObuColorConfig colorConfig, int qIndex, int effort, - bool encodeAlpha) + FrameEncodingKind encodingKind) where TPixel : unmanaged, IPixel { - int width = image.Width; - int height = image.Height; + int width = frameSize.Width; + int height = frameSize.Height; + bool encodeAlpha = encodingKind == FrameEncodingKind.StillAlpha; + Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); + ObuSequenceHeader sequenceHeader = CreateSequenceHeader( + width, + height, + colorConfig, + effort, + true); + + ObuFrameHeader frameHeader = CreateFrameHeader( + width, + height, + qIndex, + effort, + ObuFrameType.KeyFrame); + + int tileBufferLength = GetTileBufferLength(width, height, colorConfig); + if (colorConfig.BitDepth == Av1BitDepth.EightBit) + { + EncodeByte( + configuration, + image, + sourceRectangle, + frameSize, + stream, + sequenceHeader, + frameHeader, + colorFormat, + tileBufferLength, + effort, + encodeAlpha); + } + else + { + EncodeHighBitDepth( + configuration, + image, + sourceRectangle, + frameSize, + stream, + sequenceHeader, + frameHeader, + colorFormat, + tileBufferLength, + effort, + encodeAlpha); + } + + return sequenceHeader; + } + + private static SequenceEncoder CreateSequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort, + bool encodeAlpha) + { + if (colorConfig.BitDepth == Av1BitDepth.EightBit) + { + return new ByteSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, encodeAlpha); + } + + return new HighBitDepthSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, encodeAlpha); + } + + private static ObuSequenceHeader CreateSequenceHeader( + int width, + int height, + ObuColorConfig colorConfig, + int effort, + bool isStillPicture) + { Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); ObuSequenceProfile sequenceProfile = colorConfig.BitDepth == Av1BitDepth.TwelveBit || colorFormat == Av1ColorFormat.Yuv422 @@ -103,49 +384,106 @@ internal static class Av1FrameEncoder ? ObuSequenceProfile.High : ObuSequenceProfile.Main; - ObuSequenceHeader sequenceHeader = new() + return new ObuSequenceHeader { - IsStillPicture = true, - IsReducedStillPictureHeader = true, + IsStillPicture = isStillPicture, + IsReducedStillPictureHeader = isStillPicture, SequenceProfile = sequenceProfile, - OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }], + OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = UnconstrainedSequenceLevelIndex }], FrameWidthBits = width > 1 ? Av1Math.MostSignificantBit((uint)(width - 1)) + 1 : 1, FrameHeightBits = height > 1 ? Av1Math.MostSignificantBit((uint)(height - 1)) + 1 : 1, MaxFrameWidth = width, MaxFrameHeight = height, - Use128x128Superblock = effort == 10 && width >= 128 && height >= 128, - ForceScreenContentTools = 2, - ForceIntegerMotionVector = 2, + Use128x128Superblock = Uses128x128Superblock(width, height, effort), + ForceScreenContentTools = Av1Constants.SelectScreenContentTools, + ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector, EnableFilterIntra = effort >= 4, + EnableDualFilter = !isStillPicture && effort >= MinimumDualInterpolationEffort, EnableIntraEdgeFilter = false, EnableSuperResolution = false, EnableCdef = false, EnableRestoration = false, ColorConfig = colorConfig }; + } - int modeInfoColumnCount = 2 * ((width + 7) >> 3); - int modeInfoRowCount = 2 * ((height + 7) >> 3); + private static bool Uses128x128Superblock(int width, int height, int effort) + => effort == MaximumEffort && + width >= Av1BlockSize.Block128x128.GetWidth() && + height >= Av1BlockSize.Block128x128.GetHeight(); + + private static ObuTileGroupHeader CreateTileGroupHeader( + int width, + int height, + int modeInfoColumnCount, + int modeInfoRowCount, + int effort) + { + int superblockSizeLog2 = Uses128x128Superblock(width, height, effort) + ? Av1Constants.MaxSuperBlockSizeLog2 + : Av1Constants.MaxSuperBlockSizeLog2 - 1; + + int superblockShift = superblockSizeLog2 - Av1Constants.ModeInfoSizeLog2; + int superblockColumns = Av1Math.DivideLog2Ceiling(modeInfoColumnCount, superblockShift); + int superblockRows = Av1Math.DivideLog2Ceiling(modeInfoRowCount, superblockShift); + int maximumTileWidth = Av1Constants.MaxTileWidth >> superblockSizeLog2; + int maximumTileArea = Av1Constants.MaxTileArea >> (2 * superblockSizeLog2); + int tileColumnCountLog2 = ObuReader.TileLog2(maximumTileWidth, superblockColumns); + int minimumTileCountLog2 = Math.Max( + tileColumnCountLog2, + ObuReader.TileLog2(maximumTileArea, superblockColumns * superblockRows)); + + int tileRowCountLog2 = minimumTileCountLog2 - tileColumnCountLog2; + int tileWidthSuperblocks = Av1Math.DivideLog2Ceiling(superblockColumns, tileColumnCountLog2); + int tileHeightSuperblocks = Av1Math.DivideLog2Ceiling(superblockRows, tileRowCountLog2); ObuTileGroupHeader tiles = new() { HasUniformTileSpacing = true, - TileColumnCount = 1, - TileRowCount = 1, - TileSizeBytes = 4 + TileColumnCountLog2 = tileColumnCountLog2, + TileRowCountLog2 = tileRowCountLog2, + TileSizeBytes = sizeof(uint) }; - tiles.TileColumnStartModeInfo[1] = modeInfoColumnCount; - tiles.TileRowStartModeInfo[1] = modeInfoRowCount; + // Uniform tile boundaries are derived in superblock units. The terminal entries retain the exact + // visible mode-info dimensions so clipped right and bottom superblocks end at the frame boundary. + int tileColumn = 0; + for (int startSuperblock = 0; startSuperblock < superblockColumns; startSuperblock += tileWidthSuperblocks) + { + tiles.TileColumnStartModeInfo[tileColumn++] = startSuperblock << superblockShift; + } + + tiles.TileColumnStartModeInfo[tileColumn] = modeInfoColumnCount; + tiles.TileColumnCount = tileColumn; + + int tileRow = 0; + for (int startSuperblock = 0; startSuperblock < superblockRows; startSuperblock += tileHeightSuperblocks) + { + tiles.TileRowStartModeInfo[tileRow++] = startSuperblock << superblockShift; + } + + tiles.TileRowStartModeInfo[tileRow] = modeInfoRowCount; + tiles.TileRowCount = tileRow; + return tiles; + } + + private static ObuFrameHeader CreateFrameHeader( + int width, + int height, + int qIndex, + int effort, + ObuFrameType frameType) + { + int modeInfoColumnCount = 2 * ((width + 7) >> 3); + int modeInfoRowCount = 2 * ((height + 7) >> 3); + ObuTileGroupHeader tiles = CreateTileGroupHeader( + width, + height, + modeInfoColumnCount, + modeInfoRowCount, + effort); + ObuFrameHeader frameHeader = new() { - FrameType = ObuFrameType.KeyFrame, - ShowFrame = true, - ErrorResilientMode = true, - RefreshFrameFlags = byte.MaxValue, - DisableFrameEndUpdateCdf = true, - TransformMode = qIndex == 0 - ? Av1TransformMode.Only4x4 - : effort >= 6 ? Av1TransformMode.Select : Av1TransformMode.Largest, ModeInfoColumnCount = modeInfoColumnCount, ModeInfoRowCount = modeInfoRowCount, TilesInfo = tiles, @@ -160,9 +498,51 @@ internal static class Av1FrameEncoder } }; + ConfigureFrameHeader(frameHeader, qIndex, effort, frameType); + return frameHeader; + } + + /// + /// Restores every frame-varying encoder field while retaining the fixed geometry and syntax object graph. + /// + private static void ConfigureFrameHeader( + ObuFrameHeader frameHeader, + int qIndex, + int effort, + ObuFrameType frameType) + { + frameHeader.FrameType = frameType; + frameHeader.ShowFrame = true; + frameHeader.ErrorResilientMode = true; + frameHeader.RefreshFrameFlags = byte.MaxValue; + frameHeader.DisableFrameEndUpdateCdf = true; + frameHeader.ReferenceMode = ObuReferenceMode.SingleReference; + frameHeader.InterpolationFilter = Av1InterpolationFilter.Regular; + frameHeader.IsMotionModeSwitchable = false; + frameHeader.TransformMode = qIndex == 0 + ? Av1TransformMode.Only4x4 + : effort >= 6 ? Av1TransformMode.Select : Av1TransformMode.Largest; + + frameHeader.AllowScreenContentTools = false; + frameHeader.AllowIntraBlockCopy = false; + frameHeader.ForceIntegerMotionVector = false; + frameHeader.AllowHighPrecisionMotionVector = false; + if (frameType == ObuFrameType.InterFrame) + { + // Disabling screen-content tools makes force_integer_mv implicitly false. Lower-effort searches + // still stop at full pixels, but their vectors use the normal fractional-motion syntax. + frameHeader.AllowHighPrecisionMotionVector = effort >= 8; + frameHeader.InterpolationFilter = effort >= MinimumSwitchableInterpolationEffort + ? Av1InterpolationFilter.Switchable + : Av1InterpolationFilter.Regular; + } + frameHeader.QuantizationParameters.BaseQIndex = qIndex; Av1QuantizationLookup.UpdateFrameQuantizationState(frameHeader); + } + private static int GetTileBufferLength(int width, int height, ObuColorConfig colorConfig) + { // Libaom reserves 2.5 times the 32-sample-aligned native input for an all-intra output packet. // Counting the active planes directly retains that headroom without charging monochrome for unused chroma. int alignedWidth = Av1Math.AlignPowerOf2(width, OutputAlignmentLog2); @@ -179,17 +559,7 @@ internal static class Av1FrameEncoder long scaledInputLength = (sampleCount * sampleSize * AllIntraBufferScaleNumerator) / AllIntraBufferScaleDenominator; - int tileBufferLength = checked((int)Math.Max(MinimumCompressedFrameBufferLength, scaledInputLength)); - if (colorConfig.BitDepth == Av1BitDepth.EightBit) - { - EncodeByte(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, tileBufferLength, effort, encodeAlpha); - } - else - { - EncodeHighBitDepth(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, tileBufferLength, effort, encodeAlpha); - } - - return sequenceHeader; + return checked((int)Math.Max(MinimumCompressedFrameBufferLength, scaledInputLength)); } /// @@ -206,7 +576,10 @@ internal static class Av1FrameEncoder Av1EncoderFrame source, ObuColorConfig colorConfig) where TPixel : unmanaged, IPixel - => PrepareSource(configuration, image, source, colorConfig, false); + { + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + PrepareSource(configuration, image, sourceRectangle, source, colorConfig, false); + } /// /// Converts packed pixels directly into a high-bit-depth bordered AV1 source frame. @@ -222,11 +595,16 @@ internal static class Av1FrameEncoder Av1EncoderFrame source, ObuColorConfig colorConfig) where TPixel : unmanaged, IPixel - => PrepareSource(configuration, image, source, colorConfig, false); + { + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + PrepareSource(configuration, image, sourceRectangle, source, colorConfig, false); + } private static void EncodeByte( Configuration configuration, ImageFrame image, + Rectangle sourceRectangle, + Size frameSize, Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, @@ -238,28 +616,81 @@ internal static class Av1FrameEncoder { using Av1EncoderFrameBuffer source = new( configuration, - image.Width, - image.Height, - 8, + frameSize.Width, + frameSize.Height, + ByteSampleBitDepth, colorFormat, - chromaPositionX: 1, - chromaPositionY: 1); + chromaPositionX: CenteredChromaSamplePosition, + chromaPositionY: CenteredChromaSamplePosition); using Av1EncoderFrameBuffer reconstruction = new( configuration, - image.Width, - image.Height, - 8, + frameSize.Width, + frameSize.Height, + ByteSampleBitDepth, colorFormat, - chromaPositionX: 1, - chromaPositionY: 1); + chromaPositionX: CenteredChromaSamplePosition, + chromaPositionY: CenteredChromaSamplePosition); + + using Av1EncoderCoefficientBuffer coefficients = new( + configuration, + sequenceHeader, + frameSize.Width, + frameSize.Height); + + using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); + + Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace); + using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); + using Av1SymbolEncoder symbolEncoder = new( + configuration, + tileBufferLength, + frameHeader.QuantizationParameters.BaseQIndex, + updateCdf: !frameHeader.DisableCdfUpdate); + + using ObuWriter obuWriter = new(configuration); + + PrepareFrame( + configuration, + image, + sourceRectangle, + source.Frame, + reconstruction.Frame, + sequenceHeader, + frameHeader, + effort, + encodeAlpha); + + using Av1EncoderPictureBuffer picture = new( + configuration, + sequenceHeader, + frameHeader, + source.Frame.Width, + source.Frame.Height, + disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9); - Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, tileBufferLength, effort, encodeAlpha); + Encode( + obuWriter, + stream, + sequenceHeader, + frameHeader, + picture.Picture, + source, + reconstruction, + reconstruction, + coefficients, + tileWorkspace, + blockWorkspace, + symbolEncoder, + effort, + true); } private static void EncodeHighBitDepth( Configuration configuration, ImageFrame image, + Rectangle sourceRectangle, + Size frameSize, Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, @@ -272,167 +703,348 @@ internal static class Av1FrameEncoder int bitDepth = sequenceHeader.ColorConfig.BitDepth.GetBitCount(); using Av1EncoderFrameBuffer source = new( configuration, - image.Width, - image.Height, + frameSize.Width, + frameSize.Height, bitDepth, colorFormat, - chromaPositionX: 1, - chromaPositionY: 1); + chromaPositionX: CenteredChromaSamplePosition, + chromaPositionY: CenteredChromaSamplePosition); using Av1EncoderFrameBuffer reconstruction = new( configuration, - image.Width, - image.Height, + frameSize.Width, + frameSize.Height, bitDepth, colorFormat, - chromaPositionX: 1, - chromaPositionY: 1); + chromaPositionX: CenteredChromaSamplePosition, + chromaPositionY: CenteredChromaSamplePosition); - Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, tileBufferLength, effort, encodeAlpha); - } + using Av1EncoderCoefficientBuffer coefficients = new( + configuration, + sequenceHeader, + frameSize.Width, + frameSize.Height); - private static void Encode( - Configuration configuration, - ImageFrame image, - Stream stream, - ObuSequenceHeader sequenceHeader, - ObuFrameHeader frameHeader, - Av1EncoderFrameBuffer source, - Av1EncoderFrameBuffer reconstruction, - int tileBufferLength, - int effort, - bool encodeAlpha) - where TPixel : unmanaged, IPixel - { - PrepareSource( + using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); + + Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace); + using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); + using Av1SymbolEncoder symbolEncoder = new( + configuration, + tileBufferLength, + frameHeader.QuantizationParameters.BaseQIndex, + updateCdf: !frameHeader.DisableCdfUpdate); + + using ObuWriter obuWriter = new(configuration); + + PrepareFrame( configuration, image, + sourceRectangle, source.Frame, - sequenceHeader.ColorConfig, + reconstruction.Frame, + sequenceHeader, + frameHeader, + effort, encodeAlpha); - bool allowScreenContentTools = false; - bool allowIntraBlockCopy = false; - if (effort >= 5) - { - // Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan. - Av1ScreenContentDetector.Detect( - source.Frame, - out allowScreenContentTools, - out allowIntraBlockCopy); - } - - frameHeader.AllowScreenContentTools = allowScreenContentTools; - - // The current IBC search is specialized for one 8x8 transform. Coded lossless requires reversible 4x4 - // transforms, so retain palette search but omit this candidate until it has a matching tiled implementation. - frameHeader.AllowIntraBlockCopy = !frameHeader.CodedLossless && allowIntraBlockCopy; using Av1EncoderPictureBuffer picture = new( configuration, sequenceHeader, frameHeader, - image.Width, - image.Height, + source.Frame.Width, + source.Frame.Height, disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9); - using Av1EncoderCoefficientBuffer coefficients = new( - configuration, + Encode( + obuWriter, + stream, sequenceHeader, - image.Width, - image.Height); - - using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); - using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); - using Av1SymbolEncoder symbolEncoder = new( - configuration, - tileBufferLength, - frameHeader.QuantizationParameters.BaseQIndex, - updateCdf: !frameHeader.DisableCdfUpdate); - - Av1IntraTileWriter tileWriter = new( - symbolEncoder, - source.Frame, - reconstruction.Frame, + frameHeader, picture.Picture, + source, + reconstruction, + reconstruction, coefficients, - superblockWorkspace, + tileWorkspace, blockWorkspace, - effort); - - ObuWriter writer = new(); - writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter); + symbolEncoder, + effort, + true); } - private static void Encode( + /// + /// Converts one source frame and resolves every content-dependent coding tool before picture-state allocation. + /// + private static void PrepareFrame( Configuration configuration, ImageFrame image, - Stream stream, + Rectangle sourceRectangle, + Av1EncoderFrame source, + Av1EncoderFrame reference, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, - Av1EncoderFrameBuffer source, - Av1EncoderFrameBuffer reconstruction, - int tileBufferLength, int effort, bool encodeAlpha) where TPixel : unmanaged, IPixel { - PrepareSource( + PrepareSource( configuration, image, - source.Frame, + sourceRectangle, + source, sequenceHeader.ColorConfig, encodeAlpha); + ConfigureFrameTools( + source, + reference, + sequenceHeader, + frameHeader, + effort); + } + + /// + /// Converts one sequence sample through its retained row workspace before resolving frame coding tools. + /// + private static void PrepareFrame( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int effort, + Av1EncoderConversionWorkspace conversionWorkspace) + where TPixel : unmanaged, IPixel + { + PrepareSource( + configuration, + image, + sourceRectangle, + source, + conversionWorkspace); + + ConfigureFrameTools( + source, + reference, + sequenceHeader, + frameHeader, + effort); + } + + /// + /// Resolves the eight-bit frame tools whose syntax depends on the converted source samples. + /// + private static void ConfigureFrameTools( + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int effort) + { + ConfigureGlobalMotion( + source, + reference, + frameHeader, + sequenceHeader.ColorConfig.BitDepth, + effort); + bool allowScreenContentTools = false; bool allowIntraBlockCopy = false; if (effort >= 5) { - // Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan. + // Lower effort levels never search palette or intra-block-copy modes, so scanning the complete + // luma plane cannot affect their bitstream decisions. Av1ScreenContentDetector.Detect( - source.Frame, + source, out allowScreenContentTools, out allowIntraBlockCopy); } frameHeader.AllowScreenContentTools = allowScreenContentTools; - // The current IBC search is specialized for one 8x8 transform. Coded lossless requires reversible 4x4 - // transforms, so retain palette search but omit this candidate until it has a matching tiled implementation. - frameHeader.AllowIntraBlockCopy = !frameHeader.CodedLossless && allowIntraBlockCopy; - using Av1EncoderPictureBuffer picture = new( + // The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible + // 4x4 transforms, so palette remains available while this incompatible candidate is omitted. + frameHeader.AllowIntraBlockCopy = + frameHeader.IsIntra && + !frameHeader.CodedLossless && + allowIntraBlockCopy; + } + + /// + /// Converts one high-bit-depth source frame and resolves every content-dependent coding tool before picture-state allocation. + /// + private static void PrepareFrame( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int effort, + bool encodeAlpha) + where TPixel : unmanaged, IPixel + { + PrepareSource( configuration, + image, + sourceRectangle, + source, + sequenceHeader.ColorConfig, + encodeAlpha); + + ConfigureFrameTools( + source, + reference, sequenceHeader, frameHeader, - image.Width, - image.Height, - disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9); + effort); + } - using Av1EncoderCoefficientBuffer coefficients = new( + /// + /// Converts one high-bit-depth sequence sample through retained row storage before resolving frame coding tools. + /// + private static void PrepareFrame( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int effort, + Av1EncoderConversionWorkspace conversionWorkspace) + where TPixel : unmanaged, IPixel + { + PrepareSource( configuration, + image, + sourceRectangle, + source, + conversionWorkspace); + + ConfigureFrameTools( + source, + reference, sequenceHeader, - image.Width, - image.Height); + frameHeader, + effort); + } - using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); - using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); - using Av1SymbolEncoder symbolEncoder = new( - configuration, - tileBufferLength, - frameHeader.QuantizationParameters.BaseQIndex, - updateCdf: !frameHeader.DisableCdfUpdate); + /// + /// Resolves the high-bit-depth frame tools whose syntax depends on the converted source samples. + /// + private static void ConfigureFrameTools( + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int effort) + { + ConfigureGlobalMotion( + source, + reference, + frameHeader, + sequenceHeader.ColorConfig.BitDepth, + effort); - Av1IntraTileWriter tileWriter = new( + bool allowScreenContentTools = false; + bool allowIntraBlockCopy = false; + if (effort >= 5) + { + // Lower effort levels never search palette or intra-block-copy modes, so scanning the complete + // luma plane cannot affect their bitstream decisions. + Av1ScreenContentDetector.Detect( + source, + out allowScreenContentTools, + out allowIntraBlockCopy); + } + + frameHeader.AllowScreenContentTools = allowScreenContentTools; + + // The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible + // 4x4 transforms, so palette remains available while this incompatible candidate is omitted. + frameHeader.AllowIntraBlockCopy = + frameHeader.IsIntra && + !frameHeader.CodedLossless && + allowIntraBlockCopy; + } + + private static void Encode( + ObuWriter obuWriter, + Stream stream, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1PictureControlSet picture, + Av1EncoderFrameBuffer source, + Av1EncoderFrameBuffer reference, + Av1EncoderFrameBuffer reconstruction, + Av1EncoderCoefficientBuffer coefficients, + Av1EncoderTileWorkspace tileWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + Av1SymbolEncoder symbolEncoder, + int effort, + bool writeSequenceHeader) + { + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, + reference.Frame, reconstruction.Frame, - picture.Picture, + picture, coefficients, - superblockWorkspace, + tileWorkspace, blockWorkspace, effort); - ObuWriter writer = new(); - writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter); + if (writeSequenceHeader) + { + obuWriter.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileWriter); + } + else + { + obuWriter.WriteFrame(stream, sequenceHeader, frameHeader, tileWriter); + } + } + + private static void Encode( + ObuWriter obuWriter, + Stream stream, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1PictureControlSet picture, + Av1EncoderFrameBuffer source, + Av1EncoderFrameBuffer reference, + Av1EncoderFrameBuffer reconstruction, + Av1EncoderCoefficientBuffer coefficients, + Av1EncoderTileWorkspace tileWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + Av1SymbolEncoder symbolEncoder, + int effort, + bool writeSequenceHeader) + { + Av1TileEncoder tileWriter = new( + symbolEncoder, + source.Frame, + reference.Frame, + reconstruction.Frame, + picture, + coefficients, + tileWorkspace, + blockWorkspace, + effort); + + if (writeSequenceHeader) + { + obuWriter.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileWriter); + } + else + { + obuWriter.WriteFrame(stream, sequenceHeader, frameHeader, tileWriter); + } } /// @@ -441,6 +1053,7 @@ internal static class Av1FrameEncoder private static void PrepareSource( Configuration configuration, ImageFrame image, + Rectangle sourceRectangle, Av1EncoderFrame source, ObuColorConfig colorConfig, bool encodeAlpha) @@ -448,6 +1061,10 @@ internal static class Av1FrameEncoder where TSample : unmanaged where TStorer : struct, IHeifSampleConverter { + Av1EncoderFrame.PlanarView destination = source.CodedView.GetSubView( + sourceRectangle.Width, + sourceRectangle.Height); + if (encodeAlpha) { HeifPlanarAlphaEncoder.Convert< @@ -457,7 +1074,8 @@ internal static class Av1FrameEncoder TStorer>( configuration, image, - source.View); + sourceRectangle, + destination); } else { @@ -474,11 +1092,791 @@ internal static class Av1FrameEncoder TStorer>( configuration, image, - source.View, + sourceRectangle, + destination, in parameters, mode); } - source.ExtendBorders(); + // A short grid edge may occupy only the top-left of its AV1 frame. Replicating that edge initializes + // both the remaining coded cell and the physical prediction border without another image or plane copy. + source.CodedView.ExtendBorders(sourceRectangle.Width, sourceRectangle.Height); + } + + /// + /// Converts one sequence sample with track-owned row storage and initializes every coded and physical edge. + /// + private static void PrepareSource( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + Av1EncoderFrame source, + Av1EncoderConversionWorkspace conversionWorkspace) + where TPixel : unmanaged, IPixel + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter + { + Av1EncoderFrame.PlanarView destination = source.CodedView.GetSubView( + sourceRectangle.Width, + sourceRectangle.Height); + + conversionWorkspace.Convert< + TPixel, + Av1EncoderFrame.PlanarView, + TSample, + TStorer>( + configuration, + image, + sourceRectangle, + destination); + + // Sequence geometry is fixed, but grid-edge cells can still expose less source data than their coded + // extent. The same edge replication completes both coded padding and the physical prediction border. + source.CodedView.ExtendBorders(sourceRectangle.Width, sourceRectangle.Height); + } + + /// + /// Selects a bounded whole-frame translation model for an inter frame. + /// + private static void ConfigureGlobalMotion( + Av1EncoderFrame source, + Av1EncoderFrame reference, + ObuFrameHeader frameHeader, + Av1BitDepth bitDepth, + int effort) + where TSample : unmanaged + where TOperator : struct, IGlobalMotionSearchOperator + { + Span models = frameHeader.GetGlobalMotionParameters(); + models.Fill(Av1GlobalMotionParameters.Identity); + if (frameHeader.IsIntra || effort < MinimumGlobalMotionSearchEffort) + { + return; + } + + Buffer2DRegion sourceLuma = source.CodedView.GetPlane(Av1Plane.Y); + Buffer2DRegion referenceLuma = reference.CodedView.GetPlane(Av1Plane.Y); + int analysisWidth = Math.Min(source.CodedWidth, MaximumGlobalMotionAnalysisDimension); + int analysisHeight = Math.Min(source.CodedHeight, MaximumGlobalMotionAnalysisDimension); + Point analysisOrigin = new( + (source.CodedWidth - analysisWidth) >> 1, + (source.CodedHeight - analysisHeight) >> 1); + + int effortShift = effort - MinimumGlobalMotionSearchEffort; + int searchRadius = Math.Min( + MinimumGlobalMotionSearchRadius << effortShift, + Av1EncoderFrame.LumaBorder); + + Point bestOffset = default; + long bestAnalysisError = GetGlobalMotionSquaredError( + sourceLuma, + referenceLuma, + analysisOrigin, + analysisWidth, + analysisHeight, + bestOffset); + + for (int step = searchRadius; step > 0; step >>= 1) + { + Point stageBestOffset = bestOffset; + long stageBestError = bestAnalysisError; + for (int directionIndex = 0; directionIndex < GlobalMotionSearchDirectionCount; directionIndex++) + { + Point direction = GetGlobalMotionSearchDirection(directionIndex); + Point candidateOffset = new( + bestOffset.X + (direction.X * step), + bestOffset.Y + (direction.Y * step)); + + if (Math.Abs(candidateOffset.X) > searchRadius || + Math.Abs(candidateOffset.Y) > searchRadius) + { + continue; + } + + long directionError = GetGlobalMotionSquaredError( + sourceLuma, + referenceLuma, + analysisOrigin, + analysisWidth, + analysisHeight, + candidateOffset); + + // Strict replacement preserves identity and the earlier reference search order on ties. + if (directionError < stageBestError) + { + stageBestError = directionError; + stageBestOffset = candidateOffset; + } + } + + bestOffset = stageBestOffset; + bestAnalysisError = stageBestError; + } + + if (bestOffset == default) + { + return; + } + + Point frameOrigin = default; + long identityError = GetGlobalMotionSquaredError( + sourceLuma, + referenceLuma, + frameOrigin, + source.CodedWidth, + source.CodedHeight, + frameOrigin); + + long candidateError = GetGlobalMotionSquaredError( + sourceLuma, + referenceLuma, + frameOrigin, + source.CodedWidth, + source.CodedHeight, + bestOffset); + + Av1GlobalMotionParameters candidate = Av1GlobalMotionParameters.Identity; + + // Pure translation is stored as identity-scale rotation/zoom because the translation-only AV1 model + // has a published row/column assignment defect. The resulting block vector remains exact. + candidate.Type = Av1GlobalMotionType.RotationZoom; + candidate[0] = bestOffset.X * Av1GlobalMotionParameters.ModelScale; + candidate[1] = bestOffset.Y * Av1GlobalMotionParameters.ModelScale; + candidate.UpdateShearParameters(); + + int rateMultiplier = Av1RateDistortion.GetInterFrameRateMultiplier( + frameHeader.QuantizationParameters.BaseQIndex, + bitDepth); + + int identityRate = + ObuWriter.GetGlobalMotionModelBitCount( + Av1GlobalMotionParameters.Identity, + frameHeader.AllowHighPrecisionMotionVector) << + Av1ProbabilityCost.CostShift; + + int candidateRate = + ObuWriter.GetGlobalMotionModelBitCount( + candidate, + frameHeader.AllowHighPrecisionMotionVector) << + Av1ProbabilityCost.CostShift; + + long identityCost = Av1RateDistortion.GetCost( + rateMultiplier, + identityRate, + NormalizeGlobalMotionSquaredError(identityError, bitDepth)); + + long candidateCost = Av1RateDistortion.GetCost( + rateMultiplier, + candidateRate, + NormalizeGlobalMotionSquaredError(candidateError, bitDepth)); + + if (candidateCost < identityCost) + { + models[0] = candidate; + } + } + + /// + /// Calculates squared error for one translated luma candidate using the physical reference border. + /// + private static long GetGlobalMotionSquaredError( + Buffer2DRegion source, + Buffer2DRegion reference, + Point sourceOrigin, + int width, + int height, + Point referenceOffset) + where TSample : unmanaged + where TOperator : struct, IGlobalMotionSearchOperator + { + Rectangle sourceBounds = source.Bounds; + Rectangle referenceBounds = reference.Bounds; + int sourceIndex = + ((sourceBounds.Y + sourceOrigin.Y) * source.Stride) + + sourceBounds.X + + sourceOrigin.X; + + int referenceIndex = + ((referenceBounds.Y + sourceOrigin.Y + referenceOffset.Y) * reference.Stride) + + referenceBounds.X + + sourceOrigin.X + + referenceOffset.X; + + return TOperator.SumSquaredError( + source.Buffer.DangerousGetSingleSpan()[sourceIndex..], + source.Stride, + reference.Buffer.DangerousGetSingleSpan()[referenceIndex..], + reference.Stride, + width, + height); + } + + /// + /// Gets one cardinal or diagonal direction in the reference encoder's search order. + /// + private static Point GetGlobalMotionSearchDirection(int index) + => index switch + { + 0 => new Point(0, -1), + 1 => new Point(0, 1), + 2 => new Point(-1, 0), + 3 => new Point(1, 0), + 4 => new Point(-1, -1), + 5 => new Point(1, 1), + 6 => new Point(1, -1), + _ => new Point(-1, 1) + }; + + /// + /// Normalizes high-bit-depth frame error to the eight-bit distortion domain. + /// + private static long NormalizeGlobalMotionSquaredError(long error, Av1BitDepth bitDepth) + { + int shift = (bitDepth.GetBitCount() - ByteSampleBitDepth) * 2; + return shift == 0 ? error : (error + (1L << (shift - 1))) >> shift; + } + + /// + /// Routes byte samples through the SIMD-first shared residual operation. + /// + private readonly struct ByteGlobalMotionSearchOperator : IGlobalMotionSearchOperator + { + /// + public static long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height) + => Av1ResidualBuilder.SumSquaredError( + source, + sourceStride, + prediction, + predictionStride, + width, + height); + } + + /// + /// Routes high-bit-depth samples through the SIMD-first shared residual operation. + /// + private readonly struct UInt16GlobalMotionSearchOperator : IGlobalMotionSearchOperator + { + /// + public static long SumSquaredError( + ReadOnlySpan source, + int sourceStride, + ReadOnlySpan prediction, + int predictionStride, + int width, + int height) + => Av1ResidualBuilder.SumSquaredError( + source, + sourceStride, + prediction, + predictionStride, + width, + height); + } + + /// + /// Owns the fixed-size packed and planar row storage reused by every sample in one sequence track. + /// + internal sealed class Av1EncoderConversionWorkspace : IDisposable + { + private readonly IMemoryOwner storageOwner; + private readonly Memory componentMemory; + private readonly Memory packedMemory; + private readonly HeifColorConversionParameters parameters; + private readonly HeifColorConverterBase colorConverter; + private readonly bool encodeAlpha; + private readonly int packedPixelCount; + + /// + /// Initializes a new instance of the class. + /// + /// The configuration providing reusable row storage. + /// The fixed sequence width. + /// The native component layout. + /// Whether the workspace converts the auxiliary alpha track. + /// Whether color conversion requires a packed row. + public Av1EncoderConversionWorkspace( + Configuration configuration, + int width, + ObuColorConfig colorConfig, + bool encodeAlpha, + bool usesHighBitDepth) + { + int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; + int componentLength = encodeAlpha + ? HeifPlanarAlphaEncoder.GetRowStorageLength(width) + : HeifPlanarColorConverter.GetRgbToYuvComponentBufferLength( + width, + colorConfig.IsMonochrome, + subsamplingY); + + int packedByteLength = usesHighBitDepth && !encodeAlpha + ? width * Unsafe.SizeOf() + : 0; + + int packedFloatLength = (int)Numerics.DivideCeil((uint)packedByteLength, sizeof(float)); + this.storageOwner = configuration.MemoryAllocator.Allocate( + componentLength + packedFloatLength); + + Memory storage = this.storageOwner.Memory; + this.componentMemory = storage[..componentLength]; + this.packedMemory = storage.Slice(componentLength, packedFloatLength); + this.parameters = Av1YuvConverter.GetConversionParameters(colorConfig, out HeifColorConversionMode mode); + this.colorConverter = HeifColorConverterBase.Create(mode, in this.parameters, colorConfig.IsMonochrome); + this.encodeAlpha = encodeAlpha; + this.packedPixelCount = usesHighBitDepth && !encodeAlpha ? width : 0; + } + + /// + /// Converts one packed frame directly into its final native component planes. + /// + /// The packed source pixel type. + /// The native destination-plane adapter. + /// The native sample storage type. + /// The SIMD narrowing and storage operation. + /// The configuration used for packed-pixel conversion. + /// The source image frame. + /// The source region mapped to the destination planes. + /// The destination component planes. + public void Convert( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + TBuffer buffer) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter + { + if (this.encodeAlpha) + { + HeifPlanarAlphaEncoder.Convert( + configuration, + image, + sourceRectangle, + buffer, + this.componentMemory.Span); + + return; + } + + Span packed = MemoryMarshal.Cast( + this.packedMemory.Span)[..this.packedPixelCount]; + + HeifPlanarColorConverter.ConvertFromRgb( + configuration, + image, + sourceRectangle, + buffer, + in this.parameters, + this.colorConverter, + packed, + this.componentMemory.Span); + } + + /// + public void Dispose() => this.storageOwner.Dispose(); + } + + /// + /// Retains the reconstructed reference state shared by the samples of one AV1 sequence track. + /// + internal abstract class SequenceEncoder : IDisposable + { + protected SequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort, + bool encodeAlpha, + bool usesHighBitDepth) + { + this.Configuration = configuration; + this.SequenceHeader = CreateSequenceHeader(width, height, colorConfig, effort, false); + this.QIndex = qIndex; + this.Effort = effort; + this.TileBufferLength = GetTileBufferLength(width, height, colorConfig); + this.EncodeAlpha = encodeAlpha; + this.FrameHeader = CreateFrameHeader( + width, + height, + qIndex, + effort, + ObuFrameType.KeyFrame); + + this.ConversionWorkspace = new Av1EncoderConversionWorkspace( + configuration, + width, + colorConfig, + encodeAlpha, + usesHighBitDepth); + + bool allocateScreenContentState = effort >= 5; + bool allocateIntraBlockCopySearch = + allocateScreenContentState && + !this.FrameHeader.CodedLossless; + + // Sequence geometry and maximum tool capacity are fixed before the first sample. Reusing this owner + // avoids renting the complete mode grid and optional screen-content index for every frame. + this.PictureBuffer = new Av1EncoderPictureBuffer( + configuration, + this.SequenceHeader, + this.FrameHeader, + width, + height, + disallow4x4AllFrames: !this.FrameHeader.CodedLossless && effort < 9, + allocateScreenContentState: allocateScreenContentState, + allocateMotionVectorState: true, + allocateIntraBlockCopySearch: allocateIntraBlockCopySearch); + + this.Coefficients = new Av1EncoderCoefficientBuffer( + configuration, + this.SequenceHeader, + width, + height); + + this.SuperblockWorkspace = new Av1EncoderSuperblockWorkspace(configuration); + + this.TileWorkspace = new Av1EncoderTileWorkspace(this.FrameHeader, this.SuperblockWorkspace); + this.BlockWorkspace = new Av1EncoderBlockWorkspace(configuration); + + // Tile probabilities adapt within a sample, while error-resilient frame headers prohibit carrying + // those updates into the next sample. The retained encoder is therefore reset before each frame. + this.SymbolEncoder = new Av1SymbolEncoder( + configuration, + this.TileBufferLength, + qIndex, + updateCdf: true); + + this.ObuWriter = new ObuWriter(configuration); + } + + /// + /// Gets the sequence header shared by every sample written by this encoder. + /// + public ObuSequenceHeader SequenceHeader { get; } + + protected Configuration Configuration { get; } + + protected int QIndex { get; } + + protected int Effort { get; } + + protected int TileBufferLength { get; } + + protected bool EncodeAlpha { get; } + + /// + /// Gets the packed and planar row storage reused by every sample in the track. + /// + protected Av1EncoderConversionWorkspace ConversionWorkspace { get; } + + /// + /// Gets the fixed-geometry picture state reused by every sample in the track. + /// + protected Av1EncoderPictureBuffer PictureBuffer { get; } + + /// + /// Gets the frame header and nested syntax state reused by every sample in the track. + /// + protected ObuFrameHeader FrameHeader { get; } + + /// + /// Gets the frame-sized coefficient storage reused by every sample in the track. + /// + protected Av1EncoderCoefficientBuffer Coefficients { get; } + + /// + /// Gets the superblock decision workspace reused serially across the track. + /// + protected Av1EncoderSuperblockWorkspace SuperblockWorkspace { get; } + + /// + /// Gets the tile, superblock, and entropy cursor graph reused serially across the track. + /// + protected Av1EncoderTileWorkspace TileWorkspace { get; } + + /// + /// Gets the block arithmetic workspace reused serially across the track. + /// + protected Av1EncoderBlockWorkspace BlockWorkspace { get; } + + /// + /// Gets the tile probability graph and bounded output buffer reused by every sample in the track. + /// + protected Av1SymbolEncoder SymbolEncoder { get; } + + /// + /// Gets the reusable OBU header writer for the track. + /// + protected ObuWriter ObuWriter { get; } + + /// + /// Encodes an independently decodable sample with the sequence header required for random access. + /// + public void EncodeKeyFrame(ImageFrame image, Stream stream) + where TPixel : unmanaged, IPixel + => this.EncodeFrame(image, stream, ObuFrameType.KeyFrame, true); + + /// + /// Encodes a continuation sample predicted from the preceding reconstructed frame. + /// + public void EncodeInterFrame(ImageFrame image, Stream stream) + where TPixel : unmanaged, IPixel + => this.EncodeFrame(image, stream, ObuFrameType.InterFrame, false); + + /// + public void Dispose() + { + this.DisposeFrames(); + this.ConversionWorkspace.Dispose(); + this.PictureBuffer.Dispose(); + this.Coefficients.Dispose(); + this.SuperblockWorkspace.Dispose(); + this.BlockWorkspace.Dispose(); + this.ObuWriter.Dispose(); + this.SymbolEncoder.Dispose(); + } + + /// + /// Releases the sample-type-specific frame buffers retained by the track encoder. + /// + protected abstract void DisposeFrames(); + + protected abstract void EncodeFrame( + ImageFrame image, + Stream stream, + ObuFrameType frameType, + bool writeSequenceHeader) + where TPixel : unmanaged, IPixel; + } + + private sealed class ByteSequenceEncoder : SequenceEncoder + { + private readonly Av1EncoderFrameBuffer source; + private Av1EncoderFrameBuffer reference; + private Av1EncoderFrameBuffer reconstruction; + + public ByteSequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort, + bool encodeAlpha) + : base( + configuration, + width, + height, + colorConfig, + qIndex, + effort, + encodeAlpha, + usesHighBitDepth: false) + { + Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); + this.source = new( + configuration, + width, + height, + ByteSampleBitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + + this.reference = new( + configuration, + width, + height, + ByteSampleBitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + + this.reconstruction = new( + configuration, + width, + height, + ByteSampleBitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + } + + protected override void DisposeFrames() + { + this.source.Dispose(); + this.reference.Dispose(); + this.reconstruction.Dispose(); + } + + protected override void EncodeFrame( + ImageFrame image, + Stream stream, + ObuFrameType frameType, + bool writeSequenceHeader) + { + ObuFrameHeader frameHeader = this.FrameHeader; + ConfigureFrameHeader( + frameHeader, + this.QIndex, + this.Effort, + frameType); + + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + this.SymbolEncoder.Reset(); + PrepareFrame( + this.Configuration, + image, + sourceRectangle, + this.source.Frame, + this.reference.Frame, + this.SequenceHeader, + frameHeader, + this.Effort, + this.ConversionWorkspace); + + this.PictureBuffer.Reset(frameHeader); + Encode( + this.ObuWriter, + stream, + this.SequenceHeader, + frameHeader, + this.PictureBuffer.Picture, + this.source, + this.reference, + this.reconstruction, + this.Coefficients, + this.TileWorkspace, + this.BlockWorkspace, + this.SymbolEncoder, + this.Effort, + writeSequenceHeader); + + this.reconstruction.Frame.ExtendBorders(); + + // The just-reconstructed frame becomes LAST_FRAME for the next sample without copying any plane. + (this.reference, this.reconstruction) = (this.reconstruction, this.reference); + } + } + + private sealed class HighBitDepthSequenceEncoder : SequenceEncoder + { + private readonly Av1EncoderFrameBuffer source; + private Av1EncoderFrameBuffer reference; + private Av1EncoderFrameBuffer reconstruction; + + public HighBitDepthSequenceEncoder( + Configuration configuration, + int width, + int height, + ObuColorConfig colorConfig, + int qIndex, + int effort, + bool encodeAlpha) + : base( + configuration, + width, + height, + colorConfig, + qIndex, + effort, + encodeAlpha, + usesHighBitDepth: true) + { + int bitDepth = colorConfig.BitDepth.GetBitCount(); + Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); + this.source = new( + configuration, + width, + height, + bitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + + this.reference = new( + configuration, + width, + height, + bitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + + this.reconstruction = new( + configuration, + width, + height, + bitDepth, + colorFormat, + CenteredChromaSamplePosition, + CenteredChromaSamplePosition); + } + + protected override void DisposeFrames() + { + this.source.Dispose(); + this.reference.Dispose(); + this.reconstruction.Dispose(); + } + + protected override void EncodeFrame( + ImageFrame image, + Stream stream, + ObuFrameType frameType, + bool writeSequenceHeader) + { + ObuFrameHeader frameHeader = this.FrameHeader; + ConfigureFrameHeader( + frameHeader, + this.QIndex, + this.Effort, + frameType); + + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + this.SymbolEncoder.Reset(); + PrepareFrame( + this.Configuration, + image, + sourceRectangle, + this.source.Frame, + this.reference.Frame, + this.SequenceHeader, + frameHeader, + this.Effort, + this.ConversionWorkspace); + + this.PictureBuffer.Reset(frameHeader); + Encode( + this.ObuWriter, + stream, + this.SequenceHeader, + frameHeader, + this.PictureBuffer.Picture, + this.source, + this.reference, + this.reconstruction, + this.Coefficients, + this.TileWorkspace, + this.BlockWorkspace, + this.SymbolEncoder, + this.Effort, + writeSequenceHeader); + + this.reconstruction.Frame.ExtendBorders(); + + // Swapping the frame owners preserves the complete reconstructed reference, including extended borders. + (this.reference, this.reconstruction) = (this.reconstruction, this.reference); + } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs deleted file mode 100644 index cb21938f9e..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs +++ /dev/null @@ -1,586 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; -using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; -using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; -using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; -using SixLabors.ImageSharp.Memory; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; - -/// -/// Provides full rate-distortion selection for intra-block-copy candidates. -/// -internal static partial class Av1IntraSuperblockEncoder -{ - internal partial struct ModeDecision - where TSample : unmanaged - where TOperator : struct, IBlockEncodingOperator - { - private long SelectIntraBlockCopy( - Av1SymbolEncoder writer, - Av1MacroBlockD macroBlock, - Point blockOrigin, - ushort tileIndex, - long regularCost, - ref Av1MacroBlockModeInfo modeInfo, - ref Av1EncoderBlockStruct block, - ref Av1EncoderPaletteInfo paletteInfo) - { - const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; - const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8; - Buffer2DRegion lumaSource = this.source.GetPlane(Av1Plane.Y); - Buffer2DRegion lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y); - Point modeInfoPosition = new( - blockOrigin.X >> Av1Constants.ModeInfoSizeLog2, - blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2); - - Span referenceCandidates = stackalloc Av1MotionVector[8]; - Span referenceWeights = stackalloc int[8]; - Av1MotionVector reference = Av1IntraBlockCopy.FindReference( - this.picture, - macroBlock, - modeInfoPosition, - BlockSize, - Av1PartitionType.None, - referenceCandidates, - referenceWeights); - - Span candidates = stackalloc Av1MotionVector[4]; - Av1IntraBlockCopySearchIndex search = this.picture.IntraBlockCopySearch; - int candidateCount = search.FindCandidates( - lumaSource, - lumaReconstruction, - blockOrigin, - macroBlock.Tile, - this.picture.Sequence.SequenceHeader, - writer, - reference, - this.rateMultiplier, - candidates); - - candidateCount += search.FindPixelCandidates( - lumaSource, - lumaReconstruction, - blockOrigin, - macroBlock.Tile, - this.picture.Sequence.SequenceHeader, - writer, - reference, - this.quantization.QIndex[0], - this.rateMultiplier, - candidates[candidateCount..]); - - // Hash and full-pixel searches can converge on the same vector. Preserve the first search-order - // occurrence so repeated vectors do not pay for duplicate transform searches or alter ties. - int uniqueCandidateCount = 0; - for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++) - { - Av1MotionVector candidate = candidates[candidateIndex]; - bool duplicate = false; - for (int uniqueIndex = 0; uniqueIndex < uniqueCandidateCount; uniqueIndex++) - { - if (candidate == candidates[uniqueIndex]) - { - duplicate = true; - break; - } - } - - if (!duplicate) - { - candidates[uniqueCandidateCount++] = candidate; - } - } - - if (uniqueCandidateCount == 0) - { - return regularCost; - } - - int skipContext = Av1TileWriter.GetSkipContext(macroBlock); - long bestCost = regularCost; - bool hasSelectedCandidate = false; - bool selectedSkip = false; - Av1MotionVector selectedVector = default; - Av1EncoderTransformBlockState selectedLumaState = default; - Av1EncoderTransformBlockState selectedBlueState = default; - Av1EncoderTransformBlockState selectedRedState = default; - Av1EncoderIntraBlockCopyWorkspace workspace = - this.blockWorkspace.GetIntraBlockCopyWorkspace(); - - Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts( - Av1ComponentType.Luminance, - this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex], - blockOrigin, - BlockSize, - LumaTransformSize); - - // A coded IBC residual uses the unsplit transform root at this fixed block size. A skipped block - // omits both the transform-partition bit and coefficient syntax, so this rate is added only below. - int transformPartitionRate = 0; - if (this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select) - { - Av1NeighborArrayUnit transformContexts = this.picture.TransformFunctionContexts[tileIndex]; - int topIndex = transformContexts.GetTopIndex(blockOrigin); - int leftIndex = transformContexts.GetLeftIndex(blockOrigin); - int transformPartitionContext = Av1SymbolContextHelper.GetTransformPartitionContext( - transformContexts.Top[topIndex], - transformContexts.Left[leftIndex], - BlockSize, - LumaTransformSize); - - transformPartitionRate = writer.GetTransformPartitionCost( - false, - transformPartitionContext); - } - - ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig; - int subsamplingX = colorConfig.SubSamplingX ? 1 : 0; - int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; - Point chromaOrigin = new(blockOrigin.X >> subsamplingX, blockOrigin.Y >> subsamplingY); - Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize( - colorConfig.SubSamplingX, - colorConfig.SubSamplingY); - - Av1TransformBlockContext blueContext = default; - Av1TransformBlockContext redContext = default; - if (!this.source.IsMonochrome) - { - Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled( - colorConfig.SubSamplingX, - colorConfig.SubSamplingY); - - blueContext = Av1TileWriter.GetTransformBlockContexts( - Av1ComponentType.Chroma, - this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex], - chromaOrigin, - chromaBlockSize, - chromaTransformSize); - - redContext = Av1TileWriter.GetTransformBlockContexts( - Av1ComponentType.Chroma, - this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex], - chromaOrigin, - chromaBlockSize, - chromaTransformSize); - } - - // Per-vector plane results reuse candidate scratch. Separate selected spans retain only a new - // global winner, allowing the complete search to finish before committed reconstruction changes. - for (int candidateIndex = 0; candidateIndex < uniqueCandidateCount; candidateIndex++) - { - Av1MotionVector candidate = candidates[candidateIndex]; - this.EvaluateIntraBlockCopyPlane( - writer, - candidate, - Av1Plane.Y, - Av1ComponentType.Luminance, - blockOrigin, - 0, - 0, - LumaTransformSize, - Av1TransformType.AllTransformTypes, - lumaContext, - workspace.LumaPrediction, - workspace.Residual, - workspace.TransformReconstruction, - workspace.TransformCoefficients, - workspace.LumaCandidateReconstruction, - workspace.LumaCandidateCoefficients, - out Av1EncoderTransformBlockState lumaCandidateState, - out int lumaRate, - out long lumaDistortion, - out bool hasEmptyLuma, - out Av1EncoderTransformBlockState emptyLumaState, - out long emptyLumaDistortion); - - int blueRate = 0; - int redRate = 0; - long blueDistortion = 0; - long redDistortion = 0; - long emptyBlueDistortion = 0; - long emptyRedDistortion = 0; - bool hasEmptyBlue = true; - bool hasEmptyRed = true; - Av1EncoderTransformBlockState blueCandidateState = default; - Av1EncoderTransformBlockState redCandidateState = default; - Av1EncoderTransformBlockState emptyBlueState = default; - Av1EncoderTransformBlockState emptyRedState = default; - if (!this.source.IsMonochrome) - { - Av1TransformType chromaTransformType = lumaCandidateState.TransformType; - Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType( - chromaTransformSize, - isInter: true, - this.picture.Parent.FrameHeader.UseReducedTransformSet); - - // Inter prediction does not signal an independent chroma transform type. Chroma reuses the - // selected luma type when that type belongs to its transform set and otherwise falls back to DCT. - if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet)) - { - chromaTransformType = Av1TransformType.DctDct; - } - - this.EvaluateIntraBlockCopyPlane( - writer, - candidate, - Av1Plane.U, - Av1ComponentType.Chroma, - blockOrigin, - subsamplingX, - subsamplingY, - chromaTransformSize, - chromaTransformType, - blueContext, - workspace.BluePrediction, - workspace.Residual, - workspace.TransformReconstruction, - workspace.TransformCoefficients, - workspace.BlueCandidateReconstruction, - workspace.BlueCandidateCoefficients, - out blueCandidateState, - out blueRate, - out blueDistortion, - out hasEmptyBlue, - out emptyBlueState, - out emptyBlueDistortion); - - this.EvaluateIntraBlockCopyPlane( - writer, - candidate, - Av1Plane.V, - Av1ComponentType.Chroma, - blockOrigin, - subsamplingX, - subsamplingY, - chromaTransformSize, - chromaTransformType, - redContext, - workspace.RedPrediction, - workspace.Residual, - workspace.TransformReconstruction, - workspace.TransformCoefficients, - workspace.RedCandidateReconstruction, - workspace.RedCandidateCoefficients, - out redCandidateState, - out redRate, - out redDistortion, - out hasEmptyRed, - out emptyRedState, - out emptyRedDistortion); - } - - int displacementRate = writer.GetDisplacementVectorCost(candidate, reference); - int candidateRate = writer.GetUseIntraBlockCopyCost(true) + - displacementRate + - writer.GetSkipCost(false, skipContext) + - transformPartitionRate + - lumaRate + - blueRate + - redRate; - - long candidateDistortion = lumaDistortion + blueDistortion + redDistortion; - long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, candidateDistortion); - bool candidateSkip = false; - - // The skip alternative is available only when every coded plane has an empty transform. Its - // distortion comes from prediction alone and its rate excludes the transform tree and coefficients. - if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed) - { - int skipRate = writer.GetUseIntraBlockCopyCost(true) + - displacementRate + - writer.GetSkipCost(true, skipContext); - - long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion; - long skipCost = Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion); - if (skipCost < candidateCost) - { - candidateCost = skipCost; - candidateSkip = true; - } - } - - // Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD. - if (candidateCost >= bestCost) - { - continue; - } - - bestCost = candidateCost; - hasSelectedCandidate = true; - selectedSkip = candidateSkip; - selectedVector = candidate; - if (candidateSkip) - { - workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction); - workspace.SelectedLumaCoefficients.Clear(); - selectedLumaState = emptyLumaState; - if (!this.source.IsMonochrome) - { - int chromaSampleCount = chromaTransformSize.GetSize2d(); - workspace.BluePrediction[..chromaSampleCount].CopyTo(workspace.SelectedBlueReconstruction); - workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction); - workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear(); - workspace.SelectedRedCoefficients[..chromaSampleCount].Clear(); - selectedBlueState = emptyBlueState; - selectedRedState = emptyRedState; - } - } - else - { - workspace.LumaCandidateReconstruction.CopyTo(workspace.SelectedLumaReconstruction); - workspace.LumaCandidateCoefficients.CopyTo(workspace.SelectedLumaCoefficients); - selectedLumaState = lumaCandidateState; - if (!this.source.IsMonochrome) - { - int chromaSampleCount = chromaTransformSize.GetSize2d(); - workspace.BlueCandidateReconstruction[..chromaSampleCount] - .CopyTo(workspace.SelectedBlueReconstruction); - - workspace.RedCandidateReconstruction[..chromaSampleCount] - .CopyTo(workspace.SelectedRedReconstruction); - - workspace.BlueCandidateCoefficients[..chromaSampleCount] - .CopyTo(workspace.SelectedBlueCoefficients); - - workspace.RedCandidateCoefficients[..chromaSampleCount] - .CopyTo(workspace.SelectedRedCoefficients); - - selectedBlueState = blueCandidateState; - selectedRedState = redCandidateState; - } - } - } - - if (!hasSelectedCandidate) - { - return bestCost; - } - - // Only the winning vector is now visible to later coding blocks. This single publication keeps - // rejected motion vectors from contaminating intra references or entropy contexts. - Span retainedLumaCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.Y); - Span retainedLumaTransformBlocks = - this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.Y); - - int lumaTransformIndex = this.codedAreaLuma / - Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; - - ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex]; - CopyCandidate( - workspace.SelectedLumaReconstruction, - workspace.SelectedLumaCoefficients, - lumaReconstruction, - blockOrigin, - retainedLumaCoefficients[this.codedAreaLuma..], - LumaTransformSize, - selectedLumaState, - ref retainedLumaState); - - if (!this.source.IsMonochrome) - { - Span retainedBlueCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.U); - Span retainedRedCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.V); - Span retainedBlueTransformBlocks = - this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.U); - - Span retainedRedTransformBlocks = - this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.V); - - int chromaTransformIndex = this.codedAreaChroma / - Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; - - ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex]; - ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex]; - CopyCandidate( - workspace.SelectedBlueReconstruction, - workspace.SelectedBlueCoefficients, - this.reconstruction.GetPlane(Av1Plane.U), - chromaOrigin, - retainedBlueCoefficients[this.codedAreaChroma..], - chromaTransformSize, - selectedBlueState, - ref retainedBlueState); - - CopyCandidate( - workspace.SelectedRedReconstruction, - workspace.SelectedRedCoefficients, - this.reconstruction.GetPlane(Av1Plane.V), - chromaOrigin, - retainedRedCoefficients[this.codedAreaChroma..], - chromaTransformSize, - selectedRedState, - ref retainedRedState); - } - - modeInfo.Block.Mode = Av1PredictionMode.DC; - modeInfo.Block.UvMode = Av1ChromaPredictionMode.DC; - modeInfo.Block.TransformSize = LumaTransformSize; - modeInfo.Block.Skip = selectedSkip; - modeInfo.Block.UseIntraBlockCopy = true; - block.FilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; - block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 0; - block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 0; - block.PredictionUnit.ChromaFromLumaIndex = 0; - block.PredictionUnit.ChromaFromLumaSigns = 0; - paletteInfo = default; - this.picture.SetDisplacementVector(modeInfoPosition, selectedVector); - return bestCost; - } - - private void EvaluateIntraBlockCopyPlane( - Av1SymbolEncoder writer, - Av1MotionVector vector, - Av1Plane plane, - Av1ComponentType componentType, - Point lumaOrigin, - int subsamplingX, - int subsamplingY, - Av1TransformSize transformSize, - Av1TransformType transformTypeSelection, - Av1TransformBlockContext blockContext, - Span prediction, - Span residual, - Span transformReconstruction, - Span transformCoefficients, - Span selectedReconstruction, - Span selectedCoefficients, - out Av1EncoderTransformBlockState selectedState, - out int selectedRate, - out long selectedDistortion, - out bool hasEmptyTransform, - out Av1EncoderTransformBlockState emptyState, - out long emptyDistortion) - { - Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY); - int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX)); - int sourceRowQ4 = (planeOrigin.Y << 4) + (vector.Row << (1 - subsamplingY)); - Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4); - int sampleCount = transformSize.GetSize2d(); - Buffer2DRegion sourcePlane = this.source.GetPlane(plane); - Buffer2DRegion reconstructionPlane = this.reconstruction.GetPlane(plane); - TOperator.PrepareIntraBlockCopy( - sourcePlane, - planeOrigin, - reconstructionPlane, - predictionOrigin, - (sourceColumnQ4 & 15) != 0, - (sourceRowQ4 & 15) != 0, - prediction[..sampleCount], - residual[..sampleCount], - transformSize); - - // Motion compensation and subtraction do not depend on transform type. Keep them outside the - // transform loop so exhaustive luma search traverses the source and reference blocks only once. - Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType( - transformSize, - isInter: true, - this.picture.Parent.FrameHeader.UseReducedTransformSet); - - Av1TransformType firstTransformType = transformTypeSelection == Av1TransformType.AllTransformTypes - ? Av1TransformType.DctDct - : transformTypeSelection; - Av1TransformType transformTypeLimit = transformTypeSelection == Av1TransformType.AllTransformTypes - ? Av1TransformType.AllTransformTypes - : (Av1TransformType)((int)transformTypeSelection + 1); - - long bestCost = long.MaxValue; - selectedState = default; - selectedRate = 0; - selectedDistortion = 0; - hasEmptyTransform = false; - emptyState = default; - emptyDistortion = 0; - - // The candidate and best spans alternate ownership whenever a transform improves the result. - // This mirrors the reference's buffer-pointer swap and replaces a copy on every improvement - // with at most one normalization copy after the transform search. - Span candidateReconstruction = transformReconstruction[..sampleCount]; - Span candidateCoefficients = transformCoefficients[..sampleCount]; - Span bestReconstruction = selectedReconstruction[..sampleCount]; - Span bestCoefficients = selectedCoefficients[..sampleCount]; - bool bestUsesSelectedStorage = true; - for (Av1TransformType transformType = firstTransformType; - transformType < transformTypeLimit; - transformType++) - { - if (!transformType.IsExtendedSetUsed(transformSetType)) - { - continue; - } - - Av1EncoderTransformBlockState candidateState = default; - long candidateDistortion = TOperator.EncodePredictionCandidate( - this.blockWorkspace, - sourcePlane, - planeOrigin, - prediction[..sampleCount], - residual[..sampleCount], - candidateReconstruction, - transformSize.GetWidth(), - candidateCoefficients, - transformSize, - transformType, - plane, - this.quantization.QIndex[0], - this.quantization.DeltaQDc[(int)plane], - this.quantization.DeltaQAc[(int)plane], - this.bitDepth, - ref candidateState); - - int candidateRate = writer.GetCoefficientCost( - transformSize, - transformType, - Av1PredictionMode.DC, - candidateCoefficients, - componentType, - blockContext, - candidateState.EndOfBlock, - this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes, - usesInterTransformSet: true); - - long candidateCost = Av1RateDistortion.GetCost( - this.rateMultiplier, - candidateRate, - candidateDistortion); - - if (candidateCost < bestCost) - { - Span previousBestReconstruction = bestReconstruction; - bestReconstruction = candidateReconstruction; - candidateReconstruction = previousBestReconstruction; - - Span previousBestCoefficients = bestCoefficients; - bestCoefficients = candidateCoefficients; - candidateCoefficients = previousBestCoefficients; - bestUsesSelectedStorage = !bestUsesSelectedStorage; - bestCost = candidateCost; - selectedState = candidateState; - selectedRate = candidateRate; - selectedDistortion = candidateDistortion; - } - - if (candidateState.EndOfBlock == 0 && - (!hasEmptyTransform || candidateDistortion < emptyDistortion)) - { - hasEmptyTransform = true; - emptyState = candidateState; - emptyDistortion = candidateDistortion; - } - } - - // Callers retain the designated selected spans after this scratch workspace is reused by the - // next plane or motion vector, so normalize only when the final best result occupies scratch. - if (!bestUsesSelectedStorage) - { - bestReconstruction.CopyTo(selectedReconstruction); - bestCoefficients.CopyTo(selectedCoefficients); - } - } - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs index 9692fef379..b0fa0317a0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs @@ -127,6 +127,7 @@ internal static partial class Av1IntraSuperblockEncoder where TOperator : struct, IBlockEncodingOperator { private readonly Av1EncoderFrame.PlanarView source; + private readonly Av1EncoderFrame.PlanarView reference; private readonly Av1EncoderFrame.PlanarView reconstruction; private readonly Av1PictureControlSet picture; private readonly Av1Superblock superblock; @@ -144,6 +145,7 @@ internal static partial class Av1IntraSuperblockEncoder /// Initializes a new instance of the struct. /// /// The coded source frame. + /// The reconstructed inter reference, or the current reconstruction for an intra frame. /// The reconstructed frame updated by winning candidates. /// The frame coding and mode-information state. /// The current superblock. @@ -152,6 +154,7 @@ internal static partial class Av1IntraSuperblockEncoder /// The mode-search effort in the inclusive range zero through ten. public ModeDecision( Av1EncoderFrame source, + Av1EncoderFrame reference, Av1EncoderFrame reconstruction, Av1PictureControlSet picture, Av1Superblock superblock, @@ -160,6 +163,7 @@ internal static partial class Av1IntraSuperblockEncoder int effort) { this.source = source.CodedView; + this.reference = reference.CodedView; this.reconstruction = reconstruction.CodedView; this.picture = picture; this.superblock = superblock; @@ -167,7 +171,10 @@ internal static partial class Av1IntraSuperblockEncoder this.blockWorkspace = blockWorkspace; this.quantization = picture.Parent.FrameHeader.QuantizationParameters; this.bitDepth = picture.Sequence.SequenceHeader.ColorConfig.BitDepth; - this.rateMultiplier = Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth); + this.rateMultiplier = picture.Parent.FrameHeader.IsIntra + ? Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth) + : Av1RateDistortion.GetInterFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth); + this.effort = effort; this.codedAreaLuma = 0; this.codedAreaChroma = 0; @@ -183,6 +190,13 @@ internal static partial class Av1IntraSuperblockEncoder Av1BlockSize blockSize, Av1PartitionType preparedPartition) { + if (!this.picture.Parent.FrameHeader.IsIntra) + { + // The first inter implementation retains the prepared 8x8 tree so every prediction and residual + // transform fits the single reusable block workspace while larger inter partitions remain unsearched. + return preparedPartition; + } + bool searchPartition = blockSize is Av1BlockSize.Block8x8 or Av1BlockSize.Block16x16 || (this.effort == 10 && blockSize is Av1BlockSize.Block32x32 or Av1BlockSize.Block64x64 or Av1BlockSize.Block128x128); @@ -640,8 +654,9 @@ internal static partial class Av1IntraSuperblockEncoder modeInfo.Block.Skip, allowIntraBlockCopy); - this.selectedBlockCost = allowIntraBlockCopy - ? this.SelectIntraBlockCopy( + if (!this.picture.Parent.FrameHeader.IsIntra) + { + this.selectedBlockCost = this.SelectInterPrediction( writer, macroBlock, blockOrigin, @@ -649,8 +664,22 @@ internal static partial class Av1IntraSuperblockEncoder regularCost, ref modeInfo, ref block, - ref paletteInfo) - : regularCost; + ref paletteInfo); + } + else + { + this.selectedBlockCost = allowIntraBlockCopy + ? this.SelectIntraBlockCopy( + writer, + macroBlock, + blockOrigin, + tileIndex, + regularCost, + ref modeInfo, + ref block, + ref paletteInfo) + : regularCost; + } this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight(); return; @@ -787,8 +816,9 @@ internal static partial class Av1IntraSuperblockEncoder modeInfo.Block.Skip, allowColorIntraBlockCopy); - this.selectedBlockCost = allowColorIntraBlockCopy - ? this.SelectIntraBlockCopy( + if (!this.picture.Parent.FrameHeader.IsIntra) + { + this.selectedBlockCost = this.SelectInterPrediction( writer, macroBlock, blockOrigin, @@ -796,8 +826,22 @@ internal static partial class Av1IntraSuperblockEncoder regularColorCost, ref modeInfo, ref block, - ref paletteInfo) - : regularColorCost; + ref paletteInfo); + } + else + { + this.selectedBlockCost = allowColorIntraBlockCopy + ? this.SelectIntraBlockCopy( + writer, + macroBlock, + blockOrigin, + tileIndex, + regularColorCost, + ref modeInfo, + ref block, + ref paletteInfo) + : regularColorCost; + } this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight(); if (block.HasChroma) @@ -1237,6 +1281,12 @@ internal static partial class Av1IntraSuperblockEncoder bool allowIntraBlockCopy) { int rateAdjustment = writer.GetSkipCost(skip, Av1TileWriter.GetSkipContext(macroBlock)); + if (!this.picture.Parent.FrameHeader.IsIntra) + { + int intraInterContext = Av1TileWriter.GetIntraInterContext(macroBlock); + rateAdjustment += writer.GetIsInterCost(false, intraInterContext); + } + if (allowIntraBlockCopy) { rateAdjustment += writer.GetUseIntraBlockCopyCost(false); @@ -2082,7 +2132,14 @@ internal static partial class Av1IntraSuperblockEncoder leftContexts, out int coefficientRate); - int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, blockSize, mode, angleDelta); + int rate = Av1TileWriter.GetLumaModeCost( + writer, + macroBlock, + blockSize, + mode, + angleDelta, + this.picture.Parent.FrameHeader.IsIntra); + rate += transformSizeRate + coefficientRate; if (mode == Av1PredictionMode.DC) { @@ -2193,7 +2250,14 @@ internal static partial class Av1IntraSuperblockEncoder } else { - rate += Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta); + rate += Av1TileWriter.GetLumaModeCost( + writer, + macroBlock, + BlockSize, + mode, + angleDelta, + this.picture.Parent.FrameHeader.IsIntra); + if (mode == Av1PredictionMode.DC) { rate += paletteDisabledCost; @@ -2643,7 +2707,14 @@ internal static partial class Av1IntraSuperblockEncoder // Charge every block-level choice that distinguishes this spatial candidate before adding // coefficient syntax derived from the live neighboring-transform context. - int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, blockSize, mode, angleDelta); + int rate = Av1TileWriter.GetLumaModeCost( + writer, + macroBlock, + blockSize, + mode, + angleDelta, + this.picture.Parent.FrameHeader.IsIntra); + rate += transformSizeRate; if (mode == Av1PredictionMode.DC) { @@ -2714,7 +2785,8 @@ internal static partial class Av1IntraSuperblockEncoder macroBlock, blockSize, Av1PredictionMode.DC, - 0); + 0, + this.picture.Parent.FrameHeader.IsIntra); rate += transformSizeRate; rate += paletteDisabledCost; diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs index e3108325d0..51bdcf4be8 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs @@ -6,6 +6,7 @@ using System.Runtime.Intrinsics; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -18,6 +19,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// internal static partial class Av1IntraSuperblockEncoder { + /// + /// The width and height of the fixed block currently used by inter motion search. + /// + private const int InterSearchBlockDimension = 8; + /// /// Defines type-specific block encoding without coupling traversal to sample storage width. /// @@ -259,7 +265,7 @@ internal static partial class Av1IntraSuperblockEncoder /// The contiguous prediction destination. /// The contiguous source-minus-prediction destination. /// The prediction dimensions. - public static abstract void PrepareIntraBlockCopy( + public static abstract void PrepareIntraBlockCopyPrediction( Buffer2DRegion source, Point blockOrigin, Buffer2DRegion reconstruction, @@ -270,6 +276,68 @@ internal static partial class Av1IntraSuperblockEncoder Span residual, Av1TransformSize transformSize); + /// + /// Subtracts a retained prediction from its source without rebuilding the inter predictor. + /// + /// The source plane. + /// The block origin in plane samples. + /// The tightly packed prediction samples. + /// The destination signed residual samples. + /// The plane block geometry. + public static abstract void SubtractPrediction( + Buffer2DRegion source, + Point blockOrigin, + ReadOnlySpan prediction, + Span residual, + Av1TransformSize transformSize); + + /// + /// Builds a translational prediction from a retained reference frame and the matching source residual. + /// + /// The coded source plane. + /// The destination block origin in plane samples. + /// The padded retained reference plane. + /// The integer reference origin preceding the subpixel phase. + /// The horizontal interpolation filter. + /// The vertical interpolation filter. + /// The horizontal phase in one-sixteenth-sample units. + /// The vertical phase in one-sixteenth-sample units. + /// The contiguous prediction destination. + /// The contiguous source-minus-prediction destination. + /// The intermediate storage used by two-dimensional filtering. + /// The prediction dimensions. + /// The coded sample bit depth. + public static abstract void PrepareTranslationalInterPrediction( + Buffer2DRegion source, + Point blockOrigin, + Buffer2DRegion reference, + Point predictionOrigin, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span prediction, + Span residual, + Span predictionScratch, + Av1TransformSize transformSize, + Av1BitDepth bitDepth); + + /// + /// Measures an 8x8 full-pixel reference candidate through the bordered plane storage. + /// + /// The coded source plane. + /// The source block origin in visible-plane coordinates. + /// The padded retained reference plane. + /// The candidate origin, which may lie inside the physical border. + /// The coded sample precision. + /// The squared error normalized to the eight-bit distortion domain. + public static abstract long GetInterPredictionError( + Buffer2DRegion source, + Point sourceOrigin, + Buffer2DRegion reference, + Point predictionOrigin, + Av1BitDepth bitDepth); + /// /// Encodes one prepared prediction with the selected transform into decision scratch. /// @@ -384,9 +452,8 @@ internal static partial class Av1IntraSuperblockEncoder ReadOnlySpan firstRow = plane.DangerousGetRowSpan(first.Y + row)[first.X..]; ReadOnlySpan secondRow = plane.DangerousGetRowSpan(second.Y + row)[second.X..]; - // An 8x8 search row occupies one machine word, so one unaligned load and comparison replaces - // eight dependent scalar branches while retaining exact collision rejection. - if (MemoryMarshal.Read(firstRow) != MemoryMarshal.Read(secondRow)) + // Compare the complete row as byte lanes so collision rejection remains independent of native endianness. + if (Vector64.Create(firstRow) != Vector64.Create(secondRow)) { return false; } @@ -395,49 +462,47 @@ internal static partial class Av1IntraSuperblockEncoder return true; } + /// + public static long GetInterPredictionError( + Buffer2DRegion source, + Point sourceOrigin, + Buffer2DRegion reference, + Point predictionOrigin, + Av1BitDepth bitDepth) + { + Rectangle sourceBounds = source.Bounds; + Rectangle referenceBounds = reference.Bounds; + int sourceIndex = + ((sourceBounds.Y + sourceOrigin.Y) * source.Stride) + + sourceBounds.X + + sourceOrigin.X; + + int referenceIndex = + ((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) + + referenceBounds.X + + predictionOrigin.X; + + // The shared residual kernel selects the widest available vector width and handles the scalar tail. + return Av1ResidualBuilder.SumSquaredError( + source.Buffer.DangerousGetSingleSpan()[sourceIndex..], + source.Stride, + reference.Buffer.DangerousGetSingleSpan()[referenceIndex..], + reference.Stride, + InterSearchBlockDimension, + InterSearchBlockDimension); + } + /// public static int GetSumOfAbsoluteDifferences( Buffer2DRegion source, Point sourceOrigin, Buffer2DRegion reconstruction, Point predictionOrigin) - { - int sum = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - Vector128 difference = - (Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(sourceRow)).AsByte()) - - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow)).AsByte())) - .AsInt16(); - - // Widened signed differences retain both subtraction directions; absolute values then reduce - // the complete eight-sample row without scalar extraction or per-sample branches. - sum += Vector128.Sum(Vector128.Abs(difference)); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - sum += Math.Abs(sourceRow[column] - predictionRow[column]); - } - } - } - - return sum; - } + => Av1ResidualBuilder.SumAbsoluteDifferences8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin), + reconstruction.Stride); /// public static void GetFourSumsOfAbsoluteDifferences( @@ -446,66 +511,12 @@ internal static partial class Av1IntraSuperblockEncoder Buffer2DRegion reconstruction, Point firstPredictionOrigin, Span sums) - { - int sum0 = 0; - int sum1 = 0; - int sum2 = 0; - int sum3 = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..]; - - // The four candidates reuse one widened source vector; only their overlapping predictor - // windows are loaded separately before the packed absolute-difference reductions. - Vector128 sourceSamples = - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(sourceRow)).AsByte()).AsInt16(); - - Vector128 prediction0 = - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow)).AsByte()).AsInt16(); - - Vector128 prediction1 = - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow[1..])).AsByte()).AsInt16(); - - Vector128 prediction2 = - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow[2..])).AsByte()).AsInt16(); - - Vector128 prediction3 = - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow[3..])).AsByte()).AsInt16(); - - sum0 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction0)); - sum1 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction1)); - sum2 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction2)); - sum3 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction3)); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - int sourceSample = sourceRow[column]; - sum0 += Math.Abs(sourceSample - predictionRow[column]); - sum1 += Math.Abs(sourceSample - predictionRow[column + 1]); - sum2 += Math.Abs(sourceSample - predictionRow[column + 2]); - sum3 += Math.Abs(sourceSample - predictionRow[column + 3]); - } - } - } - - sums[0] = sum0; - sums[1] = sum1; - sums[2] = sum2; - sums[3] = sum3; - } + => Av1ResidualBuilder.SumFourAbsoluteDifferences8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, firstPredictionOrigin), + reconstruction.Stride, + sums); /// public static int GetVariance( @@ -515,44 +526,13 @@ internal static partial class Av1IntraSuperblockEncoder Point predictionOrigin, Av1BitDepth bitDepth) { - int sum = 0; - int sumOfSquares = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - Vector128 difference = - (Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(sourceRow)).AsByte()) - - Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read(predictionRow)).AsByte())) - .AsInt16(); - - // Widen before squaring so signed residuals cannot wrap in 16-bit lanes. - Vector128 lower = Vector128.WidenLower(difference); - Vector128 upper = Vector128.WidenUpper(difference); - sum += Vector128.Sum(difference); - sumOfSquares += Vector128.Sum(lower * lower) + Vector128.Sum(upper * upper); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - int difference = sourceRow[column] - predictionRow[column]; - sum += difference; - sumOfSquares += difference * difference; - } - } - } + Av1ResidualBuilder.GetMoments8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin), + reconstruction.Stride, + out int sum, + out int sumOfSquares); return GetNormalizedVariance(sum, sumOfSquares, bitDepth); } @@ -575,10 +555,7 @@ internal static partial class Av1IntraSuperblockEncoder // A complete row widens in one vector; clipped edge rows retain scalar bounds. if (columns == 8 && Vector128.IsHardwareAccelerated) { - ulong packed = MemoryMarshal.Read(sourceRow); - Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()) - .AsInt16() - .CopyTo(samples[sampleOffset..]); + Vector128.WidenLower(Vector128.Create(Vector64.Create(sourceRow), Vector64.Zero)).AsInt16().CopyTo(samples[sampleOffset..]); sampleOffset += columns; continue; @@ -803,7 +780,7 @@ internal static partial class Av1IntraSuperblockEncoder } /// - public static void PrepareIntraBlockCopy( + public static void PrepareIntraBlockCopyPrediction( Buffer2DRegion source, Point blockOrigin, Buffer2DRegion reconstruction, @@ -837,6 +814,64 @@ internal static partial class Av1IntraSuperblockEncoder height); } + /// + public static void SubtractPrediction( + Buffer2DRegion source, + Point blockOrigin, + ReadOnlySpan prediction, + Span residual, + Av1TransformSize transformSize) + => Av1ResidualBuilder.Subtract( + Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin), + source.Stride, + prediction, + transformSize.GetWidth(), + residual, + transformSize.GetWidth(), + transformSize.GetWidth(), + transformSize.GetHeight()); + + /// + public static void PrepareTranslationalInterPrediction( + Buffer2DRegion source, + Point blockOrigin, + Buffer2DRegion reference, + Point predictionOrigin, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span prediction, + Span residual, + Span predictionScratch, + Av1TransformSize transformSize, + Av1BitDepth bitDepth) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + Rectangle referenceBounds = reference.Bounds; + int referenceOrigin = + ((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) + + referenceBounds.X + + predictionOrigin.X; + + Av1TranslationalInterPredictor.Predict( + reference.Buffer.DangerousGetSingleSpan(), + reference.Stride, + referenceOrigin, + prediction, + width, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + predictionScratch); + + SubtractPrediction(source, blockOrigin, prediction, residual, transformSize); + } + /// public static long EncodePredictionCandidate( Av1EncoderBlockWorkspace workspace, @@ -947,49 +982,49 @@ internal static partial class Av1IntraSuperblockEncoder } /// - public static int GetSumOfAbsoluteDifferences( + public static long GetInterPredictionError( Buffer2DRegion source, Point sourceOrigin, - Buffer2DRegion reconstruction, - Point predictionOrigin) + Buffer2DRegion reference, + Point predictionOrigin, + Av1BitDepth bitDepth) { - int sum = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - // Twelve-bit samples remain within signed 16-bit subtraction and absolute-value ranges, - // allowing all eight row differences to stay packed until their horizontal reduction. - Vector128 difference = - (Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)) - - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow))) - .AsInt16(); - - sum += Vector128.Sum(Vector128.Abs(difference)); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - sum += Math.Abs(sourceRow[column] - predictionRow[column]); - } - } - } + Rectangle sourceBounds = source.Bounds; + Rectangle referenceBounds = reference.Bounds; + int sourceIndex = + ((sourceBounds.Y + sourceOrigin.Y) * source.Stride) + + sourceBounds.X + + sourceOrigin.X; + + int referenceIndex = + ((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) + + referenceBounds.X + + predictionOrigin.X; + + long error = Av1ResidualBuilder.SumSquaredError( + source.Buffer.DangerousGetSingleSpan()[sourceIndex..], + source.Stride, + reference.Buffer.DangerousGetSingleSpan()[referenceIndex..], + reference.Stride, + InterSearchBlockDimension, + InterSearchBlockDimension); - return sum; + int shift = (bitDepth.GetBitCount() - 8) * 2; + return shift == 0 ? error : (error + (1L << (shift - 1))) >> shift; } + /// + public static int GetSumOfAbsoluteDifferences( + Buffer2DRegion source, + Point sourceOrigin, + Buffer2DRegion reconstruction, + Point predictionOrigin) + => Av1ResidualBuilder.SumAbsoluteDifferences8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin), + reconstruction.Stride); + /// public static void GetFourSumsOfAbsoluteDifferences( Buffer2DRegion source, @@ -997,66 +1032,12 @@ internal static partial class Av1IntraSuperblockEncoder Buffer2DRegion reconstruction, Point firstPredictionOrigin, Span sums) - { - int sum0 = 0; - int sum1 = 0; - int sum2 = 0; - int sum3 = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..]; - - // Signed 16-bit lanes preserve every AV1 sample difference while four horizontally adjacent - // candidates reuse the same source load and stay packed through horizontal reduction. - Vector128 sourceSamples = - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)).AsInt16(); - - Vector128 prediction0 = - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow)).AsInt16(); - - Vector128 prediction1 = - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[1..])).AsInt16(); - - Vector128 prediction2 = - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[2..])).AsInt16(); - - Vector128 prediction3 = - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[3..])).AsInt16(); - - sum0 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction0)); - sum1 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction1)); - sum2 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction2)); - sum3 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction3)); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - int sourceSample = sourceRow[column]; - sum0 += Math.Abs(sourceSample - predictionRow[column]); - sum1 += Math.Abs(sourceSample - predictionRow[column + 1]); - sum2 += Math.Abs(sourceSample - predictionRow[column + 2]); - sum3 += Math.Abs(sourceSample - predictionRow[column + 3]); - } - } - } - - sums[0] = sum0; - sums[1] = sum1; - sums[2] = sum2; - sums[3] = sum3; - } + => Av1ResidualBuilder.SumFourAbsoluteDifferences8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, firstPredictionOrigin), + reconstruction.Stride, + sums); /// public static int GetVariance( @@ -1066,45 +1047,13 @@ internal static partial class Av1IntraSuperblockEncoder Point predictionOrigin, Av1BitDepth bitDepth) { - int sum = 0; - int sumOfSquares = 0; - if (Vector128.IsHardwareAccelerated) - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - // AV1's high-bit-depth domain tops out at 4095, so signed 16-bit subtraction preserves - // every possible sample difference before the square is widened to 32-bit lanes. - Vector128 difference = - (Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)) - - Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow))) - .AsInt16(); - - Vector128 lower = Vector128.WidenLower(difference); - Vector128 upper = Vector128.WidenUpper(difference); - sum += Vector128.Sum(difference); - sumOfSquares += Vector128.Sum(lower * lower) + Vector128.Sum(upper * upper); - } - } - else - { - for (int row = 0; row < 8; row++) - { - ReadOnlySpan sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; - ReadOnlySpan predictionRow = - reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; - - for (int column = 0; column < 8; column++) - { - int difference = sourceRow[column] - predictionRow[column]; - sum += difference; - sumOfSquares += difference * difference; - } - } - } + Av1ResidualBuilder.GetMoments8x8( + Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin), + source.Stride, + Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin), + reconstruction.Stride, + out int sum, + out int sumOfSquares); return GetNormalizedVariance(sum, sumOfSquares, bitDepth); } @@ -1353,7 +1302,7 @@ internal static partial class Av1IntraSuperblockEncoder } /// - public static void PrepareIntraBlockCopy( + public static void PrepareIntraBlockCopyPrediction( Buffer2DRegion source, Point blockOrigin, Buffer2DRegion reconstruction, @@ -1387,6 +1336,65 @@ internal static partial class Av1IntraSuperblockEncoder height); } + /// + public static void SubtractPrediction( + Buffer2DRegion source, + Point blockOrigin, + ReadOnlySpan prediction, + Span residual, + Av1TransformSize transformSize) + => Av1ResidualBuilder.Subtract( + Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin), + source.Stride, + prediction, + transformSize.GetWidth(), + residual, + transformSize.GetWidth(), + transformSize.GetWidth(), + transformSize.GetHeight()); + + /// + public static void PrepareTranslationalInterPrediction( + Buffer2DRegion source, + Point blockOrigin, + Buffer2DRegion reference, + Point predictionOrigin, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int horizontalPhase, + int verticalPhase, + Span prediction, + Span residual, + Span predictionScratch, + Av1TransformSize transformSize, + Av1BitDepth bitDepth) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + Rectangle referenceBounds = reference.Bounds; + int referenceOrigin = + ((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) + + referenceBounds.X + + predictionOrigin.X; + + Av1TranslationalInterPredictor.Predict( + reference.Buffer.DangerousGetSingleSpan(), + reference.Stride, + referenceOrigin, + prediction, + width, + width, + height, + horizontalFilter, + verticalFilter, + horizontalPhase, + verticalPhase, + bitDepth.GetBitCount(), + predictionScratch); + + SubtractPrediction(source, blockOrigin, prediction, residual, transformSize); + } + /// public static long EncodePredictionCandidate( Av1EncoderBlockWorkspace workspace, diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs index 31abf04b95..fa8cdb39c4 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs @@ -372,7 +372,8 @@ internal static partial class Av1IntraSuperblockEncoder macroBlock, BlockSize, Av1PredictionMode.DC, - 0); + 0, + this.picture.Parent.FrameHeader.IsIntra); rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext); rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs new file mode 100644 index 0000000000..2c810a0741 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs @@ -0,0 +1,1751 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Provides full rate-distortion selection for reference-frame and intra-block-copy candidates. +/// +internal static partial class Av1IntraSuperblockEncoder +{ + /// + /// The first effort tier that searches a block-local motion vector. + /// + private const int MinimumInterMotionSearchEffort = 6; + + /// + /// The smallest full-pixel radius used by block-local inter search. + /// + private const int MinimumInterMotionSearchRadius = 4; + + /// + /// The first effort tier that refines full-pixel motion to quarter-pixel precision. + /// + private const int MinimumSubpixelMotionSearchEffort = 7; + + /// + /// The first effort tier that adds the final eighth-pixel refinement step. + /// + private const int MinimumHighPrecisionMotionSearchEffort = 8; + + /// + /// The physical border reserved on each side for fractional eight-tap filtering. + /// + private const int FractionalInterpolationBorder = 4; + + /// + /// The number of cardinal and diagonal candidates examined at each search step. + /// + private const int InterMotionSearchDirectionCount = 8; + + /// + /// One nearest, three near, one global, and three new-motion candidates. + /// + private const int MaximumInterModeCandidateCount = 8; + + internal partial struct ModeDecision + where TSample : unmanaged + where TOperator : struct, IBlockEncodingOperator + { + private long SelectIntraBlockCopy( + Av1SymbolEncoder writer, + Av1MacroBlockD macroBlock, + Point blockOrigin, + ushort tileIndex, + long regularCost, + ref Av1MacroBlockModeInfo modeInfo, + ref Av1EncoderBlockStruct block, + ref Av1EncoderPaletteInfo paletteInfo) + { + const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; + const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8; + Buffer2DRegion lumaSource = this.source.GetPlane(Av1Plane.Y); + Buffer2DRegion lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y); + Point modeInfoPosition = new( + blockOrigin.X >> Av1Constants.ModeInfoSizeLog2, + blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2); + + Span referenceCandidates = stackalloc Av1MotionVector[8]; + Span referenceWeights = stackalloc int[8]; + Av1MotionVector reference = Av1IntraBlockCopy.FindReference( + this.picture, + macroBlock, + modeInfoPosition, + BlockSize, + Av1PartitionType.None, + referenceCandidates, + referenceWeights); + + Span candidates = stackalloc Av1MotionVector[4]; + Av1IntraBlockCopySearchIndex search = this.picture.IntraBlockCopySearch; + int candidateCount = search.FindCandidates( + lumaSource, + lumaReconstruction, + blockOrigin, + macroBlock.Tile, + this.picture.Sequence.SequenceHeader, + writer, + reference, + this.rateMultiplier, + candidates); + + candidateCount += search.FindPixelCandidates( + lumaSource, + lumaReconstruction, + blockOrigin, + macroBlock.Tile, + this.picture.Sequence.SequenceHeader, + writer, + reference, + this.quantization.QIndex[0], + this.rateMultiplier, + candidates[candidateCount..]); + + // Hash and full-pixel searches can converge on the same vector. Preserve the first search-order + // occurrence so repeated vectors do not pay for duplicate transform searches or alter ties. + int uniqueCandidateCount = 0; + for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++) + { + Av1MotionVector candidate = candidates[candidateIndex]; + bool duplicate = false; + for (int uniqueIndex = 0; uniqueIndex < uniqueCandidateCount; uniqueIndex++) + { + if (candidate == candidates[uniqueIndex]) + { + duplicate = true; + break; + } + } + + if (!duplicate) + { + candidates[uniqueCandidateCount++] = candidate; + } + } + + if (uniqueCandidateCount == 0) + { + return regularCost; + } + + int skipContext = Av1TileWriter.GetSkipContext(macroBlock); + long bestCost = regularCost; + bool hasSelectedCandidate = false; + bool selectedSkip = false; + Av1MotionVector selectedVector = default; + Av1EncoderTransformBlockState selectedLumaState = default; + Av1EncoderTransformBlockState selectedBlueState = default; + Av1EncoderTransformBlockState selectedRedState = default; + Av1EncoderInterPredictionWorkspace workspace = + this.blockWorkspace.GetInterPredictionWorkspace(); + + Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Luminance, + this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex], + blockOrigin, + BlockSize, + LumaTransformSize); + + // A coded IBC residual uses the unsplit transform root at this fixed block size. A skipped block + // omits both the transform-partition bit and coefficient syntax, so this rate is added only below. + int transformPartitionRate = 0; + if (this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select) + { + Av1NeighborArrayUnit transformContexts = this.picture.TransformFunctionContexts[tileIndex]; + int topIndex = transformContexts.GetTopIndex(blockOrigin); + int leftIndex = transformContexts.GetLeftIndex(blockOrigin); + int transformPartitionContext = Av1SymbolContextHelper.GetTransformPartitionContext( + transformContexts.Top[topIndex], + transformContexts.Left[leftIndex], + BlockSize, + LumaTransformSize); + + transformPartitionRate = writer.GetTransformPartitionCost( + false, + transformPartitionContext); + } + + ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig; + int subsamplingX = colorConfig.SubSamplingX ? 1 : 0; + int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; + Point chromaOrigin = new(blockOrigin.X >> subsamplingX, blockOrigin.Y >> subsamplingY); + Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize( + colorConfig.SubSamplingX, + colorConfig.SubSamplingY); + + Av1TransformBlockContext blueContext = default; + Av1TransformBlockContext redContext = default; + if (!this.source.IsMonochrome) + { + Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled( + colorConfig.SubSamplingX, + colorConfig.SubSamplingY); + + blueContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Chroma, + this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex], + chromaOrigin, + chromaBlockSize, + chromaTransformSize); + + redContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Chroma, + this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex], + chromaOrigin, + chromaBlockSize, + chromaTransformSize); + } + + // Per-vector plane results reuse candidate scratch. Separate selected spans retain only a new + // global winner, allowing the complete search to finish before committed reconstruction changes. + for (int candidateIndex = 0; candidateIndex < uniqueCandidateCount; candidateIndex++) + { + Av1MotionVector candidate = candidates[candidateIndex]; + this.EvaluateInterPlane( + writer, + candidate, + Av1Plane.Y, + Av1ComponentType.Luminance, + Av1PredictionMode.DC, + usePreparedPrediction: false, + Av1InterpolationFilter.Bilinear, + Av1InterpolationFilter.Bilinear, + lumaReconstruction, + blockOrigin, + 0, + 0, + LumaTransformSize, + Av1TransformType.AllTransformTypes, + lumaContext, + workspace.LumaPrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + workspace.LumaCandidateReconstruction, + workspace.LumaCandidateCoefficients, + out Av1EncoderTransformBlockState lumaCandidateState, + out int lumaRate, + out long lumaDistortion, + out bool hasEmptyLuma, + out Av1EncoderTransformBlockState emptyLumaState, + out long emptyLumaDistortion); + + int blueRate = 0; + int redRate = 0; + long blueDistortion = 0; + long redDistortion = 0; + long emptyBlueDistortion = 0; + long emptyRedDistortion = 0; + bool hasEmptyBlue = true; + bool hasEmptyRed = true; + Av1EncoderTransformBlockState blueCandidateState = default; + Av1EncoderTransformBlockState redCandidateState = default; + Av1EncoderTransformBlockState emptyBlueState = default; + Av1EncoderTransformBlockState emptyRedState = default; + if (!this.source.IsMonochrome) + { + Av1TransformType chromaTransformType = lumaCandidateState.TransformType; + Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType( + chromaTransformSize, + isInter: true, + this.picture.Parent.FrameHeader.UseReducedTransformSet); + + // Inter prediction does not signal an independent chroma transform type. Chroma reuses the + // selected luma type when that type belongs to its transform set and otherwise falls back to DCT. + if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet)) + { + chromaTransformType = Av1TransformType.DctDct; + } + + this.EvaluateInterPlane( + writer, + candidate, + Av1Plane.U, + Av1ComponentType.Chroma, + Av1PredictionMode.DC, + usePreparedPrediction: false, + Av1InterpolationFilter.Bilinear, + Av1InterpolationFilter.Bilinear, + this.reconstruction.GetPlane(Av1Plane.U), + blockOrigin, + subsamplingX, + subsamplingY, + chromaTransformSize, + chromaTransformType, + blueContext, + workspace.BluePrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + workspace.BlueCandidateReconstruction, + workspace.BlueCandidateCoefficients, + out blueCandidateState, + out blueRate, + out blueDistortion, + out hasEmptyBlue, + out emptyBlueState, + out emptyBlueDistortion); + + this.EvaluateInterPlane( + writer, + candidate, + Av1Plane.V, + Av1ComponentType.Chroma, + Av1PredictionMode.DC, + usePreparedPrediction: false, + Av1InterpolationFilter.Bilinear, + Av1InterpolationFilter.Bilinear, + this.reconstruction.GetPlane(Av1Plane.V), + blockOrigin, + subsamplingX, + subsamplingY, + chromaTransformSize, + chromaTransformType, + redContext, + workspace.RedPrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + workspace.RedCandidateReconstruction, + workspace.RedCandidateCoefficients, + out redCandidateState, + out redRate, + out redDistortion, + out hasEmptyRed, + out emptyRedState, + out emptyRedDistortion); + } + + int displacementRate = writer.GetDisplacementVectorCost(candidate, reference); + int candidateRate = writer.GetUseIntraBlockCopyCost(true) + + displacementRate + + writer.GetSkipCost(false, skipContext) + + transformPartitionRate + + lumaRate + + blueRate + + redRate; + + long candidateDistortion = lumaDistortion + blueDistortion + redDistortion; + long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, candidateDistortion); + bool candidateSkip = false; + + // The skip alternative is available only when every coded plane has an empty transform. Its + // distortion comes from prediction alone and its rate excludes the transform tree and coefficients. + if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed) + { + int skipRate = writer.GetUseIntraBlockCopyCost(true) + + displacementRate + + writer.GetSkipCost(true, skipContext); + + long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion; + long skipCost = Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion); + if (skipCost < candidateCost) + { + candidateCost = skipCost; + candidateSkip = true; + } + } + + // Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD. + if (candidateCost >= bestCost) + { + continue; + } + + bestCost = candidateCost; + hasSelectedCandidate = true; + selectedSkip = candidateSkip; + selectedVector = candidate; + if (candidateSkip) + { + workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction); + workspace.SelectedLumaCoefficients.Clear(); + selectedLumaState = emptyLumaState; + if (!this.source.IsMonochrome) + { + int chromaSampleCount = chromaTransformSize.GetSize2d(); + workspace.BluePrediction[..chromaSampleCount].CopyTo(workspace.SelectedBlueReconstruction); + workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction); + workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear(); + workspace.SelectedRedCoefficients[..chromaSampleCount].Clear(); + selectedBlueState = emptyBlueState; + selectedRedState = emptyRedState; + } + } + else + { + workspace.LumaCandidateReconstruction.CopyTo(workspace.SelectedLumaReconstruction); + workspace.LumaCandidateCoefficients.CopyTo(workspace.SelectedLumaCoefficients); + selectedLumaState = lumaCandidateState; + if (!this.source.IsMonochrome) + { + int chromaSampleCount = chromaTransformSize.GetSize2d(); + workspace.BlueCandidateReconstruction[..chromaSampleCount] + .CopyTo(workspace.SelectedBlueReconstruction); + + workspace.RedCandidateReconstruction[..chromaSampleCount] + .CopyTo(workspace.SelectedRedReconstruction); + + workspace.BlueCandidateCoefficients[..chromaSampleCount] + .CopyTo(workspace.SelectedBlueCoefficients); + + workspace.RedCandidateCoefficients[..chromaSampleCount] + .CopyTo(workspace.SelectedRedCoefficients); + + selectedBlueState = blueCandidateState; + selectedRedState = redCandidateState; + } + } + } + + if (!hasSelectedCandidate) + { + return bestCost; + } + + // Only the winning vector is now visible to later coding blocks. This single publication keeps + // rejected motion vectors from contaminating intra references or entropy contexts. + Span retainedLumaCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.Y); + Span retainedLumaTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.Y); + + int lumaTransformIndex = this.codedAreaLuma / + Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + + ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex]; + CopyCandidate( + workspace.SelectedLumaReconstruction, + workspace.SelectedLumaCoefficients, + lumaReconstruction, + blockOrigin, + retainedLumaCoefficients[this.codedAreaLuma..], + LumaTransformSize, + selectedLumaState, + ref retainedLumaState); + + if (!this.source.IsMonochrome) + { + Span retainedBlueCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.U); + Span retainedRedCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.V); + Span retainedBlueTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.U); + + Span retainedRedTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.V); + + int chromaTransformIndex = this.codedAreaChroma / + Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + + ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex]; + ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex]; + CopyCandidate( + workspace.SelectedBlueReconstruction, + workspace.SelectedBlueCoefficients, + this.reconstruction.GetPlane(Av1Plane.U), + chromaOrigin, + retainedBlueCoefficients[this.codedAreaChroma..], + chromaTransformSize, + selectedBlueState, + ref retainedBlueState); + + CopyCandidate( + workspace.SelectedRedReconstruction, + workspace.SelectedRedCoefficients, + this.reconstruction.GetPlane(Av1Plane.V), + chromaOrigin, + retainedRedCoefficients[this.codedAreaChroma..], + chromaTransformSize, + selectedRedState, + ref retainedRedState); + } + + modeInfo.Block.Mode = Av1PredictionMode.DC; + modeInfo.Block.UvMode = Av1ChromaPredictionMode.DC; + modeInfo.Block.TransformSize = LumaTransformSize; + modeInfo.Block.Skip = selectedSkip; + modeInfo.Block.UseIntraBlockCopy = true; + block.FilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; + block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 0; + block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 0; + block.PredictionUnit.ChromaFromLumaIndex = 0; + block.PredictionUnit.ChromaFromLumaSigns = 0; + paletteInfo = default; + this.picture.SetDisplacementVector(modeInfoPosition, selectedVector); + return bestCost; + } + + /// + /// Compares the retained intra result with an inter candidate without disturbing the intra result on loss. + /// + private long SelectInterPrediction( + Av1SymbolEncoder writer, + Av1MacroBlockD macroBlock, + Point blockOrigin, + ushort tileIndex, + long regularCost, + ref Av1MacroBlockModeInfo modeInfo, + ref Av1EncoderBlockStruct block, + ref Av1EncoderPaletteInfo paletteInfo) + { + Av1MacroBlockModeInfo interModeInfo = modeInfo; + Av1EncoderBlockStruct interBlock = block; + Av1EncoderPaletteInfo interPaletteInfo = default; + long selectedCost = this.SelectInterBlock( + writer, + macroBlock, + blockOrigin, + tileIndex, + regularCost, + ref interModeInfo, + ref interBlock, + ref interPaletteInfo); + + if (selectedCost < regularCost) + { + modeInfo = interModeInfo; + block = interBlock; + paletteInfo = interPaletteInfo; + } + + return selectedCost; + } + + /// + /// Evaluates the supported LAST_FRAME modes and publishes only a strict improvement over the intra result. + /// + private long SelectInterBlock( + Av1SymbolEncoder writer, + Av1MacroBlockD macroBlock, + Point blockOrigin, + ushort tileIndex, + long regularCost, + ref Av1MacroBlockModeInfo modeInfo, + ref Av1EncoderBlockStruct block, + ref Av1EncoderPaletteInfo paletteInfo) + { + const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; + const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8; + const int LastReferenceIndex = 0; + DebugGuard.IsTrue(modeInfo.Block.BlockSize == BlockSize, "Inter prediction currently uses the prepared 8x8 partition tree."); + + modeInfo.Block.ReferenceFrame = Av1ReferenceFrameType.Last; + modeInfo.Block.UvMode = Av1ChromaPredictionMode.DC; + modeInfo.Block.TransformSize = LumaTransformSize; + modeInfo.Block.UseIntraBlockCopy = false; + block.FilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; + block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 0; + block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 0; + block.PredictionUnit.ChromaFromLumaIndex = 0; + block.PredictionUnit.ChromaFromLumaSigns = 0; + paletteInfo = default; + + Av1EncoderInterPredictionWorkspace workspace = + this.blockWorkspace.GetInterPredictionWorkspace(); + + ObuFrameHeader frameHeader = this.picture.Parent.FrameHeader; + Point modeInfoPosition = blockOrigin >> Av1Constants.ModeInfoSizeLog2; + ref Av1ReferenceMotionVectors referenceMotionVectors = ref this.blockWorkspace.ReferenceMotionVectors; + referenceMotionVectors.Build( + this.picture, + macroBlock, + modeInfoPosition, + BlockSize, + modeInfo.Block.PartitionType, + this.picture.Sequence.SequenceHeader, + frameHeader, + Av1ReferenceFrameType.Last); + + Av1MotionVector globalMotion = frameHeader + .GetGlobalMotionParameters()[LastReferenceIndex] + .GetMotionVector( + frameHeader.AllowHighPrecisionMotionVector, + BlockSize, + modeInfoPosition, + frameHeader.ForceIntegerMotionVector); + + Span candidateVectors = stackalloc Av1MotionVector[MaximumInterModeCandidateCount]; + Span candidateModes = stackalloc Av1PredictionMode[MaximumInterModeCandidateCount]; + Span candidateReferenceIndices = stackalloc byte[MaximumInterModeCandidateCount]; + int candidateCount = 0; + if (this.effort >= MinimumInterMotionSearchEffort) + { + // Predictor-stack modes precede global and new motion so strict ties retain the reference order. + candidateVectors[candidateCount] = referenceMotionVectors.Nearest; + candidateModes[candidateCount] = Av1PredictionMode.NearestMotionVector; + candidateReferenceIndices[candidateCount++] = 0; + + int maximumNearIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 2)); + for (int referenceIndex = 0; referenceIndex <= maximumNearIndex; referenceIndex++) + { + candidateVectors[candidateCount] = referenceMotionVectors.GetNearReference(referenceIndex); + candidateModes[candidateCount] = Av1PredictionMode.NearMotionVector; + candidateReferenceIndices[candidateCount++] = (byte)referenceIndex; + } + } + + candidateVectors[candidateCount] = globalMotion; + candidateModes[candidateCount] = Av1PredictionMode.GlobalMotionVector; + candidateReferenceIndices[candidateCount++] = 0; + if (this.effort >= MinimumInterMotionSearchEffort) + { + int maximumNewIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 1)); + for (int referenceIndex = 0; referenceIndex <= maximumNewIndex; referenceIndex++) + { + Av1MotionVector newReference = referenceMotionVectors.GetNewReference(referenceIndex); + Av1MotionVector searched = this.FindInterMotionVector( + writer, + blockOrigin, + newReference, + referenceIndex); + + // Equal prediction vectors can carry different DRL and mode costs. Preserve each syntax choice + // as an independent candidate instead of deduplicating solely by reconstructed pixels. + candidateVectors[candidateCount] = searched; + candidateModes[candidateCount] = Av1PredictionMode.NewMotionVector; + candidateReferenceIndices[candidateCount++] = (byte)referenceIndex; + } + } + + Span selectedLumaReconstruction = workspace.SelectedLumaReconstruction; + Span candidateLumaReconstruction = workspace.LumaCandidateReconstruction; + Span selectedBlueReconstruction = workspace.SelectedBlueReconstruction; + Span candidateBlueReconstruction = workspace.BlueCandidateReconstruction; + Span selectedRedReconstruction = workspace.SelectedRedReconstruction; + Span candidateRedReconstruction = workspace.RedCandidateReconstruction; + Span selectedLumaCoefficients = workspace.SelectedLumaCoefficients; + Span candidateLumaCoefficients = workspace.LumaCandidateCoefficients; + Span selectedBlueCoefficients = workspace.SelectedBlueCoefficients; + Span candidateBlueCoefficients = workspace.BlueCandidateCoefficients; + Span selectedRedCoefficients = workspace.SelectedRedCoefficients; + Span candidateRedCoefficients = workspace.RedCandidateCoefficients; + int skipContext = Av1TileWriter.GetSkipContext(macroBlock); + Span referenceCounts = stackalloc byte[Av1Constants.ReferenceFrameCount]; + Av1TileWriter.CollectNeighborReferenceCounts(macroBlock, referenceCounts); + int commonPredictionRate = writer.GetIsInterCost( + isInter: true, + Av1TileWriter.GetIntraInterContext(macroBlock)) + + writer.GetSingleReferenceCost(Av1ReferenceFrameType.Last, referenceCounts); + + int transformPartitionRate = 0; + if (frameHeader.TransformMode == Av1TransformMode.Select) + { + Av1NeighborArrayUnit transformContexts = this.picture.TransformFunctionContexts[tileIndex]; + int topIndex = transformContexts.GetTopIndex(blockOrigin); + int leftIndex = transformContexts.GetLeftIndex(blockOrigin); + int transformPartitionContext = Av1SymbolContextHelper.GetTransformPartitionContext( + transformContexts.Top[topIndex], + transformContexts.Left[leftIndex], + BlockSize, + LumaTransformSize); + + transformPartitionRate = writer.GetTransformPartitionCost(false, transformPartitionContext); + } + + long selectedCost = regularCost; + Av1MotionVector selectedVector = default; + Av1PredictionMode selectedMode = default; + int selectedReferenceIndex = 0; + bool selectedSkip = false; + Av1EncoderTransformBlockState selectedLumaState = default; + Av1EncoderTransformBlockState selectedBlueState = default; + Av1EncoderTransformBlockState selectedRedState = default; + bool hasInterWinner = false; + + ObuSequenceHeader sequenceHeader = this.picture.Sequence.SequenceHeader; + bool isSwitchable = frameHeader.InterpolationFilter == Av1InterpolationFilter.Switchable; + bool isDualFilter = sequenceHeader.EnableDualFilter; + Av1InterpolationFilter defaultFilter = isSwitchable ? Av1InterpolationFilter.Regular : frameHeader.InterpolationFilter; + Av1InterpolationFilter selectedVerticalFilter = defaultFilter; + Av1InterpolationFilter selectedHorizontalFilter = defaultFilter; + const int FilterCount = Av1SymbolContextHelper.SwitchableInterpolationFilterCount; + Span verticalFilterRates = stackalloc int[FilterCount]; + Span horizontalFilterRates = stackalloc int[FilterCount]; + int cheapestVerticalFilter = 0; + int cheapestHorizontalFilter = 0; + if (isSwitchable) + { + int verticalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext(modeInfo.Block, macroBlock, direction: 0); + int horizontalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext(modeInfo.Block, macroBlock, direction: 1); + for (int filterIndex = 0; filterIndex < FilterCount; filterIndex++) + { + Av1InterpolationFilter filter = (Av1InterpolationFilter)filterIndex; + verticalFilterRates[filterIndex] = writer.GetSwitchableInterpolationFilterCost(filter, verticalContext); + horizontalFilterRates[filterIndex] = isDualFilter ? writer.GetSwitchableInterpolationFilterCost(filter, horizontalContext) : 0; + if (verticalFilterRates[filterIndex] < verticalFilterRates[cheapestVerticalFilter]) + { + cheapestVerticalFilter = filterIndex; + } + + if (horizontalFilterRates[filterIndex] < horizontalFilterRates[cheapestHorizontalFilter]) + { + cheapestHorizontalFilter = filterIndex; + } + } + } + + // An integer luma displacement can still land between chroma samples. Test the finest active plane's + // phase before collapsing filter choices; otherwise odd translations would skip real chroma differences. + int horizontalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingX ? 1 : 0)) - 1; + int verticalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingY ? 1 : 0)) - 1; + + // Rank interpolation families with prediction-error modeling before running a full transform search. + // The selected inter reconstruction remains untouched while two existing prediction views alternate. + for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++) + { + modeInfo.Block.Mode = candidateModes[candidateIndex]; + bool writesFilters = Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo.Block); + Av1InterpolationFilter verticalFilter = defaultFilter; + Av1InterpolationFilter horizontalFilter = defaultFilter; + int filterRate = 0; + if (writesFilters) + { + bool hasHorizontalPhase = (candidateVectors[candidateIndex].Column & horizontalFractionMask) != 0; + bool hasVerticalPhase = (candidateVectors[candidateIndex].Row & verticalFractionMask) != 0; + int filterPairCount = isDualFilter ? FilterCount * FilterCount : FilterCount; + long bestModelCost = long.MaxValue; + bool bestPredictionUsesWorkspace = true; + Span bestLumaPrediction = workspace.LumaPrediction; + Span trialLumaPrediction = candidateLumaReconstruction; + Span bestBluePrediction = workspace.BluePrediction; + Span trialBluePrediction = candidateBlueReconstruction; + Span bestRedPrediction = workspace.RedPrediction; + Span trialRedPrediction = candidateRedReconstruction; + for (int filterPairIndex = 0; filterPairIndex < filterPairCount; filterPairIndex++) + { + int verticalFilterIndex = isDualFilter ? filterPairIndex / FilterCount : filterPairIndex; + int horizontalFilterIndex = isDualFilter ? filterPairIndex % FilterCount : filterPairIndex; + + // At zero phase every filter produces identical samples. Retain only the cheapest signaled + // choice for that axis, or for the common filter when neither axis has a fractional phase. + bool redundantFilter = isDualFilter + ? (!hasVerticalPhase && verticalFilterIndex != cheapestVerticalFilter) || + (!hasHorizontalPhase && horizontalFilterIndex != cheapestHorizontalFilter) + : !hasVerticalPhase && !hasHorizontalPhase && verticalFilterIndex != cheapestVerticalFilter; + + if (redundantFilter) + { + continue; + } + + Av1InterpolationFilter trialVerticalFilter = (Av1InterpolationFilter)verticalFilterIndex; + Av1InterpolationFilter trialHorizontalFilter = (Av1InterpolationFilter)horizontalFilterIndex; + int trialFilterRate = verticalFilterRates[verticalFilterIndex] + horizontalFilterRates[horizontalFilterIndex]; + long modelCost = this.GetInterFilterModelCost( + candidateVectors[candidateIndex], + blockOrigin, + block.HasChroma, + trialHorizontalFilter, + trialVerticalFilter, + trialFilterRate, + trialLumaPrediction, + trialBluePrediction, + trialRedPrediction); + + if (modelCost >= bestModelCost) + { + continue; + } + + bestModelCost = modelCost; + verticalFilter = trialVerticalFilter; + horizontalFilter = trialHorizontalFilter; + filterRate = trialFilterRate; + Span previousLumaPrediction = bestLumaPrediction; + bestLumaPrediction = trialLumaPrediction; + trialLumaPrediction = previousLumaPrediction; + Span previousBluePrediction = bestBluePrediction; + bestBluePrediction = trialBluePrediction; + trialBluePrediction = previousBluePrediction; + Span previousRedPrediction = bestRedPrediction; + bestRedPrediction = trialRedPrediction; + trialRedPrediction = previousRedPrediction; + bestPredictionUsesWorkspace = !bestPredictionUsesWorkspace; + } + + // Keep the winner in the prediction views before transforms reuse candidate reconstruction. + // This requires at most one copy per plane and never rebuilds the chosen interpolation. + if (!bestPredictionUsesWorkspace) + { + bestLumaPrediction.CopyTo(workspace.LumaPrediction); + if (block.HasChroma) + { + bestBluePrediction.CopyTo(workspace.BluePrediction); + bestRedPrediction.CopyTo(workspace.RedPrediction); + } + } + } + + long candidateCost = this.EvaluateInterCandidate( + writer, + blockOrigin, + tileIndex, + block.HasChroma, + commonPredictionRate + filterRate, + skipContext, + transformPartitionRate, + candidateVectors[candidateIndex], + candidateModes[candidateIndex], + writesFilters, + horizontalFilter, + verticalFilter, + candidateReferenceIndices[candidateIndex], + in referenceMotionVectors, + candidateLumaReconstruction, + candidateLumaCoefficients, + candidateBlueReconstruction, + candidateBlueCoefficients, + candidateRedReconstruction, + candidateRedCoefficients, + out bool candidateSkip, + out Av1EncoderTransformBlockState candidateLumaState, + out Av1EncoderTransformBlockState candidateBlueState, + out Av1EncoderTransformBlockState candidateRedState); + + // Strict replacement preserves predictor-stack, global, then new-motion order on equal RD cost. + if (candidateCost >= selectedCost) + { + continue; + } + + Span previousLumaReconstruction = selectedLumaReconstruction; + selectedLumaReconstruction = candidateLumaReconstruction; + candidateLumaReconstruction = previousLumaReconstruction; + + Span previousBlueReconstruction = selectedBlueReconstruction; + selectedBlueReconstruction = candidateBlueReconstruction; + candidateBlueReconstruction = previousBlueReconstruction; + + Span previousRedReconstruction = selectedRedReconstruction; + selectedRedReconstruction = candidateRedReconstruction; + candidateRedReconstruction = previousRedReconstruction; + + Span previousLumaCoefficients = selectedLumaCoefficients; + selectedLumaCoefficients = candidateLumaCoefficients; + candidateLumaCoefficients = previousLumaCoefficients; + + Span previousBlueCoefficients = selectedBlueCoefficients; + selectedBlueCoefficients = candidateBlueCoefficients; + candidateBlueCoefficients = previousBlueCoefficients; + + Span previousRedCoefficients = selectedRedCoefficients; + selectedRedCoefficients = candidateRedCoefficients; + candidateRedCoefficients = previousRedCoefficients; + + selectedCost = candidateCost; + selectedVector = candidateVectors[candidateIndex]; + selectedMode = candidateModes[candidateIndex]; + selectedHorizontalFilter = horizontalFilter; + selectedVerticalFilter = verticalFilter; + selectedReferenceIndex = candidateReferenceIndices[candidateIndex]; + selectedSkip = candidateSkip; + selectedLumaState = candidateLumaState; + selectedBlueState = candidateBlueState; + selectedRedState = candidateRedState; + hasInterWinner = true; + } + + // Inter trials never overwrite retained picture state. The complete intra result remains authoritative + // when no inter candidate strictly improves its rate-distortion cost. + if (!hasInterWinner) + { + return regularCost; + } + + Span retainedLumaCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.Y); + Span retainedLumaTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.Y); + + int lumaTransformIndex = this.codedAreaLuma / + Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + + CopyCandidate( + selectedLumaReconstruction, + selectedLumaCoefficients, + this.reconstruction.GetPlane(Av1Plane.Y), + blockOrigin, + retainedLumaCoefficients[this.codedAreaLuma..], + LumaTransformSize, + selectedLumaState, + ref retainedLumaTransformBlocks[lumaTransformIndex]); + + if (block.HasChroma) + { + ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig; + int subsamplingX = colorConfig.SubSamplingX ? 1 : 0; + int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; + Point chromaOrigin = Av1TileWriter.GetChromaBlockOrigin( + blockOrigin, + subsamplingX, + subsamplingY); + + Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize( + colorConfig.SubSamplingX, + colorConfig.SubSamplingY); + + Span retainedBlueCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.U); + Span retainedRedCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.V); + Span retainedBlueTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.U); + + Span retainedRedTransformBlocks = + this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.V); + + int chromaTransformIndex = this.codedAreaChroma / + Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + + CopyCandidate( + selectedBlueReconstruction, + selectedBlueCoefficients, + this.reconstruction.GetPlane(Av1Plane.U), + chromaOrigin, + retainedBlueCoefficients[this.codedAreaChroma..], + chromaTransformSize, + selectedBlueState, + ref retainedBlueTransformBlocks[chromaTransformIndex]); + + CopyCandidate( + selectedRedReconstruction, + selectedRedCoefficients, + this.reconstruction.GetPlane(Av1Plane.V), + chromaOrigin, + retainedRedCoefficients[this.codedAreaChroma..], + chromaTransformSize, + selectedRedState, + ref retainedRedTransformBlocks[chromaTransformIndex]); + } + + modeInfo.Block.Mode = selectedMode; + modeInfo.Block.Skip = selectedSkip; + modeInfo.Block.VerticalInterpolationFilter = selectedVerticalFilter; + modeInfo.Block.HorizontalInterpolationFilter = selectedHorizontalFilter; + block.ReferenceMotionVectorIndex = selectedReferenceIndex; + this.picture.SetDisplacementVector(modeInfoPosition, selectedVector); + return selectedCost; + } + + /// + /// Ranks a filter pair from visible prediction error using the reference curve model, without transforming samples. + /// + private long GetInterFilterModelCost( + Av1MotionVector vector, + Point blockOrigin, + bool hasChroma, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int filterRate, + Span lumaPrediction, + Span bluePrediction, + Span redPrediction) + { + const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; + const int InterpolationPrecisionBits = 4; + const int InterpolationPhaseMask = (1 << InterpolationPrecisionBits) - 1; + Av1EncoderInterPredictionWorkspace workspace = this.blockWorkspace.GetInterPredictionWorkspace(); + int planeCount = hasChroma ? 3 : 1; + int rate = filterRate; + long distortion = 0; + for (int planeIndex = 0; planeIndex < planeCount; planeIndex++) + { + Av1Plane plane = (Av1Plane)planeIndex; + int subsamplingX = plane == Av1Plane.Y ? 0 : this.source.ChromaSubsamplingX; + int subsamplingY = plane == Av1Plane.Y ? 0 : this.source.ChromaSubsamplingY; + Av1BlockSize planeBlockSize = BlockSize.GetSubsampled(subsamplingX != 0, subsamplingY != 0); + Av1TransformSize transformSize = BlockSize.GetMaxUvTransformSize(subsamplingX != 0, subsamplingY != 0); + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + Point planeOrigin = new(blockOrigin.X >> subsamplingX, blockOrigin.Y >> subsamplingY); + int sourceColumn = (planeOrigin.X << InterpolationPrecisionBits) + (vector.Column << (1 - subsamplingX)); + int sourceRow = (planeOrigin.Y << InterpolationPrecisionBits) + (vector.Row << (1 - subsamplingY)); + Point predictionOrigin = new(sourceColumn >> InterpolationPrecisionBits, sourceRow >> InterpolationPrecisionBits); + Span prediction = plane == Av1Plane.Y ? lumaPrediction : plane == Av1Plane.U ? bluePrediction : redPrediction; + Span residual = workspace.Residual[..(width * height)]; + TOperator.PrepareTranslationalInterPrediction( + this.source.GetPlane(plane), + planeOrigin, + this.reference.GetPlane(plane), + predictionOrigin, + horizontalFilter, + verticalFilter, + sourceColumn & InterpolationPhaseMask, + sourceRow & InterpolationPhaseMask, + prediction, + residual, + workspace.PredictionScratch, + transformSize, + this.bitDepth); + + int visibleWidth = Math.Min(width, ((this.source.Width + subsamplingX) >> subsamplingX) - planeOrigin.X); + int visibleHeight = Math.Min(height, ((this.source.Height + subsamplingY) >> subsamplingY) - planeOrigin.Y); + long squaredError = 0; + + // Cropped border samples participate in prediction, but not in the reference model's visible SSE. + // Full blocks use one SIMD reduction; only a partial right edge needs row-sized reductions. + if (visibleWidth == width) + { + squaredError = Av1ResidualBuilder.SumSquares(residual[..(width * visibleHeight)]); + } + else + { + for (int row = 0; row < visibleHeight; row++) + { + squaredError += Av1ResidualBuilder.SumSquares(residual.Slice(row * width, visibleWidth)); + } + } + + int normalizationShift = (this.bitDepth.GetBitCount() - 8) * 2; + if (normalizationShift != 0) + { + squaredError = (squaredError + (1L << (normalizationShift - 1))) >> normalizationShift; + } + + int acQuantizer = Av1QuantizationLookup.GetAcQuant( + this.quantization.QIndex[0], + this.quantization.DeltaQAc[planeIndex], + this.bitDepth); + + Av1RateDistortion.ModelPredictionError( + planeBlockSize, + squaredError, + visibleWidth * visibleHeight, + acQuantizer, + this.bitDepth, + this.rateMultiplier, + out int planeRate, + out long planeDistortion); + + rate += planeRate; + distortion += planeDistortion; + } + + return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); + } + + /// + /// Evaluates one inter mode through prediction, transform, coefficient, skip, and distortion selection. + /// + private long EvaluateInterCandidate( + Av1SymbolEncoder writer, + Point blockOrigin, + ushort tileIndex, + bool hasChroma, + int commonPredictionRate, + int skipContext, + int transformPartitionRate, + Av1MotionVector vector, + Av1PredictionMode mode, + bool usePreparedPrediction, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + int referenceMotionVectorIndex, + in Av1ReferenceMotionVectors referenceMotionVectors, + Span lumaReconstruction, + Span lumaCoefficients, + Span blueReconstruction, + Span blueCoefficients, + Span redReconstruction, + Span redCoefficients, + out bool skip, + out Av1EncoderTransformBlockState lumaState, + out Av1EncoderTransformBlockState blueState, + out Av1EncoderTransformBlockState redState) + { + const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; + const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8; + Av1EncoderInterPredictionWorkspace workspace = + this.blockWorkspace.GetInterPredictionWorkspace(); + + Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Luminance, + this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex], + blockOrigin, + BlockSize, + LumaTransformSize); + + this.EvaluateInterPlane( + writer, + vector, + Av1Plane.Y, + Av1ComponentType.Luminance, + mode, + usePreparedPrediction, + horizontalFilter, + verticalFilter, + this.reference.GetPlane(Av1Plane.Y), + blockOrigin, + 0, + 0, + LumaTransformSize, + Av1TransformType.AllTransformTypes, + lumaContext, + workspace.LumaPrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + lumaReconstruction, + lumaCoefficients, + out lumaState, + out int lumaRate, + out long lumaDistortion, + out bool hasEmptyLuma, + out Av1EncoderTransformBlockState emptyLumaState, + out long emptyLumaDistortion); + + ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig; + int subsamplingX = colorConfig.SubSamplingX ? 1 : 0; + int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; + Point chromaOrigin = Av1TileWriter.GetChromaBlockOrigin( + blockOrigin, + subsamplingX, + subsamplingY); + + Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize( + colorConfig.SubSamplingX, + colorConfig.SubSamplingY); + + int blueRate = 0; + int redRate = 0; + long blueDistortion = 0; + long redDistortion = 0; + long emptyBlueDistortion = 0; + long emptyRedDistortion = 0; + bool hasEmptyBlue = true; + bool hasEmptyRed = true; + blueState = default; + redState = default; + Av1EncoderTransformBlockState emptyBlueState = default; + Av1EncoderTransformBlockState emptyRedState = default; + if (hasChroma) + { + Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled( + colorConfig.SubSamplingX, + colorConfig.SubSamplingY); + + Av1TransformBlockContext blueContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Chroma, + this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex], + chromaOrigin, + chromaBlockSize, + chromaTransformSize); + + Av1TransformBlockContext redContext = Av1TileWriter.GetTransformBlockContexts( + Av1ComponentType.Chroma, + this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex], + chromaOrigin, + chromaBlockSize, + chromaTransformSize); + + Av1TransformType chromaTransformType = lumaState.TransformType; + Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType( + chromaTransformSize, + isInter: true, + this.picture.Parent.FrameHeader.UseReducedTransformSet); + + if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet)) + { + chromaTransformType = Av1TransformType.DctDct; + } + + this.EvaluateInterPlane( + writer, + vector, + Av1Plane.U, + Av1ComponentType.Chroma, + mode, + usePreparedPrediction, + horizontalFilter, + verticalFilter, + this.reference.GetPlane(Av1Plane.U), + blockOrigin, + subsamplingX, + subsamplingY, + chromaTransformSize, + chromaTransformType, + blueContext, + workspace.BluePrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + blueReconstruction, + blueCoefficients, + out blueState, + out blueRate, + out blueDistortion, + out hasEmptyBlue, + out emptyBlueState, + out emptyBlueDistortion); + + this.EvaluateInterPlane( + writer, + vector, + Av1Plane.V, + Av1ComponentType.Chroma, + mode, + usePreparedPrediction, + horizontalFilter, + verticalFilter, + this.reference.GetPlane(Av1Plane.V), + blockOrigin, + subsamplingX, + subsamplingY, + chromaTransformSize, + chromaTransformType, + redContext, + workspace.RedPrediction, + workspace.Residual, + workspace.TransformReconstruction, + workspace.TransformCoefficients, + redReconstruction, + redCoefficients, + out redState, + out redRate, + out redDistortion, + out hasEmptyRed, + out emptyRedState, + out emptyRedDistortion); + } + + int predictionRate = commonPredictionRate + + this.GetInterModeRate( + writer, + mode, + vector, + referenceMotionVectorIndex, + in referenceMotionVectors); + + int codedRate = predictionRate + + writer.GetSkipCost(false, skipContext) + + transformPartitionRate + + lumaRate + + blueRate + + redRate; + + long codedDistortion = lumaDistortion + blueDistortion + redDistortion; + long selectedCost = Av1RateDistortion.GetCost(this.rateMultiplier, codedRate, codedDistortion); + skip = false; + if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed) + { + int skipRate = predictionRate + writer.GetSkipCost(true, skipContext); + long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion; + long skipCost = Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion); + if (skipCost < selectedCost) + { + selectedCost = skipCost; + skip = true; + workspace.LumaPrediction[..LumaTransformSize.GetSize2d()].CopyTo(lumaReconstruction); + lumaCoefficients[..LumaTransformSize.GetSize2d()].Clear(); + lumaState = emptyLumaState; + if (hasChroma) + { + int chromaSampleCount = chromaTransformSize.GetSize2d(); + workspace.BluePrediction[..chromaSampleCount].CopyTo(blueReconstruction); + workspace.RedPrediction[..chromaSampleCount].CopyTo(redReconstruction); + blueCoefficients[..chromaSampleCount].Clear(); + redCoefficients[..chromaSampleCount].Clear(); + blueState = emptyBlueState; + redState = emptyRedState; + } + } + } + + return selectedCost; + } + + /// + /// Searches a bounded full-pixel neighborhood around the spatial reference vector. + /// + /// The live tile entropy model used to measure vector syntax. + /// The current 8x8 luma origin. + /// The differential reference from the spatial candidate stack. + /// The selected dynamic-reference-list entry. + /// The lowest-cost full-pixel vector found by the effort-scaled search. + private Av1MotionVector FindInterMotionVector( + Av1SymbolEncoder writer, + Point blockOrigin, + Av1MotionVector referenceVector, + int referenceMotionVectorIndex) + { + int effortShift = this.effort - MinimumInterMotionSearchEffort; + int searchRadius = Math.Min( + MinimumInterMotionSearchRadius << effortShift, + Av1EncoderFrame.LumaBorder); + + int referenceColumn = referenceVector.Column >> Av1MotionVector.SubpixelBits; + int referenceRow = referenceVector.Row >> Av1MotionVector.SubpixelBits; + int minimumColumn = Math.Max(-Av1EncoderFrame.LumaBorder, referenceColumn - searchRadius); + int maximumColumn = Math.Min(Av1EncoderFrame.LumaBorder, referenceColumn + searchRadius); + int minimumRow = Math.Max(-Av1EncoderFrame.LumaBorder, referenceRow - searchRadius); + int maximumRow = Math.Min(Av1EncoderFrame.LumaBorder, referenceRow + searchRadius); + Point best = new( + Av1Math.Clamp(referenceColumn, minimumColumn, maximumColumn), + Av1Math.Clamp(referenceRow, minimumRow, maximumRow)); + + ref Av1ReferenceMotionVectors referenceMotionVectors = ref this.blockWorkspace.ReferenceMotionVectors; + Av1MotionVector bestVector = new( + best.Y * Av1MotionVector.SubpixelScale, + best.X * Av1MotionVector.SubpixelScale); + + long bestCost = this.GetInterMotionCandidateCost( + writer, + blockOrigin, + bestVector, + Av1PredictionMode.NewMotionVector, + referenceMotionVectorIndex, + in referenceMotionVectors); + + for (int step = searchRadius; step > 0; step >>= 1) + { + Point stageBest = best; + long stageBestCost = bestCost; + for (int directionIndex = 0; directionIndex < InterMotionSearchDirectionCount; directionIndex++) + { + Point direction = GetInterMotionSearchDirection(directionIndex); + Point candidate = new( + best.X + (direction.X * step), + best.Y + (direction.Y * step)); + + if (candidate.X < minimumColumn || candidate.X > maximumColumn || + candidate.Y < minimumRow || candidate.Y > maximumRow) + { + continue; + } + + Av1MotionVector candidateVector = new( + candidate.Y * Av1MotionVector.SubpixelScale, + candidate.X * Av1MotionVector.SubpixelScale); + + long candidateCost = this.GetInterMotionCandidateCost( + writer, + blockOrigin, + candidateVector, + Av1PredictionMode.NewMotionVector, + referenceMotionVectorIndex, + in referenceMotionVectors); + + // Strict replacement preserves the earlier reference-centered search position on ties. + if (candidateCost < stageBestCost) + { + stageBestCost = candidateCost; + stageBest = candidate; + } + } + + best = stageBest; + bestCost = stageBestCost; + } + + bestVector = new( + best.Y * Av1MotionVector.SubpixelScale, + best.X * Av1MotionVector.SubpixelScale); + + if (this.effort < MinimumSubpixelMotionSearchEffort) + { + return bestVector; + } + + int minimumSubpixel = (-Av1EncoderFrame.LumaBorder + FractionalInterpolationBorder) * + Av1MotionVector.SubpixelScale; + + int maximumSubpixel = (Av1EncoderFrame.LumaBorder - FractionalInterpolationBorder) * + Av1MotionVector.SubpixelScale; + + if (bestVector.Column < minimumSubpixel || bestVector.Column > maximumSubpixel || + bestVector.Row < minimumSubpixel || bestVector.Row > maximumSubpixel) + { + return bestVector; + } + + int finalStep = this.effort >= MinimumHighPrecisionMotionSearchEffort ? 1 : 2; + for (int step = Av1MotionVector.SubpixelScale >> 1; step >= finalStep; step >>= 1) + { + Av1MotionVector stageBest = bestVector; + long stageBestCost = bestCost; + for (int directionIndex = 0; directionIndex < InterMotionSearchDirectionCount; directionIndex++) + { + Point direction = GetInterMotionSearchDirection(directionIndex); + Av1MotionVector candidate = new( + bestVector.Row + (direction.Y * step), + bestVector.Column + (direction.X * step)); + + if (candidate.Column < minimumSubpixel || candidate.Column > maximumSubpixel || + candidate.Row < minimumSubpixel || candidate.Row > maximumSubpixel) + { + continue; + } + + long candidateCost = this.GetInterMotionCandidateCost( + writer, + blockOrigin, + candidate, + Av1PredictionMode.NewMotionVector, + referenceMotionVectorIndex, + in referenceMotionVectors); + + // Each precision stage remains centered on its incoming winner; strict replacement keeps + // the integer or coarser fractional vector when an interpolated candidate only ties it. + if (candidateCost < stageBestCost) + { + stageBestCost = candidateCost; + stageBest = candidate; + } + } + + bestVector = stageBest; + bestCost = stageBestCost; + } + + return bestVector; + } + + /// + /// Combines normalized prediction error with the exact mode and vector syntax rate. + /// + /// The live tile entropy model. + /// The current 8x8 luma origin. + /// The candidate motion vector. + /// The candidate single-reference inter mode. + /// The selected dynamic-reference-list entry. + /// The current spatial candidate stack. + /// The rate-distortion search cost. + private long GetInterMotionCandidateCost( + Av1SymbolEncoder writer, + Point blockOrigin, + Av1MotionVector vector, + Av1PredictionMode mode, + int referenceMotionVectorIndex, + in Av1ReferenceMotionVectors referenceMotionVectors) + { + long predictionError; + if (((vector.Row | vector.Column) & (Av1MotionVector.SubpixelScale - 1)) == 0) + { + Point predictionOrigin = new( + blockOrigin.X + (vector.Column >> Av1MotionVector.SubpixelBits), + blockOrigin.Y + (vector.Row >> Av1MotionVector.SubpixelBits)); + + predictionError = TOperator.GetInterPredictionError( + this.source.GetPlane(Av1Plane.Y), + blockOrigin, + this.reference.GetPlane(Av1Plane.Y), + predictionOrigin, + this.bitDepth); + } + else + { + const Av1TransformSize SearchTransformSize = Av1TransformSize.Size8x8; + Av1EncoderInterPredictionWorkspace workspace = + this.blockWorkspace.GetInterPredictionWorkspace(); + + int sourceColumnQ4 = (blockOrigin.X << 4) + (vector.Column << 1); + int sourceRowQ4 = (blockOrigin.Y << 4) + (vector.Row << 1); + Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4); + ObuFrameHeader frameHeader = this.picture.Parent.FrameHeader; + + // Fractional candidates must pass through the same interpolation and residual kernels used by + // final reconstruction; comparing only their integer origins would choose the wrong phase. + TOperator.PrepareTranslationalInterPrediction( + this.source.GetPlane(Av1Plane.Y), + blockOrigin, + this.reference.GetPlane(Av1Plane.Y), + predictionOrigin, + frameHeader.InterpolationFilter == Av1InterpolationFilter.Switchable ? Av1InterpolationFilter.Regular : frameHeader.InterpolationFilter, + frameHeader.InterpolationFilter == Av1InterpolationFilter.Switchable ? Av1InterpolationFilter.Regular : frameHeader.InterpolationFilter, + sourceColumnQ4 & 15, + sourceRowQ4 & 15, + workspace.LumaPrediction, + workspace.Residual, + workspace.PredictionScratch, + SearchTransformSize, + this.bitDepth); + + predictionError = Av1ResidualBuilder.SumSquares(workspace.Residual); + int normalizationShift = (this.bitDepth.GetBitCount() - 8) * 2; + if (normalizationShift != 0) + { + predictionError = (predictionError + (1L << (normalizationShift - 1))) >> + normalizationShift; + } + } + + int rate = this.GetInterModeRate( + writer, + mode, + vector, + referenceMotionVectorIndex, + in referenceMotionVectors); + + return Av1RateDistortion.GetCost(this.rateMultiplier, rate, predictionError); + } + + /// + /// Measures the complete mode, dynamic-reference-list, and differential-vector syntax for one candidate. + /// + /// The live tile entropy model. + /// The candidate single-reference inter mode. + /// The candidate motion vector. + /// The selected dynamic-reference-list entry. + /// The current spatial candidate stack. + /// The syntax rate in 1/512-bit units. + private int GetInterModeRate( + Av1SymbolEncoder writer, + Av1PredictionMode mode, + Av1MotionVector vector, + int referenceMotionVectorIndex, + in Av1ReferenceMotionVectors referenceMotionVectors) + { + int rate = writer.GetInterModeCost(mode, referenceMotionVectors.ModeContext); + if (mode == Av1PredictionMode.NearMotionVector) + { + for (int index = 1; index < 3 && referenceMotionVectors.Count > index + 1; index++) + { + bool advance = referenceMotionVectorIndex >= index; + int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index); + rate += writer.GetDynamicReferenceListCost(advance, context); + if (!advance) + { + break; + } + } + + return rate; + } + + if (mode != Av1PredictionMode.NewMotionVector) + { + return rate; + } + + for (int index = 0; index < 2 && referenceMotionVectors.Count > index + 1; index++) + { + bool advance = referenceMotionVectorIndex > index; + int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index); + rate += writer.GetDynamicReferenceListCost(advance, context); + if (!advance) + { + break; + } + } + + Av1MotionVector reference = referenceMotionVectors.GetNewReference(referenceMotionVectorIndex); + return rate + writer.GetMotionVectorCost( + vector, + reference, + this.picture.Parent.FrameHeader.MotionVectorPrecision); + } + + /// + /// Gets one cardinal or diagonal search direction in stable reference order. + /// + /// The zero-based direction index. + /// The unit full-pixel direction. + private static Point GetInterMotionSearchDirection(int index) + => index switch + { + 0 => new Point(0, -1), + 1 => new Point(0, 1), + 2 => new Point(-1, 0), + 3 => new Point(1, 0), + 4 => new Point(-1, -1), + 5 => new Point(1, 1), + 6 => new Point(1, -1), + _ => new Point(-1, 1) + }; + + /// + /// Builds one plane prediction and selects its transform without repeating interpolation for each transform type. + /// + private void EvaluateInterPlane( + Av1SymbolEncoder writer, + Av1MotionVector vector, + Av1Plane plane, + Av1ComponentType componentType, + Av1PredictionMode predictionMode, + bool usePreparedPrediction, + Av1InterpolationFilter horizontalFilter, + Av1InterpolationFilter verticalFilter, + Buffer2DRegion referencePlane, + Point lumaOrigin, + int subsamplingX, + int subsamplingY, + Av1TransformSize transformSize, + Av1TransformType transformTypeSelection, + Av1TransformBlockContext blockContext, + Span prediction, + Span residual, + Span transformReconstruction, + Span transformCoefficients, + Span selectedReconstruction, + Span selectedCoefficients, + out Av1EncoderTransformBlockState selectedState, + out int selectedRate, + out long selectedDistortion, + out bool hasEmptyTransform, + out Av1EncoderTransformBlockState emptyState, + out long emptyDistortion) + { + Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY); + int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX)); + int sourceRowQ4 = (planeOrigin.Y << 4) + (vector.Row << (1 - subsamplingY)); + Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4); + int sampleCount = transformSize.GetSize2d(); + Buffer2DRegion sourcePlane = this.source.GetPlane(plane); + if (usePreparedPrediction) + { + TOperator.SubtractPrediction(sourcePlane, planeOrigin, prediction[..sampleCount], residual[..sampleCount], transformSize); + } + else if (predictionMode >= Av1PredictionMode.InterModeStart) + { + Span predictionScratch = this.blockWorkspace + .GetInterPredictionWorkspace() + .PredictionScratch; + + // Reference-frame modes use the complete interpolation pipeline even when the current zero-phase + // global vector reduces to a SIMD copy. Later fractional vectors therefore share decoder arithmetic. + TOperator.PrepareTranslationalInterPrediction( + sourcePlane, + planeOrigin, + referencePlane, + predictionOrigin, + horizontalFilter, + verticalFilter, + sourceColumnQ4 & 15, + sourceRowQ4 & 15, + prediction[..sampleCount], + residual[..sampleCount], + predictionScratch, + transformSize, + this.picture.Sequence.SequenceHeader.ColorConfig.BitDepth); + } + else + { + // Intra-block copy has its own bilinear half-sample rules and reads the current reconstruction. + TOperator.PrepareIntraBlockCopyPrediction( + sourcePlane, + planeOrigin, + referencePlane, + predictionOrigin, + (sourceColumnQ4 & 15) != 0, + (sourceRowQ4 & 15) != 0, + prediction[..sampleCount], + residual[..sampleCount], + transformSize); + } + + // Motion compensation and subtraction do not depend on transform type. Keep them outside the + // transform loop so exhaustive luma search traverses the source and reference blocks only once. + Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType( + transformSize, + isInter: true, + this.picture.Parent.FrameHeader.UseReducedTransformSet); + + Av1TransformType firstTransformType = transformTypeSelection == Av1TransformType.AllTransformTypes + ? Av1TransformType.DctDct + : transformTypeSelection; + Av1TransformType transformTypeLimit = transformTypeSelection == Av1TransformType.AllTransformTypes + ? Av1TransformType.AllTransformTypes + : (Av1TransformType)((int)transformTypeSelection + 1); + + long bestCost = long.MaxValue; + selectedState = default; + selectedRate = 0; + selectedDistortion = 0; + hasEmptyTransform = false; + emptyState = default; + emptyDistortion = 0; + + // The candidate and best spans alternate ownership whenever a transform improves the result. + // This mirrors the reference's buffer-pointer swap and replaces a copy on every improvement + // with at most one normalization copy after the transform search. + Span candidateReconstruction = transformReconstruction[..sampleCount]; + Span candidateCoefficients = transformCoefficients[..sampleCount]; + Span bestReconstruction = selectedReconstruction[..sampleCount]; + Span bestCoefficients = selectedCoefficients[..sampleCount]; + bool bestUsesSelectedStorage = true; + for (Av1TransformType transformType = firstTransformType; + transformType < transformTypeLimit; + transformType++) + { + if (!transformType.IsExtendedSetUsed(transformSetType)) + { + continue; + } + + Av1EncoderTransformBlockState candidateState = default; + long candidateDistortion = TOperator.EncodePredictionCandidate( + this.blockWorkspace, + sourcePlane, + planeOrigin, + prediction[..sampleCount], + residual[..sampleCount], + candidateReconstruction, + transformSize.GetWidth(), + candidateCoefficients, + transformSize, + transformType, + plane, + this.quantization.QIndex[0], + this.quantization.DeltaQDc[(int)plane], + this.quantization.DeltaQAc[(int)plane], + this.bitDepth, + ref candidateState); + + int candidateRate = writer.GetCoefficientCost( + transformSize, + transformType, + predictionMode, + candidateCoefficients, + componentType, + blockContext, + candidateState.EndOfBlock, + this.picture.Parent.FrameHeader.UseReducedTransformSet, + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: true); + + long candidateCost = Av1RateDistortion.GetCost( + this.rateMultiplier, + candidateRate, + candidateDistortion); + + if (candidateCost < bestCost) + { + Span previousBestReconstruction = bestReconstruction; + bestReconstruction = candidateReconstruction; + candidateReconstruction = previousBestReconstruction; + + Span previousBestCoefficients = bestCoefficients; + bestCoefficients = candidateCoefficients; + candidateCoefficients = previousBestCoefficients; + bestUsesSelectedStorage = !bestUsesSelectedStorage; + bestCost = candidateCost; + selectedState = candidateState; + selectedRate = candidateRate; + selectedDistortion = candidateDistortion; + } + + if (candidateState.EndOfBlock == 0 && + (!hasEmptyTransform || candidateDistortion < emptyDistortion)) + { + hasEmptyTransform = true; + emptyState = candidateState; + emptyDistortion = candidateDistortion; + } + } + + // Callers retain the designated selected spans after this scratch workspace is reused by the + // next plane or motion vector, so normalize only when the final best result occupies scratch. + if (!bestUsesSelectedStorage) + { + bestReconstruction.CopyTo(selectedReconstruction); + bestCoefficients.CopyTo(selectedCoefficients); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs deleted file mode 100644 index 5fe0803227..0000000000 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Six Labors Split License. - -using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; -using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; -using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; -using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; - -namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; - -/// -/// Encodes one range-coded all-intra tile payload. -/// -internal sealed partial class Av1IntraTileWriter : IAv1TileWriter -{ - private readonly ReadOnlyMemory tileData; - private readonly int tileDataLength; - - /// - /// Initializes a new instance of the class for eight-bit samples. - /// - /// The operation-owned symbol encoder that retains the tile output memory. - /// The coded source frame. - /// The reconstructed frame updated during encoding. - /// The frame coding and mode-information state. - /// The frame-owned quantized coefficient and transform state. - /// The reusable partition and final-block decision workspace. - /// The reusable block arithmetic workspace. - /// The mode-search effort in the inclusive range zero through ten. - public Av1IntraTileWriter( - Av1SymbolEncoder writer, - Av1EncoderFrame source, - Av1EncoderFrame reconstruction, - Av1PictureControlSet picture, - Av1EncoderCoefficientBuffer coefficientBuffer, - Av1EncoderSuperblockWorkspace superblockWorkspace, - Av1EncoderBlockWorkspace blockWorkspace, - int effort) - { - this.tileData = Encode( - writer, - source, - reconstruction, - picture, - coefficientBuffer, - superblockWorkspace, - blockWorkspace, - effort, - out this.tileDataLength); - } - - /// - /// Initializes a new instance of the class for high-bit-depth samples. - /// - /// The operation-owned symbol encoder that retains the tile output memory. - /// The coded source frame. - /// The reconstructed frame updated during encoding. - /// The frame coding and mode-information state. - /// The frame-owned quantized coefficient and transform state. - /// The reusable partition and final-block decision workspace. - /// The reusable block arithmetic workspace. - /// The mode-search effort in the inclusive range zero through ten. - public Av1IntraTileWriter( - Av1SymbolEncoder writer, - Av1EncoderFrame source, - Av1EncoderFrame reconstruction, - Av1PictureControlSet picture, - Av1EncoderCoefficientBuffer coefficientBuffer, - Av1EncoderSuperblockWorkspace superblockWorkspace, - Av1EncoderBlockWorkspace blockWorkspace, - int effort) - { - this.tileData = Encode( - writer, - source, - reconstruction, - picture, - coefficientBuffer, - superblockWorkspace, - blockWorkspace, - effort, - out this.tileDataLength); - } - - /// - public ReadOnlySpan GetTileData(int tileNum) => this.tileData.Span[..this.tileDataLength]; - - private static ReadOnlyMemory Encode( - Av1SymbolEncoder writer, - Av1EncoderFrame source, - Av1EncoderFrame reconstruction, - Av1PictureControlSet picture, - Av1EncoderCoefficientBuffer coefficientBuffer, - Av1EncoderSuperblockWorkspace superblockWorkspace, - Av1EncoderBlockWorkspace blockWorkspace, - int effort, - out int tileDataLength) - where TSample : unmanaged - where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator - { - ObuFrameHeader frameHeader = picture.Parent.FrameHeader; - ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader; - const ushort TileIndex = 0; - Av1TileInfo tile = new(0, 0, frameHeader); - Av1Superblock superblock = new() - { - Workspace = superblockWorkspace, - TileInfo = tile - }; - - Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart); - Av1TileWriter.Av1EntropyCodingContext entropyContext = new() - { - MacroBlock = new Av1MacroBlockD { Tile = tile }, - MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition) - }; - - int superblockModeInfoSize = sequenceHeader.SuperblockModeInfoSize; - int superblockShift = sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2; - if (frameHeader.AllowIntraBlockCopy) - { - // Hash the visible source once before reconstruction begins so candidate discovery never depends - // on coding order and the workspace can be reused as compact bucket links afterward. - picture.IntraBlockCopySearch.Initialize( - source.View.GetPlane(Av1Plane.Y)); - } - - for (int modeInfoRow = tile.ModeInfoRowStart; - modeInfoRow < tile.ModeInfoRowEnd; - modeInfoRow += superblockModeInfoSize) - { - for (int modeInfoColumn = tile.ModeInfoColumnStart; - modeInfoColumn < tile.ModeInfoColumnEnd; - modeInfoColumn += superblockModeInfoSize) - { - int superblockRow = modeInfoRow >> superblockShift; - int superblockColumn = modeInfoColumn >> superblockShift; - superblock.Index = (superblockRow * coefficientBuffer.SuperblockColumnCount) + superblockColumn; - entropyContext.SuperblockOrigin = new Point( - modeInfoColumn << Av1Constants.ModeInfoSizeLog2, - modeInfoRow << Av1Constants.ModeInfoSizeLog2); - - Av1IntraSuperblockEncoder.Prepare( - picture, - superblock, - entropyContext.SuperblockOrigin); - - Av1IntraSuperblockEncoder.ModeDecision blockEncoder = new( - source, - reconstruction, - picture, - superblock, - coefficientBuffer, - blockWorkspace, - effort); - - Av1TileWriter.WriteSuperblock( - picture, - entropyContext, - writer, - superblock, - coefficientBuffer, - TileIndex, - ref blockEncoder); - } - } - - return writer.Exit(out tileDataLength); - } -} diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs index 4e512abba9..c418d163f9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; @@ -51,6 +52,57 @@ internal static partial class Av1ResidualBuilder /// The prediction sample. /// The signed residual. public static abstract short Subtract(TSample source, TSample prediction); + + /// + /// Measures one scalar absolute sample difference. + /// + /// The source sample. + /// The prediction sample. + /// The sum of absolute differences. + public static abstract int SumAbsoluteDifferences(TSample source, TSample prediction); + + /// + /// Measures eight absolute sample differences; byte inputs occupy only the lower eight lanes. + /// + /// The eight source samples. + /// The eight prediction samples. + /// The sum of absolute differences. + public static abstract int SumAbsoluteDifferences(Vector128 source, Vector128 prediction); + + /// + /// Measures four eight-sample predictions, returning their costs in candidate order. + /// Byte inputs occupy only the lower eight lanes. + /// + /// The eight source samples. + /// The eight samples for candidate 0. + /// The eight samples for candidate 1. + /// The eight samples for candidate 2. + /// The eight samples for candidate 3. + /// Four absolute-difference sums in increasing candidate order. + public static abstract Vector128 SumFourAbsoluteDifferences( + Vector128 source, + Vector128 prediction0, + Vector128 prediction1, + Vector128 prediction2, + Vector128 prediction3); + + /// + /// Calculates one squared difference and returns its signed difference for the first moment. + /// + /// The source sample. + /// The prediction sample. + /// The signed sum of source-minus-prediction differences. + /// The sum of squared differences. + public static abstract int SumSquaredDifferences(TSample source, TSample prediction, out int sum); + + /// + /// Calculates eight squared differences and their signed sum; byte inputs occupy only the lower eight lanes. + /// + /// The eight source samples. + /// The eight prediction samples. + /// The signed sum of source-minus-prediction differences. + /// The sum of squared differences. + public static abstract int SumSquaredDifferences(Vector128 source, Vector128 prediction, out int sum); } /// @@ -58,6 +110,59 @@ internal static partial class Av1ResidualBuilder /// internal readonly struct ByteOperator : IResidualOperator { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumAbsoluteDifferences(byte source, byte prediction) => Math.Abs(Subtract(source, prediction)); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumAbsoluteDifferences(Vector128 source, Vector128 prediction) + { + // Widen byte samples before subtraction; twelve-bit word samples already fit signed-short lanes. + // Eight absolute residuals sum to at most 32760, so the signed-short horizontal sum remains exact. + Vector128 difference = Vector128.WidenLower(source).AsInt16() - Vector128.WidenLower(prediction).AsInt16(); + return Vector128.Sum(Vector128.Abs(difference)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 SumFourAbsoluteDifferences( + Vector128 source, + Vector128 prediction0, + Vector128 prediction1, + Vector128 prediction2, + Vector128 prediction3) + { + // Reuse the source conversion across all four candidates. Each reduction contributes one independent + // 32-bit lane, allowing the traversal to accumulate all eight rows without extracting candidate costs. + Vector128 sourceSamples = Vector128.WidenLower(source).AsInt16(); + return Vector128.Create( + (int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction0).AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction1).AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction2).AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction3).AsInt16()))); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumSquaredDifferences(byte source, byte prediction, out int sum) + { + sum = Subtract(source, prediction); + return sum * sum; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumSquaredDifferences(Vector128 source, Vector128 prediction, out int sum) + { + Vector128 difference = Vector128.WidenLower(source).AsInt16() - Vector128.WidenLower(prediction).AsInt16(); + sum = Vector128.Sum(difference); + + // The shared square reduction widens to int before multiplying. All eight twelve-bit squares + // fit in the returned int; no short multiplication or premature bit-depth rounding is permitted. + return (int)SumSquares(difference); + } + /// public static Vector128 Subtract(Vector128 source, Vector128 prediction, out Vector128 upper) { @@ -91,6 +196,59 @@ internal static partial class Av1ResidualBuilder /// internal readonly struct UInt16Operator : IResidualOperator { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumAbsoluteDifferences(ushort source, ushort prediction) => Math.Abs(Subtract(source, prediction)); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumAbsoluteDifferences(Vector128 source, Vector128 prediction) + { + // Widen byte samples before subtraction; twelve-bit word samples already fit signed-short lanes. + // Eight absolute residuals sum to at most 32760, so the signed-short horizontal sum remains exact. + Vector128 difference = source.AsInt16() - prediction.AsInt16(); + return Vector128.Sum(Vector128.Abs(difference)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 SumFourAbsoluteDifferences( + Vector128 source, + Vector128 prediction0, + Vector128 prediction1, + Vector128 prediction2, + Vector128 prediction3) + { + // Reuse the source conversion across all four candidates. Each reduction contributes one independent + // 32-bit lane, allowing the traversal to accumulate all eight rows without extracting candidate costs. + Vector128 sourceSamples = source.AsInt16(); + return Vector128.Create( + (int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction0.AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction1.AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction2.AsInt16())), + (int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction3.AsInt16()))); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumSquaredDifferences(ushort source, ushort prediction, out int sum) + { + sum = Subtract(source, prediction); + return sum * sum; + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int SumSquaredDifferences(Vector128 source, Vector128 prediction, out int sum) + { + Vector128 difference = source.AsInt16() - prediction.AsInt16(); + sum = Vector128.Sum(difference); + + // The shared square reduction widens to int before multiplying. All eight twelve-bit squares + // fit in the returned int; no short multiplication or premature bit-depth rounding is permitted. + return (int)SumSquares(difference); + } + /// public static Vector128 Subtract(Vector128 source, Vector128 prediction, out Vector128 upper) { diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs index f4dac45cc7..3a3db74803 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs @@ -12,6 +12,226 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; /// internal static partial class Av1ResidualBuilder { + /// + /// The width and height of the encoder's fixed motion-search block, in samples. + /// + private const int SearchBlockDimension = 8; + + /// + /// Calculates the sum of absolute differences for an 8x8 block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The prediction samples starting at the block origin. + /// The prediction row stride, in samples. + /// The sum of absolute sample differences. + public static int SumAbsoluteDifferences8x8(ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride) + => SumAbsoluteDifferences8x8(source, sourceStride, prediction, predictionStride); + + /// + /// Measures four horizontally adjacent 8x8 predictions against one source block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The first prediction, with three additional samples available at the right of each row. + /// The prediction row stride, in samples. + /// The four results in increasing horizontal-offset order. + public static void SumFourAbsoluteDifferences8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, Span sums) + => SumFourAbsoluteDifferences8x8(source, sourceStride, prediction, predictionStride, sums); + + /// + /// Calculates the signed sum and squared sum of differences for an 8x8 block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The prediction samples starting at the block origin. + /// The prediction row stride, in samples. + /// The signed sum of sample differences. + /// The sum of squared sample differences. + public static void GetMoments8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, out int sum, out int sumOfSquares) + => GetMoments8x8(source, sourceStride, prediction, predictionStride, out sum, out sumOfSquares); + + /// + /// Calculates the sum of absolute differences for an 8x8 block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The prediction samples starting at the block origin. + /// The prediction row stride, in samples. + /// The sum of absolute sample differences. + public static int SumAbsoluteDifferences8x8(ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride) + => SumAbsoluteDifferences8x8(source, sourceStride, prediction, predictionStride); + + /// + /// Measures four horizontally adjacent 8x8 predictions against one source block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The first prediction, with three additional samples available at the right of each row. + /// The prediction row stride, in samples. + /// The four results in increasing horizontal-offset order. + public static void SumFourAbsoluteDifferences8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, Span sums) + => SumFourAbsoluteDifferences8x8(source, sourceStride, prediction, predictionStride, sums); + + /// + /// Calculates the signed sum and squared sum of differences for an 8x8 block. + /// + /// The source samples starting at the block origin. + /// The source row stride, in samples. + /// The prediction samples starting at the block origin. + /// The prediction row stride, in samples. + /// The signed sum of sample differences. + /// The sum of squared sample differences. + public static void GetMoments8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, out int sum, out int sumOfSquares) + => GetMoments8x8(source, sourceStride, prediction, predictionStride, out sum, out sumOfSquares); + + /// + /// Traverses an 8x8 block while its closed operator calculates scalar or eight-sample row costs. + /// + private static int SumAbsoluteDifferences8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride) + where TSample : unmanaged + where TOperator : struct, IResidualOperator + { + int sum = 0; + if (Vector128.IsHardwareAccelerated) + { + // Eight widened AV1 samples exactly fill 128 bits. Wider loads would cross the row boundary; + // byte storage is loaded as eight bytes and ushort storage as eight native-order words. + for (int row = 0; row < SearchBlockDimension; row++) + { + Vector128 sourceRow = LoadSearchRow(source[(row * sourceStride)..]); + Vector128 predictionRow = LoadSearchRow(prediction[(row * predictionStride)..]); + sum += TOperator.SumAbsoluteDifferences(sourceRow, predictionRow); + } + } + else + { + for (int row = 0; row < SearchBlockDimension; row++) + { + ReadOnlySpan sourceRow = source.Slice(row * sourceStride, SearchBlockDimension); + ReadOnlySpan predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension); + for (int column = 0; column < SearchBlockDimension; column++) + { + sum += TOperator.SumAbsoluteDifferences(sourceRow[column], predictionRow[column]); + } + } + } + + return sum; + } + + /// + /// Traverses four adjacent candidates together, retaining one source load per row or scalar sample. + /// + private static void SumFourAbsoluteDifferences8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, Span sums) + where TSample : unmanaged + where TOperator : struct, IResidualOperator + { + if (Vector128.IsHardwareAccelerated) + { + Vector128 totals = Vector128.Zero; + for (int row = 0; row < SearchBlockDimension; row++) + { + ReadOnlySpan predictionRow = prediction[(row * predictionStride)..]; + Vector128 sourceRow = LoadSearchRow(source[(row * sourceStride)..]); + + // The four prediction windows overlap, but each candidate owns one result lane. The operator + // widens the source only once and reuses it for all four independent absolute-difference sums. + totals += TOperator.SumFourAbsoluteDifferences( + sourceRow, + LoadSearchRow(predictionRow), + LoadSearchRow(predictionRow[1..]), + LoadSearchRow(predictionRow[2..]), + LoadSearchRow(predictionRow[3..])); + } + + totals.CopyTo(sums); + } + else + { + int sum0 = 0; + int sum1 = 0; + int sum2 = 0; + int sum3 = 0; + for (int row = 0; row < SearchBlockDimension; row++) + { + ReadOnlySpan sourceRow = source.Slice(row * sourceStride, SearchBlockDimension); + ReadOnlySpan predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension + 3); + for (int column = 0; column < SearchBlockDimension; column++) + { + TSample sample = sourceRow[column]; + sum0 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column]); + sum1 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 1]); + sum2 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 2]); + sum3 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 3]); + } + } + + sums[0] = sum0; + sums[1] = sum1; + sums[2] = sum2; + sums[3] = sum3; + } + } + + /// + /// Accumulates both residual moments in one traversal without materializing a residual buffer. + /// + private static void GetMoments8x8( + ReadOnlySpan source, int sourceStride, ReadOnlySpan prediction, int predictionStride, out int sum, out int sumOfSquares) + where TSample : unmanaged + where TOperator : struct, IResidualOperator + { + sum = 0; + sumOfSquares = 0; + + // Even 64 maximum twelve-bit residual squares fit in a signed int. Preserve the unnormalized + // moments here; the caller applies the frame's precision-dependent rounding before deriving variance. + if (Vector128.IsHardwareAccelerated) + { + for (int row = 0; row < SearchBlockDimension; row++) + { + Vector128 sourceRow = LoadSearchRow(source[(row * sourceStride)..]); + Vector128 predictionRow = LoadSearchRow(prediction[(row * predictionStride)..]); + sumOfSquares += TOperator.SumSquaredDifferences(sourceRow, predictionRow, out int rowSum); + sum += rowSum; + } + } + else + { + for (int row = 0; row < SearchBlockDimension; row++) + { + ReadOnlySpan sourceRow = source.Slice(row * sourceStride, SearchBlockDimension); + ReadOnlySpan predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension); + for (int column = 0; column < SearchBlockDimension; column++) + { + sumOfSquares += TOperator.SumSquaredDifferences(sourceRow[column], predictionRow[column], out int difference); + sum += difference; + } + } + } + } + + /// + /// Loads exactly eight native-order samples; byte rows occupy the lower half of the returned vector. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 LoadSearchRow(ReadOnlySpan source) + where TSample : unmanaged + { + // The closed byte/ushort instantiation removes this storage-width choice. The eight-byte load + // never consumes padding or a following row; the operator widens only its eight populated lanes. + return Vector128.Count == SearchBlockDimension + ? Vector128.Create(source) + : Vector128.Create(Vector64.Create(source), Vector64.Zero); + } + /// /// Subtracts an 8-bit prediction plane from its source plane. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs new file mode 100644 index 0000000000..e2cda37de0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs @@ -0,0 +1,383 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; + +/// +/// Encodes one range-coded AV1 tile payload. +/// +internal readonly struct Av1TileEncoder : IAv1TileWriter +{ + private readonly ReadOnlyMemory tileData; + private readonly Av1PictureControlSet picture; + + /// + /// Initializes a new instance of the struct for eight-bit samples. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The reusable partition and final-block decision workspace. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderSuperblockWorkspace superblockWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reconstruction, + reconstruction, + picture, + coefficientBuffer, + new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace), + blockWorkspace, + effort); + } + + /// + /// Initializes a new instance of the struct for an eight-bit inter frame. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed reference frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The reusable partition and final-block decision workspace. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reference, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderSuperblockWorkspace superblockWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reference, + reconstruction, + picture, + coefficientBuffer, + new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace), + blockWorkspace, + effort); + } + + /// + /// Initializes a new instance of the struct for an eight-bit inter frame. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed reference frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The retained tile, superblock, and entropy cursor graph. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reference, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderTileWorkspace tileWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reference, + reconstruction, + picture, + coefficientBuffer, + tileWorkspace, + blockWorkspace, + effort); + } + + /// + /// Initializes a new instance of the struct for high-bit-depth samples. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The reusable partition and final-block decision workspace. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderSuperblockWorkspace superblockWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reconstruction, + reconstruction, + picture, + coefficientBuffer, + new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace), + blockWorkspace, + effort); + } + + /// + /// Initializes a new instance of the struct for a high-bit-depth inter frame. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed reference frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The reusable partition and final-block decision workspace. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reference, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderSuperblockWorkspace superblockWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reference, + reconstruction, + picture, + coefficientBuffer, + new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace), + blockWorkspace, + effort); + } + + /// + /// Initializes a new instance of the struct for a high-bit-depth inter frame. + /// + /// The symbol encoder that retains tile output through the enclosing frame write. + /// The coded source frame. + /// The reconstructed reference frame. + /// The reconstructed frame updated during encoding. + /// The frame coding and mode-information state. + /// The frame-owned quantized coefficient and transform state. + /// The retained tile, superblock, and entropy cursor graph. + /// The reusable block arithmetic workspace. + /// The mode-search effort in the inclusive range zero through ten. + public Av1TileEncoder( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reference, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderTileWorkspace tileWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + { + this.picture = picture; + this.tileData = Encode( + writer, + source, + reference, + reconstruction, + picture, + coefficientBuffer, + tileWorkspace, + blockWorkspace, + effort); + } + + /// + public ReadOnlySpan GetTileData(int tileNum) + { + int offset = this.picture.TileDataOffsets.Span[tileNum]; + int length = this.picture.TileDataLengths.Span[tileNum]; + return this.tileData.Span.Slice(offset, length); + } + + private static ReadOnlyMemory Encode( + Av1SymbolEncoder writer, + Av1EncoderFrame source, + Av1EncoderFrame reference, + Av1EncoderFrame reconstruction, + Av1PictureControlSet picture, + Av1EncoderCoefficientBuffer coefficientBuffer, + Av1EncoderTileWorkspace tileWorkspace, + Av1EncoderBlockWorkspace blockWorkspace, + int effort) + where TSample : unmanaged + where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator + { + ObuFrameHeader frameHeader = picture.Parent.FrameHeader; + ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader; + Av1TileInfo tile = tileWorkspace.Tile; + Av1Superblock superblock = tileWorkspace.Superblock; + Av1TileWriter.Av1EntropyCodingContext entropyContext = tileWorkspace.EntropyContext; + + int superblockModeInfoSize = sequenceHeader.SuperblockModeInfoSize; + int superblockShift = sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2; + ObuTileGroupHeader tileLayout = frameHeader.TilesInfo; + Span tileDataOffsets = picture.TileDataOffsets.Span; + Span tileDataLengths = picture.TileDataLengths.Span; + if (frameHeader.AllowIntraBlockCopy) + { + // Hash the visible source once before reconstruction begins so candidate discovery never depends + // on coding order and the workspace can be reused as compact bucket links afterward. + picture.IntraBlockCopySearch.Initialize( + source.View.GetPlane(Av1Plane.Y)); + } + + int tileIndex = 0; + int tileDataEnd = 0; + for (int tileRow = 0; tileRow < tileLayout.TileRowCount; tileRow++) + { + tile.SetTileRow(tileLayout, frameHeader.ModeInfoRowCount, tileRow); + for (int tileColumn = 0; tileColumn < tileLayout.TileColumnCount; tileColumn++) + { + tile.SetTileColumn(tileLayout, frameHeader.ModeInfoColumnCount, tileColumn); + if (tileIndex > 0) + { + // Every tile begins from the same frame probabilities, while its bytes follow the preceding + // tile in the retained output allocation. + writer.Reset(tileDataEnd); + } + + Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart); + entropyContext.MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition); + for (int modeInfoRow = tile.ModeInfoRowStart; + modeInfoRow < tile.ModeInfoRowEnd; + modeInfoRow += superblockModeInfoSize) + { + for (int modeInfoColumn = tile.ModeInfoColumnStart; + modeInfoColumn < tile.ModeInfoColumnEnd; + modeInfoColumn += superblockModeInfoSize) + { + int superblockRow = modeInfoRow >> superblockShift; + int superblockColumn = modeInfoColumn >> superblockShift; + superblock.Index = (superblockRow * coefficientBuffer.SuperblockColumnCount) + superblockColumn; + entropyContext.SuperblockOrigin = new Point( + modeInfoColumn << Av1Constants.ModeInfoSizeLog2, + modeInfoRow << Av1Constants.ModeInfoSizeLog2); + + Av1IntraSuperblockEncoder.Prepare( + picture, + superblock, + entropyContext.SuperblockOrigin); + + Av1IntraSuperblockEncoder.ModeDecision blockEncoder = new( + source, + reference, + reconstruction, + picture, + superblock, + coefficientBuffer, + blockWorkspace, + effort); + + Av1TileWriter.WriteSuperblock( + picture, + entropyContext, + writer, + superblock, + coefficientBuffer, + (ushort)tileIndex, + ref blockEncoder); + } + } + + _ = writer.Exit(out int tileDataLength); + tileDataOffsets[tileIndex] = tileDataEnd; + tileDataLengths[tileIndex] = tileDataLength; + tileDataEnd += tileDataLength; + tileIndex++; + } + } + + return writer.GetOutput(tileDataEnd); + } +} + +/// +/// Retains the mutable tile, superblock, and entropy cursor graph reused by serial frame encoding. +/// +internal readonly struct Av1EncoderTileWorkspace +{ + /// + /// Initializes a new instance of the struct. + /// + /// The fixed-geometry frame header defining tile boundaries. + /// The retained superblock decision storage. + public Av1EncoderTileWorkspace( + ObuFrameHeader frameHeader, + Av1EncoderSuperblockWorkspace superblockWorkspace) + { + this.Tile = new Av1TileInfo(0, 0, frameHeader); + this.Superblock = new Av1Superblock + { + Workspace = superblockWorkspace, + TileInfo = this.Tile + }; + + this.EntropyContext = new Av1TileWriter.Av1EntropyCodingContext + { + MacroBlock = new Av1MacroBlockD { Tile = this.Tile }, + MacroBlockModeInfo = default + }; + } + + /// + /// Gets the mutable tile boundaries selected during raster traversal. + /// + public Av1TileInfo Tile { get; } + + /// + /// Gets the mutable superblock cursor connected to the retained decision workspace. + /// + public Av1Superblock Superblock { get; } + + /// + /// Gets the mutable entropy cursor shared by successive superblocks. + /// + public Av1TileWriter.Av1EntropyCodingContext EntropyContext { get; } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs index 2cfb40ecca..a45b853e04 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs @@ -35,7 +35,7 @@ internal static partial class Av1TranslationalInterPredictor /// /// The maximum number of source rows added by an eight-tap vertical filter. /// - private const int MaximumExtraRows = FilterCoefficientCount - 1; + internal const int MaximumExtraRows = FilterCoefficientCount - 1; /// /// The minimum scratch stride that lets a 128-bit byte kernel handle four- and eight-sample blocks. diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs index beb3df8ca7..45452cf3f3 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -11,16 +12,36 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// internal struct Av1EncoderBlockModeInfo { + /// The residual-skip flag in the packed prediction state. private const byte SkipMask = 1 << 0; + + /// The compound-skip flag in the packed prediction state. private const byte SkipModeMask = 1 << 1; + + /// The intra-block-copy flag in the packed prediction state. private const byte IntraBlockCopyMask = 1 << 2; - // Every stored syntax value has an AV1-defined range below 256. Byte fields and one shared flag byte - // keep the frame-wide mode allocation compact without losing any representable encoder state. + /// The three low bits store the segment identifier, followed by the primary reference. + private const int ReferenceFrameShift = 3; + + /// Both segment identifiers and primary references have eight possible values. + private const int SegmentAndReferenceMask = 7; + + /// The vertical filter follows the three prediction flags. + private const int VerticalFilterShift = 3; + + /// The horizontal filter follows the two-bit vertical filter. + private const int HorizontalFilterShift = 5; + + /// Two bits represent each concrete interpolation filter, excluding the frame-level switchable sentinel. + private const int InterpolationFilterMask = 3; + + // Primary references never use the absent-secondary sentinel. Pack their three bits beside the segment, + // and the concrete filters beside the flags, so adding inter syntax does not enlarge the frame-wide grid. private byte blockSize; private byte partitionType; private byte flags; - private byte segmentId; + private byte segmentAndReference; private byte transformSize; private byte mode; private byte uvMode; @@ -75,8 +96,8 @@ internal struct Av1EncoderBlockModeInfo /// public int SegmentId { - readonly get => this.segmentId; - set => this.segmentId = (byte)value; + readonly get => this.segmentAndReference & SegmentAndReferenceMask; + set => this.segmentAndReference = (byte)((this.segmentAndReference & ~SegmentAndReferenceMask) | value); } /// @@ -105,4 +126,31 @@ internal struct Av1EncoderBlockModeInfo readonly get => (Av1ChromaPredictionMode)this.uvMode; set => this.uvMode = (byte)value; } + + /// + /// Gets or sets the primary prediction reference selected for the block. + /// + public Av1ReferenceFrameType ReferenceFrame + { + readonly get => (Av1ReferenceFrameType)(this.segmentAndReference >> ReferenceFrameShift); + set => this.segmentAndReference = (byte)((this.segmentAndReference & SegmentAndReferenceMask) | ((int)value << ReferenceFrameShift)); + } + + /// + /// Gets or sets the concrete vertical interpolation filter selected for the block. + /// + public Av1InterpolationFilter VerticalInterpolationFilter + { + readonly get => (Av1InterpolationFilter)((this.flags >> VerticalFilterShift) & InterpolationFilterMask); + set => this.flags = (byte)((this.flags & ~(InterpolationFilterMask << VerticalFilterShift)) | ((int)value << VerticalFilterShift)); + } + + /// + /// Gets or sets the concrete horizontal interpolation filter selected for the block. + /// + public Av1InterpolationFilter HorizontalInterpolationFilter + { + readonly get => (Av1InterpolationFilter)((this.flags >> HorizontalFilterShift) & InterpolationFilterMask); + set => this.flags = (byte)((this.flags & ~(InterpolationFilterMask << HorizontalFilterShift)) | ((int)value << HorizontalFilterShift)); + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs index f5f9e3f8c7..04354869e0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs @@ -65,6 +65,18 @@ internal struct Av1EncoderBlockStruct set => this.filterIntraMode = (byte)value; } + /// + /// Gets or sets the dynamic-reference-list index selected for an inter block. + /// + /// + /// Filter-intra and inter prediction are mutually exclusive, so both syntax branches share one packed byte. + /// + public int ReferenceMotionVectorIndex + { + readonly get => this.filterIntraMode; + set => this.filterIntraMode = (byte)value; + } + /// /// Gets the encoder prediction-unit state for the block. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs index 3a3e12aa1e..1740cbecf5 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs @@ -1,13 +1,12 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. -using System.Buffers; using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// -/// Owns the reusable luma and chroma palette color-index maps for encoder block decisions. +/// Exposes reusable luma and chroma palette color-index maps over superblock-workspace storage. /// internal sealed class Av1EncoderPaletteMapBuffer : IDisposable { @@ -21,21 +20,19 @@ internal sealed class Av1EncoderPaletteMapBuffer : IDisposable /// public const int StorageLength = 2 * MapLength * MapLength; - private readonly IMemoryOwner owner; private readonly Buffer2D luma; private readonly Buffer2D chroma; /// /// Initializes a new instance of the class. /// - /// The configuration providing the encoder allocator. - public Av1EncoderPaletteMapBuffer(Configuration configuration) + /// The superblock-owned backing storage. + public Av1EncoderPaletteMapBuffer(Memory storage) { - this.owner = configuration.MemoryAllocator.Allocate(StorageLength); int mapArea = MapLength * MapLength; - Memory storage = this.owner.Memory[..StorageLength]; - this.luma = Buffer2D.WrapMemory(storage[..mapArea], MapLength, MapLength); - this.chroma = Buffer2D.WrapMemory(storage[mapArea..], MapLength, MapLength); + Memory mapStorage = storage[..StorageLength]; + this.luma = Buffer2D.WrapMemory(mapStorage[..mapArea], MapLength, MapLength); + this.chroma = Buffer2D.WrapMemory(mapStorage[mapArea..], MapLength, MapLength); } /// @@ -51,12 +48,11 @@ internal sealed class Av1EncoderPaletteMapBuffer : IDisposable new Rectangle(0, 0, width, height)); /// - /// Returns the shared palette-map owner to the configured allocator. + /// Releases the non-owning two-dimensional wrappers. /// public void Dispose() { this.luma.Dispose(); this.chroma.Dispose(); - this.owner.Dispose(); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs index 2b2ed29afa..3618efcbdc 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs @@ -10,12 +10,18 @@ using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; /// -/// Owns mode information, segmentation data, and tile-neighbor contexts for one encoded AV1 picture. +/// Owns reusable mode information, segmentation data, and tile-neighbor contexts for fixed-geometry AV1 pictures. /// internal sealed class Av1EncoderPictureBuffer : IDisposable { private readonly Av1EncoderModeInfoBuffer modeInfo; private readonly IMemoryOwner stateStorage; + + /// + /// The exact packed state region cleared between frames without touching excess pool capacity. + /// + private readonly Memory stateMemory; + private readonly ByteMemoryManager partitionContextMemory; private readonly Av1NeighborArrayUnit[] partitionContexts; private readonly Av1NeighborArrayUnit[] lumaCoefficientContexts; @@ -40,6 +46,42 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable int width, int height, bool disallow4x4AllFrames) + : this( + configuration, + sequenceHeader, + frameHeader, + width, + height, + disallow4x4AllFrames, + frameHeader.AllowScreenContentTools, + frameHeader.AllowIntraBlockCopy || !frameHeader.IsIntra, + frameHeader.AllowIntraBlockCopy) + { + } + + /// + /// Initializes a new instance of the class with the maximum state + /// required by a fixed-geometry sequence. + /// + /// The configuration providing picture-lifetime memory. + /// The sequence header defining superblock and chroma geometry. + /// The initial frame header defining dimensions and tiles. + /// The visible luma width. + /// The visible luma height. + /// Whether each allocated mode-information value represents an 8x8 region. + /// Whether palette neighbor state can be required by any frame. + /// Whether inter or intra-block-copy vectors can be required by any frame. + /// Whether intra-block-copy search state can be required by any frame. + public Av1EncoderPictureBuffer( + Configuration configuration, + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + int width, + int height, + bool disallow4x4AllFrames, + bool allocateScreenContentState, + bool allocateMotionVectorState, + bool allocateIntraBlockCopySearch) { const int ContextAlignmentLog2 = Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2; this.modeInfo = new Av1EncoderModeInfoBuffer( @@ -79,17 +121,17 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable int paletteLeftLength = alignedModeInfoRowCount; int paletteTopLength = this.modeInfo.ModeInfoStride; int paletteContextLength = checked(paletteLeftLength + paletteTopLength); - int paletteStorageOffset = frameHeader.AllowScreenContentTools + int paletteStorageOffset = allocateScreenContentState ? Av1Math.AlignPowerOf2(byteContextStorageEnd, 1) : byteContextStorageEnd; - int paletteStorageLength = frameHeader.AllowScreenContentTools + int paletteStorageLength = allocateScreenContentState ? checked(tileCount * paletteContextLength * Unsafe.SizeOf()) : 0; int paletteStorageEnd = checked(paletteStorageOffset + paletteStorageLength); - int displacementVectorLength = frameHeader.AllowIntraBlockCopy ? this.modeInfo.Allocation.Length : 0; - int displacementVectorStorageOffset = frameHeader.AllowIntraBlockCopy + int displacementVectorLength = allocateMotionVectorState ? this.modeInfo.Allocation.Length : 0; + int displacementVectorStorageOffset = allocateMotionVectorState ? Av1Math.AlignPowerOf2(paletteStorageEnd, 1) : paletteStorageEnd; @@ -97,23 +139,32 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable displacementVectorLength * Unsafe.SizeOf()); int displacementVectorStorageEnd = checked(displacementVectorStorageOffset + displacementVectorStorageLength); - int intraBlockCopySearchStorageOffset = frameHeader.AllowIntraBlockCopy + int intraBlockCopySearchStorageOffset = allocateIntraBlockCopySearch ? Av1Math.AlignPowerOf2(displacementVectorStorageEnd, 2) : displacementVectorStorageEnd; - int intraBlockCopySearchStorageLength = frameHeader.AllowIntraBlockCopy + int intraBlockCopySearchStorageLength = allocateIntraBlockCopySearch ? Av1IntraBlockCopySearchIndex.GetStorageLength(width, height) : 0; - int stateStorageLength = checked(intraBlockCopySearchStorageOffset + intraBlockCopySearchStorageLength); + int intraBlockCopySearchStorageEnd = checked( + intraBlockCopySearchStorageOffset + intraBlockCopySearchStorageLength); + + int tileStateStorageOffset = Av1Math.AlignPowerOf2(intraBlockCopySearchStorageEnd, 2); + int cdefPresetLength = tileCount * Av1Constants.CdefUnitsPerSuperblock; + int tileStateLength = cdefPresetLength + (3 * tileCount); + int tileStateStorageLength = tileStateLength * sizeof(int); + int stateStorageLength = checked(tileStateStorageOffset + tileStateStorageLength); // Segmentation and every tile edge share one clean picture lifetime. The partition region begins at its - // native alignment, while typed views keep the entropy writer independent from the packed byte owner. + // native alignment. CDEF, quantizer, and encoded-tile bounds occupy one aligned trailing integer region + // instead of allocating separate managed arrays for every picture. this.stateStorage = configuration.MemoryAllocator.Allocate( stateStorageLength, AllocationOptions.Clean); - Memory stateStorage = this.stateStorage.Memory[..stateStorageLength]; + this.stateMemory = this.stateStorage.Memory[..stateStorageLength]; + Memory stateStorage = this.stateMemory; this.partitionContextMemory = new ByteMemoryManager( stateStorage.Slice(partitionStorageOffset, partitionStorageLength)); @@ -125,7 +176,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable this.redCoefficientContexts = new Av1NeighborArrayUnit[tileCount]; this.transformContexts = new Av1NeighborArrayUnit[tileCount]; Memory paletteStorage = Memory.Empty; - if (frameHeader.AllowScreenContentTools) + if (allocateScreenContentState) { // Palette entries contain 16-bit colors, so their packed typed region begins at an even byte offset. ByteMemoryManager paletteMemory = new( @@ -140,10 +191,10 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable } Memory displacementVectors = Memory.Empty; - if (frameHeader.AllowIntraBlockCopy) + if (allocateMotionVectorState) { - // Each component lies strictly inside plus or minus 16384, so two signed 16-bit fields preserve the - // complete syntax domain without expanding every frame's compact mode-information allocation. + // Each component lies strictly inside plus or minus 16384. Two signed 16-bit fields preserve both + // inter and intra-block-copy vectors without expanding every compact mode-information entry. ByteMemoryManager displacementVectorMemory = new( stateStorage.Slice(displacementVectorStorageOffset, displacementVectorStorageLength)); @@ -151,7 +202,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable } Av1IntraBlockCopySearchIndex intraBlockCopySearch = default; - if (frameHeader.AllowIntraBlockCopy) + if (allocateIntraBlockCopySearch) { // The search index casts its packed workspace to 32-bit links, so its non-owning region begins at // a four-byte boundary inside the existing picture-state rent. @@ -161,8 +212,15 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable height); } - int[][] cdefPreset = new int[tileCount][]; - int[] previousQIndex = new int[tileCount]; + ByteMemoryManager tileStateMemory = new( + stateStorage.Slice(tileStateStorageOffset, tileStateStorageLength)); + + Memory tileState = tileStateMemory.Memory; + Memory cdefPreset = tileState[..cdefPresetLength]; + Memory previousQIndex = tileState.Slice(cdefPresetLength, tileCount); + Memory tileDataOffsets = tileState.Slice(cdefPresetLength + tileCount, tileCount); + Memory tileDataLengths = tileState.Slice(cdefPresetLength + (2 * tileCount), tileCount); + cdefPreset.Span.Fill(-1); for (int tileIndex = 0; tileIndex < tileCount; tileIndex++) { this.partitionContexts[tileIndex] = new Av1NeighborArrayUnit( @@ -214,7 +272,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable this.transformContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize); this.transformContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize); - if (frameHeader.AllowScreenContentTools) + if (allocateScreenContentState) { this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit( paletteStorage.Slice(tileIndex * paletteContextLength, paletteContextLength), @@ -225,8 +283,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable }; } - cdefPreset[tileIndex] = [-1, -1, -1, -1]; - previousQIndex[tileIndex] = frameHeader.QuantizationParameters.BaseQIndex; + previousQIndex.Span[tileIndex] = frameHeader.QuantizationParameters.BaseQIndex; } this.Picture = new Av1PictureControlSet @@ -258,7 +315,9 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable IntraBlockCopySearch = intraBlockCopySearch, ModeInfoStride = this.modeInfo.ModeInfoStride, Disallow4x4AllFrames = this.modeInfo.Disallow4x4AllFrames, - CdefPreset = cdefPreset + CdefPreset = cdefPreset, + TileDataOffsets = tileDataOffsets, + TileDataLengths = tileDataLengths }; } @@ -267,6 +326,31 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable /// public Av1PictureControlSet Picture { get; } + /// + /// Restores clean per-frame state while retaining every fixed-geometry allocation. + /// + /// The frame header consumed by the next encoding pass. + public void Reset(ObuFrameHeader frameHeader) + { + this.modeInfo.Grid.Span.Clear(); + this.modeInfo.Allocation.Span.Clear(); + this.stateMemory.Span.Clear(); + + // Transform contexts begin at the largest transform size until an encoded neighbor publishes its + // selected size. This sentinel must be restored after the packed state owner is cleared. + foreach (Av1NeighborArrayUnit context in this.transformContexts) + { + context.Left.Fill((byte)Av1Constants.MaxTransformSize); + context.Top.Fill((byte)Av1Constants.MaxTransformSize); + } + + this.Picture.CdefPreset.Span.Fill(-1); + this.Picture.Parent.PreviousQIndex.Span.Fill(frameHeader.QuantizationParameters.BaseQIndex); + this.Picture.Parent.FrameHeader = frameHeader; + this.Picture.Parent.Common.FrameSize = frameHeader.FrameSize; + this.Picture.Parent.Common.TilesInfo = frameHeader.TilesInfo; + } + /// /// Returns every picture-lifetime allocation to the configured allocator. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs index f92caf43b0..34238e3d5d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -22,14 +23,26 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable public const int MaximumPartitionCount = 1 + 4 + 16 + 64 + 256; /// - /// The complete workspace length in packed final-block storage elements. + /// The decision-region length in packed final-block storage elements. /// - public const int StorageLength = MaximumFinalBlockCount + ((MaximumPartitionCount + Av1EncoderBlockStruct.StorageSize - 1) / Av1EncoderBlockStruct.StorageSize); + public const int DecisionStorageLength = MaximumFinalBlockCount + ((MaximumPartitionCount + Av1EncoderBlockStruct.StorageSize - 1) / Av1EncoderBlockStruct.StorageSize); - private readonly Configuration configuration; - private readonly IMemoryOwner owner; - private Av1EncoderPaletteMapBuffer? paletteMaps; + /// + /// The byte length of the aligned final-block and partition decision region. + /// + public const int DecisionStorageByteLength = DecisionStorageLength * Av1EncoderBlockStruct.StorageSize; + + /// + /// The complete byte length of the decision and palette-map regions. + /// + public const int StorageByteLength = DecisionStorageByteLength + Av1EncoderPaletteMapBuffer.StorageLength; + + private const int PartitionStorageOffset = MaximumFinalBlockCount * Av1EncoderBlockStruct.StorageSize; + + private readonly IMemoryOwner owner; + private readonly Av1EncoderPaletteMapBuffer paletteMaps; private Av1EncoderPaletteInfo paletteInfo; + private Av1ReferenceMotionVectors referenceMotionVectors; /// /// Initializes a new instance of the class. @@ -37,21 +50,27 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable /// The configuration providing the encoder allocator. public Av1EncoderSuperblockWorkspace(Configuration configuration) { - this.configuration = configuration; - this.owner = configuration.MemoryAllocator.Allocate(StorageLength); + this.owner = configuration.MemoryAllocator.Allocate(StorageByteLength); + Memory storage = this.owner.Memory[..StorageByteLength]; + + // Decisions and palette maps have the same serial superblock lifetime. Keeping both regions in one + // owner preserves their distinct layouts while removing a separate palette allocation and cleanup path. + this.paletteMaps = new Av1EncoderPaletteMapBuffer( + storage.Slice(DecisionStorageByteLength, Av1EncoderPaletteMapBuffer.StorageLength)); + this.Reset(); } /// /// Gets the maximum-size final-block decision span in partition traversal order. /// - public Span FinalBlocks => this.owner.Memory.Span[..MaximumFinalBlockCount]; + public Span FinalBlocks + => MemoryMarshal.Cast(this.owner.Memory.Span[..DecisionStorageByteLength])[..MaximumFinalBlockCount]; /// /// Gets the maximum-size partition-type span in partition-tree preorder. /// - public Span PartitionTypes - => MemoryMarshal.AsBytes(this.owner.Memory.Span[MaximumFinalBlockCount..])[..MaximumPartitionCount]; + public Span PartitionTypes => this.owner.Memory.Span.Slice(PartitionStorageOffset, MaximumPartitionCount); /// /// Gets the palette sizes and colors selected for the block currently being written. @@ -59,20 +78,15 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable public ref Av1EncoderPaletteInfo PaletteInfo => ref this.paletteInfo; /// - /// Gets the reusable palette maps, allocating their shared owner only after a block enters palette search. + /// Gets the reusable reference-vector stack used while writing inter syntax. /// - /// The reusable luma and chroma palette maps. - public Av1EncoderPaletteMapBuffer GetPaletteMaps() - { - Av1EncoderPaletteMapBuffer? maps = this.paletteMaps; - if (maps is null) - { - maps = new Av1EncoderPaletteMapBuffer(this.configuration); - this.paletteMaps = maps; - } + public ref Av1ReferenceMotionVectors ReferenceMotionVectors => ref this.referenceMotionVectors; - return maps; - } + /// + /// Gets the reusable palette maps within the superblock-workspace owner. + /// + /// The reusable luma and chroma palette maps. + public Av1EncoderPaletteMapBuffer GetPaletteMaps() => this.paletteMaps; /// /// Clears all decisions before the workspace is reused for another superblock. @@ -86,7 +100,7 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable }; this.FinalBlocks.Fill(initialBlock); - MemoryMarshal.AsBytes(this.owner.Memory.Span[MaximumFinalBlockCount..]).Clear(); + this.PartitionTypes.Clear(); this.paletteInfo = default; } @@ -95,7 +109,7 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable /// public void Dispose() { - this.paletteMaps?.Dispose(); + this.paletteMaps.Dispose(); this.owner.Dispose(); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs index ee9c4cd5ab..aa96f70b98 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs @@ -86,9 +86,20 @@ internal class Av1PictureControlSet public bool Disallow4x4AllFrames { get; set; } /// - /// Gets or sets the constrained directional enhancement filter presets for each filter block. + /// Gets or sets the constrained directional enhancement filter presets for each tile. + /// Each tile occupies consecutive entries. /// - public required int[][] CdefPreset { get; set; } + public required Memory CdefPreset { get; set; } + + /// + /// Gets or sets the starting byte of each tile in the shared encoded output buffer. + /// + public required Memory TileDataOffsets { get; set; } + + /// + /// Gets or sets the encoded byte length of each tile. + /// + public required Memory TileDataLengths { get; set; } /// /// Gets the mode-information entry mapped to a frame position. diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs index dcc774f1e3..108102fb2d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs @@ -23,7 +23,7 @@ internal class Av1PictureParentControlSet /// /// Gets or sets the preceding quantizer index for each tile context. /// - public required int[] PreviousQIndex { get; set; } + public required Memory PreviousQIndex { get; set; } /// /// Gets or sets the encoder palette-search level. diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index 8cece2ddb7..124c66c527 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -134,7 +134,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable /// /// Reusable fixed-capacity storage for one block's weighted reference-motion-vector candidates. /// - private readonly Av1ReferenceMotionVectors referenceMotionVectors = new(); + private Av1ReferenceMotionVectors referenceMotionVectors; /// /// Reusable fixed-capacity state for motion-mode eligibility and local warped-motion projection. @@ -250,6 +250,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable Av1ReferenceFrameStore? referenceFrames, PaletteColorIndexMaps? sharedPaletteColorIndexMaps) { + this.referenceMotionVectors = default; this.FrameHeader = frameHeader; this.configuration = configuration; this.SequenceHeader = sequenceHeader; @@ -2044,7 +2045,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable Av1ReferenceFrameType secondaryReferenceFrame = modeInfo.ReferenceFrames[1]; bool isCompound = secondaryReferenceFrame > Av1ReferenceFrameType.Intra; - Av1ReferenceMotionVectors referenceMotionVectors = this.referenceMotionVectors; + ref Av1ReferenceMotionVectors referenceMotionVectors = ref this.referenceMotionVectors; referenceMotionVectors.Build( ref partitionInfo, tileInfo, @@ -2110,9 +2111,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable } } - Av1MotionVectorPrecision precision = this.FrameHeader.ForceIntegerMotionVector - ? Av1MotionVectorPrecision.Integer - : this.FrameHeader.AllowHighPrecisionMotionVector ? Av1MotionVectorPrecision.EighthSample : Av1MotionVectorPrecision.QuarterSample; + Av1MotionVectorPrecision precision = this.FrameHeader.MotionVectorPrecision; Span motionVectors = modeInfo.MotionVectors; if (!isCompound) diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs index 53b591ea4b..480f6d80d6 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs @@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; @@ -832,18 +833,35 @@ internal partial class Av1TileWriter bool skipWritingCoefficients = macroBlockModeInfo.Block.Skip; - // This encoder path currently writes intra frames only, so every block follows the key-frame mode syntax. + // Segmentation, skip, filter, and quantizer syntax precede the prediction-domain branch in both + // intra and inter frames. Keeping this prefix shared preserves the decoder's symbol order. { if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) { - WriteSegmentId(pcs, writer, blockSize, blockOrigin, macroBlock, ref blk_ptr, skipWritingCoefficients); + WriteSegmentId( + pcs, + writer, + blockSize, + blockOrigin, + macroBlock, + ref blk_ptr, + skipWritingCoefficients, + beforeSkip: true); } EncodeSkipCoefficients(writer, macroBlock, skipWritingCoefficients); if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && !pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) { - WriteSegmentId(pcs, writer, blockSize, blockOrigin, macroBlock, ref blk_ptr, skipWritingCoefficients); + WriteSegmentId( + pcs, + writer, + blockSize, + blockOrigin, + macroBlock, + ref blk_ptr, + skipWritingCoefficients, + beforeSkip: false); } WriteCdef( @@ -862,17 +880,122 @@ internal partial class Av1TileWriter if ((blockSize != scs.SequenceHeader.SuperblockSize || !skipWritingCoefficients) && super_block_upper_left) { Guard.MustBeGreaterThan(current_q_index, 0, nameof(current_q_index)); - int reduced_delta_qindex = (current_q_index - pcs.Parent.PreviousQIndex[tile_idx]) / + int reduced_delta_qindex = (current_q_index - pcs.Parent.PreviousQIndex.Span[tile_idx]) / frm_hdr.DeltaQParameters.Resolution; writer.WriteDeltaQuantizerIndex(reduced_delta_qindex); - pcs.Parent.PreviousQIndex[tile_idx] = current_q_index; + pcs.Parent.PreviousQIndex.Span[tile_idx] = current_q_index; } } - Av1PredictionMode intra_luma_mode = macroBlockModeInfo.Block.Mode; + bool isInterBlock = macroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra; + bool isGlobalMotionForced = false; + bool isReferenceForced = false; + if (!frm_hdr.IsIntra) + { + ObuSegmentationParameters segmentation = frm_hdr.SegmentationParameters; + int segmentId = macroBlockModeInfo.Block.SegmentId; + isReferenceForced = segmentation.IsFeatureActive( + segmentId, + ObuSegmentationLevelFeature.ReferenceFrame); + + isGlobalMotionForced = segmentation.IsFeatureActive( + segmentId, + ObuSegmentationLevelFeature.GlobalMotionVector); + + if (!isReferenceForced && !isGlobalMotionForced) + { + int intraInterContext = GetIntraInterContext(macroBlock); + writer.WriteIsInter(isInterBlock, intraInterContext); + } + } + + Av1PredictionMode lumaMode = macroBlockModeInfo.Block.Mode; Av1ChromaPredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode; - if (IsIntraBlockCopyAllowed(pcs.Parent.FrameHeader/*, pcs.Parent.SliceType*/)) + if (isInterBlock) + { + if (!isReferenceForced && !isGlobalMotionForced) + { + Span referenceCounts = stackalloc byte[Av1Constants.ReferenceFrameCount]; + CollectNeighborReferenceCounts(macroBlock, referenceCounts); + writer.WriteSingleReference( + macroBlockModeInfo.Block.ReferenceFrame, + referenceCounts); + } + + if (!isGlobalMotionForced) + { + ref Av1ReferenceMotionVectors referenceMotionVectors = ref tb_ptr.Workspace.ReferenceMotionVectors; + referenceMotionVectors.Build( + pcs, + macroBlock, + modeInfoPosition, + blockSize, + macroBlockModeInfo.Block.PartitionType, + scs.SequenceHeader, + frm_hdr, + macroBlockModeInfo.Block.ReferenceFrame); + + writer.WriteInterMode(lumaMode, referenceMotionVectors.ModeContext); + int referenceMotionVectorIndex = blk_ptr.ReferenceMotionVectorIndex; + if (lumaMode == Av1PredictionMode.NearMotionVector) + { + // NEARMV reserves stack entry zero for NEARESTMV, so its DRL decisions advance from + // near entry zero to one and then from one to two. + for (int index = 1; index < 3 && referenceMotionVectors.Count > index + 1; index++) + { + bool advance = referenceMotionVectorIndex >= index; + int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index); + writer.WriteDynamicReferenceList(advance, context); + if (!advance) + { + break; + } + } + } + else if (lumaMode == Av1PredictionMode.NewMotionVector) + { + // NEWMV begins at stack entry zero and can advance through entries one and two. + for (int index = 0; index < 2 && referenceMotionVectors.Count > index + 1; index++) + { + bool advance = referenceMotionVectorIndex > index; + int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index); + writer.WriteDynamicReferenceList(advance, context); + if (!advance) + { + break; + } + } + + Av1MotionVector vector = pcs.GetDisplacementVector(modeInfoPosition); + writer.WriteMotionVector( + vector, + referenceMotionVectors.GetNewReference(referenceMotionVectorIndex), + frm_hdr.MotionVectorPrecision); + } + } + + if (UsesSwitchableInterpolation(frm_hdr, macroBlockModeInfo.Block)) + { + // The vertical symbol is first and supplies both axes unless the sequence enables dual filters. + int verticalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext( + macroBlockModeInfo.Block, + macroBlock, + direction: 0); + + writer.WriteSwitchableInterpolationFilter(macroBlockModeInfo.Block.VerticalInterpolationFilter, verticalContext); + if (scs.SequenceHeader.EnableDualFilter) + { + int horizontalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext( + macroBlockModeInfo.Block, + macroBlock, + direction: 1); + + writer.WriteSwitchableInterpolationFilter(macroBlockModeInfo.Block.HorizontalInterpolationFilter, horizontalContext); + } + } + } + else if (IsIntraBlockCopyAllowed(pcs.Parent.FrameHeader/*, pcs.Parent.SliceType*/)) { WriteIntraBlockCopyInfo( pcs, @@ -882,12 +1005,19 @@ internal partial class Av1TileWriter macroBlockModeInfo); } - if (!macroBlockModeInfo.Block.UseIntraBlockCopy) + if (!isInterBlock && !macroBlockModeInfo.Block.UseIntraBlockCopy) { - EncodeIntraLumaMode(writer, macroBlockModeInfo, macroBlock, ref blk_ptr, blockSize, intra_luma_mode); + EncodeIntraLumaMode( + writer, + frm_hdr, + macroBlockModeInfo, + macroBlock, + ref blk_ptr, + blockSize, + lumaMode); } - if (!macroBlockModeInfo.Block.UseIntraBlockCopy) + if (!isInterBlock && !macroBlockModeInfo.Block.UseIntraBlockCopy) { if (blk_ptr.HasChroma) { @@ -898,12 +1028,13 @@ internal partial class Av1TileWriter macroBlockModeInfo, ref blk_ptr, blockSize, - intra_luma_mode, + lumaMode, intra_chroma_mode); } } - bool paletteAllowed = !macroBlockModeInfo.Block.UseIntraBlockCopy && + bool paletteAllowed = !isInterBlock && + !macroBlockModeInfo.Block.UseIntraBlockCopy && IsPaletteAllowed(frm_hdr.AllowScreenContentTools, blockSize); if (paletteAllowed) @@ -921,12 +1052,13 @@ internal partial class Av1TileWriter blk_ptr.HasChroma); } - if (!macroBlockModeInfo.Block.UseIntraBlockCopy && + if (!isInterBlock && + !macroBlockModeInfo.Block.UseIntraBlockCopy && IsFilterIntraAllowed( scs.SequenceHeader.EnableFilterIntra, blockSize, paletteInfo.PaletteSizes[0], - intra_luma_mode)) + lumaMode)) { writer.WriteFilterIntraMode(blk_ptr.FilterIntraMode, blockSize); } @@ -983,7 +1115,7 @@ internal partial class Av1TileWriter writer, ref blk_ptr, blockOrigin, - intra_luma_mode, + lumaMode, blockSize, coefficientBuffer, tb_ptr.Index, @@ -1035,7 +1167,8 @@ internal partial class Av1TileWriter ref Av1MacroBlockModeInfo aboveModeInfo = ref macroBlock.GetRelativeModeInfo(-macroBlock.ModeInfoStride); - if (aboveModeInfo.Block.UseIntraBlockCopy) + if (aboveModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra || + aboveModeInfo.Block.UseIntraBlockCopy) { above = aboveModeInfo.Block.BlockSize.GetWidth() >= maximumTransformSize.GetWidth() ? 1 : 0; } @@ -1044,7 +1177,8 @@ internal partial class Av1TileWriter if (macroBlock.IsLeftAvailable) { ref Av1MacroBlockModeInfo leftModeInfo = ref macroBlock.GetRelativeModeInfo(-1); - if (leftModeInfo.Block.UseIntraBlockCopy) + if (leftModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra || + leftModeInfo.Block.UseIntraBlockCopy) { left = leftModeInfo.Block.BlockSize.GetHeight() >= maximumTransformSize.GetHeight() ? 1 : 0; } @@ -1076,7 +1210,8 @@ internal partial class Av1TileWriter { ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; bool isLossless = frameHeader.LosslessArray[macroBlockModeInfo.Block.SegmentId]; - bool isInter = macroBlockModeInfo.Block.UseIntraBlockCopy; + bool isInter = macroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra || + macroBlockModeInfo.Block.UseIntraBlockCopy; bool writesUniformTransformSize = !isLossless && frameHeader.TransformMode == Av1TransformMode.Select && !isInter && @@ -1110,7 +1245,7 @@ internal partial class Av1TileWriter blockSize, blockSize.GetMaximumTransformSize()); - // Intra-block copy currently retains the maximum transform, so its variable-transform tree has one unsplit root. + // Current inter decisions retain the maximum transform, so their variable-transform tree has one unsplit root. writer.WriteTransformPartition(false, context); } @@ -1247,23 +1382,34 @@ internal partial class Av1TileWriter } /// - /// Gets the key-frame luma mode rate against the current neighboring modes and tile probabilities. + /// Gets the luma mode rate from the frame-appropriate distribution. /// /// The live tile symbol encoder. /// The current block's mapped neighbor state. /// The selected block size. /// The candidate luma mode. /// The signed directional-angle adjustment. + /// Whether the frame uses key-frame neighbor-conditioned mode syntax. /// The luma mode and directional-angle rate in 1/512-bit units. public static int GetLumaModeCost( Av1SymbolEncoder writer, Av1MacroBlockD macroBlock, Av1BlockSize blockSize, Av1PredictionMode mode, - int angleDelta) + int angleDelta, + bool isIntraFrame) { - GetYModeContext(macroBlock, out byte topContext, out byte leftContext); - int cost = writer.GetLumaModeCost(mode, topContext, leftContext); + int cost; + if (isIntraFrame) + { + GetYModeContext(macroBlock, out byte topContext, out byte leftContext); + cost = writer.GetLumaModeCost(mode, topContext, leftContext); + } + else + { + cost = writer.GetInterFrameLumaModeCost(mode, blockSize); + } + if (blockSize >= Av1BlockSize.Block8x8 && mode.IsDirectional()) { cost += writer.GetAngleDeltaCost(angleDelta + Av1Constants.MaxAngleDelta, mode); @@ -1273,9 +1419,10 @@ internal partial class Av1TileWriter } /// - /// Writes the key-frame luma prediction mode and any directional angle adjustment. + /// Writes the frame-appropriate luma prediction mode and any directional angle adjustment. /// /// The tile symbol encoder. + /// The frame header that selects the luma-mode probability model. /// The selected block modes. /// The reusable macroblock edge and neighbor state. /// The encoder prediction-unit state. @@ -1283,14 +1430,22 @@ internal partial class Av1TileWriter /// The selected luma prediction mode. private static void EncodeIntraLumaMode( Av1SymbolEncoder writer, + ObuFrameHeader frameHeader, Av1MacroBlockModeInfo macroBlockModeInfo, Av1MacroBlockD macroBlock, ref Av1EncoderBlockStruct blk_ptr, Av1BlockSize blockSize, Av1PredictionMode lumaMode) { - GetYModeContext(macroBlock, out byte topContext, out byte leftContext); - writer.WriteLumaMode(lumaMode, topContext, leftContext); + if (frameHeader.IsIntra) + { + GetYModeContext(macroBlock, out byte topContext, out byte leftContext); + writer.WriteLumaMode(lumaMode, topContext, leftContext); + } + else + { + writer.WriteInterFrameLumaMode(lumaMode, blockSize); + } if (blockSize >= Av1BlockSize.Block8x8 && macroBlockModeInfo.Block.Mode.IsDirectional()) { @@ -1298,6 +1453,91 @@ internal partial class Av1TileWriter } } + /// + /// Gets the prediction-domain context from the immediately above and left encoder blocks. + /// + /// The current block's mapped neighbor state. + /// The context in the inclusive range zero through three. + public static int GetIntraInterContext(Av1MacroBlockD macroBlock) + { + bool hasAbove = macroBlock.IsUpAvailable; + bool hasLeft = macroBlock.IsLeftAvailable; + if (hasAbove && hasLeft) + { + bool aboveIsIntra = macroBlock + .GetRelativeModeInfo(-macroBlock.ModeInfoStride) + .Block.ReferenceFrame <= Av1ReferenceFrameType.Intra; + + bool leftIsIntra = macroBlock + .GetRelativeModeInfo(-1) + .Block.ReferenceFrame <= Av1ReferenceFrameType.Intra; + + if (aboveIsIntra && leftIsIntra) + { + return 3; + } + + return aboveIsIntra || leftIsIntra ? 1 : 0; + } + + if (hasAbove) + { + return macroBlock + .GetRelativeModeInfo(-macroBlock.ModeInfoStride) + .Block.ReferenceFrame <= Av1ReferenceFrameType.Intra + ? 2 + : 0; + } + + if (hasLeft) + { + return macroBlock + .GetRelativeModeInfo(-1) + .Block.ReferenceFrame <= Av1ReferenceFrameType.Intra + ? 2 + : 0; + } + + return 0; + } + + /// + /// Counts the single-reference labels used by the immediately above and left encoded blocks. + /// + /// The current block's mapped neighbor state. + /// The eight-entry destination indexed by reference-frame label. + public static void CollectNeighborReferenceCounts( + Av1MacroBlockD macroBlock, + Span referenceCounts) + { + // The caller supplies short-lived fixed storage for one block. Clearing it here keeps unavailable + // neighbors from retaining votes collected for a preceding block. + referenceCounts.Clear(); + if (macroBlock.IsUpAvailable) + { + Av1ReferenceFrameType referenceFrame = macroBlock + .GetRelativeModeInfo(-macroBlock.ModeInfoStride) + .Block.ReferenceFrame; + + if (referenceFrame > Av1ReferenceFrameType.Intra) + { + referenceCounts[(int)referenceFrame]++; + } + } + + if (macroBlock.IsLeftAvailable) + { + Av1ReferenceFrameType referenceFrame = macroBlock + .GetRelativeModeInfo(-1) + .Block.ReferenceFrame; + + if (referenceFrame > Av1ReferenceFrameType.Intra) + { + referenceCounts[(int)referenceFrame]++; + } + } + } + /// /// Writes luma and chroma palette-mode syntax for a block. /// @@ -1498,6 +1738,26 @@ internal partial class Av1TileWriter } } + /// + /// Determines whether a single-reference encoder block carries switchable interpolation symbols. + /// + /// The current frame header. + /// The selected block syntax. + /// Whether the block writes a vertical filter and, when enabled, a horizontal filter. + public static bool UsesSwitchableInterpolation(ObuFrameHeader frameHeader, Av1EncoderBlockModeInfo modeInfo) + { + if (frameHeader.InterpolationFilter != Av1InterpolationFilter.Switchable || modeInfo.SkipMode) + { + return false; + } + + // Global identity and affine models infer the regular filter on blocks at least 8x8. Translation still + // carries filter symbols, including integer translations. Residual skip does not suppress these symbols. + return modeInfo.Mode != Av1PredictionMode.GlobalMotionVector || + Math.Min(modeInfo.BlockSize.GetWidth(), modeInfo.BlockSize.GetHeight()) < Av1BlockSize.Block8x8.GetWidth() || + frameHeader.GetGlobalMotionParameters()[(int)modeInfo.ReferenceFrame - 1].Type == Av1GlobalMotionType.Translation; + } + /// /// Determines whether the current frame permits intra block copy. /// @@ -1603,14 +1863,15 @@ internal partial class Av1TileWriter return; } + Span cdefPreset = pcs.CdefPreset.Span.Slice( + tileIndex * Av1Constants.CdefUnitsPerSuperblock, + Av1Constants.CdefUnitsPerSuperblock); + // Each superblock begins with all contained 64x64 filter units unassigned. if ((modeInfoPosition.Y & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0 && (modeInfoPosition.X & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0) { - pcs.CdefPreset[tileIndex][0] = -1; - pcs.CdefPreset[tileIndex][1] = -1; - pcs.CdefPreset[tileIndex][2] = -1; - pcs.CdefPreset[tileIndex][3] = -1; + cdefPreset.Fill(-1); } // The strength is coded once, at the first non-skipped block in each 64x64 CDEF filter unit. @@ -1619,7 +1880,7 @@ internal partial class Av1TileWriter int unitRow = (modeInfoPosition.Y & cdefSize) != 0 ? 1 : 0; int index = scs.SequenceHeader.Use128x128Superblock ? unitColumn + (2 * unitRow) : 0; - if (pcs.CdefPreset[tileIndex][index] == -1 && !skip) + if (cdefPreset[index] == -1 && !skip) { int firstBlockMask = ~(cdefSize - 1); Point firstBlockPosition = new( @@ -1630,7 +1891,7 @@ internal partial class Av1TileWriter // CDEF strength belongs to the first mode-info block in the 64x64 filter unit even when skipped // blocks delay transmission until a later coding block. writer.WriteCdefStrength(firstBlock.CdefStrength, frameHeader.CdefParameters.BitCount); - pcs.CdefPreset[tileIndex][index] = firstBlock.CdefStrength; + cdefPreset[index] = firstBlock.CdefStrength; } } @@ -2036,7 +2297,9 @@ internal partial class Av1TileWriter int transformBlockHeight = transformSize.Get4x4HighCount(); int transformWidth = transformSize.GetWidth(); int transformHeight = transformSize.GetHeight(); - bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy; + bool usesInterTransformSet = + entropyCodingContext.MacroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra || + entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy; Av1ComponentType componentType = isLuma ? Av1ComponentType.Luminance : Av1ComponentType.Chroma; @@ -2240,6 +2503,7 @@ internal partial class Av1TileWriter /// The reusable macroblock edge and neighbor state. /// The encoder block state. /// A value indicating whether residual coefficients are omitted. + /// Whether the segment identifier is written before the skip flag. private static void WriteSegmentId( Av1PictureControlSet pcs, Av1SymbolEncoder writer, @@ -2247,7 +2511,8 @@ internal partial class Av1TileWriter Point blockOrigin, Av1MacroBlockD macroBlock, ref Av1EncoderBlockStruct block, - bool skip) + bool skip, + bool beforeSkip) { ObuSegmentationParameters segmentation_params = pcs.Parent.FrameHeader.SegmentationParameters; if (!segmentation_params.Enabled) @@ -2256,9 +2521,9 @@ internal partial class Av1TileWriter } int spatial_pred = GetSpatialSegmentationPrediction(pcs, macroBlock, blockOrigin, out int cdf_num); - if (skip) + if (!beforeSkip && skip) { - // With segment-id-before-skip syntax, a skipped block inherits the spatial predictor without coding a residual ID. + // Post-skip segment syntax can infer the spatial predictor once the decoder already knows the block is skipped. pcs.UpdateSegmentation(blockSize, blockOrigin, spatial_pred); block.SegmentId = spatial_pred; return; diff --git a/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs index dcb54e1ad1..f9df35ad95 100644 --- a/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs @@ -15,6 +15,28 @@ namespace SixLabors.ImageSharp.Formats.Heif.Components.Alpha; /// internal static class HeifPlanarAlphaEncoder { + /// + /// The number of float elements sharing storage with one packed value. + /// + private const int Rgba64FloatElementCount = 2; + + /// + /// The additional float element receiving one extracted alpha value. + /// + private const int AlphaFloatElementCount = 1; + + /// + /// The complete reusable row-storage length per source pixel. + /// + private const int RowFloatElementCount = Rgba64FloatElementCount + AlphaFloatElementCount; + + /// + /// Gets the reusable row-storage length required for the specified source width. + /// + /// The source-row width. + /// The required number of float elements. + public static int GetRowStorageLength(int width) => width * RowFloatElementCount; + /// /// Converts one packed image frame into a full-range native alpha plane. /// @@ -34,19 +56,85 @@ internal static class HeifPlanarAlphaEncoder where TSample : unmanaged where TStorer : struct, IHeifSampleConverter { - int width = image.Width; + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + Convert(configuration, image, sourceRectangle, buffer); + } + + /// + /// Converts one packed image region into a full-range native alpha plane. + /// + /// The packed source pixel type. + /// The codec adapter exposing the destination plane. + /// The native unsigned sample storage type. + /// The SIMD narrowing and storage operations for the sample type. + /// The configuration used for row allocation and pixel conversion. + /// The packed source image frame. + /// The source region mapped to the complete destination plane. + /// The monochrome destination buffer. + public static void Convert( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + TBuffer buffer) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter + { + int width = sourceRectangle.Width; // Rgba64 preserves the source pixel's normalized alpha precision before quantization to the requested AV1 // depth. Both row views share one owner because their lifetimes never escape this conversion operation. - using IMemoryOwner rowOwner = configuration.MemoryAllocator.Allocate(width * 3); - Span rowStorage = rowOwner.GetSpan(); - Span packed = MemoryMarshal.Cast(rowStorage[..(width * 2)]); - Span alpha = rowStorage.Slice(width * 2, width); + using IMemoryOwner rowOwner = configuration.MemoryAllocator.Allocate( + GetRowStorageLength(width)); + + Convert( + configuration, + image, + sourceRectangle, + buffer, + rowOwner.GetSpan()); + } + + /// + /// Converts one packed image region using caller-owned reusable row storage. + /// + /// The packed source pixel type. + /// The codec adapter exposing the destination plane. + /// The native unsigned sample storage type. + /// The SIMD narrowing and storage operations for the sample type. + /// The configuration used for pixel conversion. + /// The packed source image frame. + /// The source region mapped to the complete destination plane. + /// The monochrome destination buffer. + /// Storage for one packed high-precision row and its extracted alpha values. + public static void Convert( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + TBuffer buffer, + Span rowStorage) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter + { + int width = sourceRectangle.Width; + Span packed = MemoryMarshal.Cast( + rowStorage[..(width * Rgba64FloatElementCount)]); + + Span alpha = rowStorage.Slice( + width * Rgba64FloatElementCount, + width * AlphaFloatElementCount); + float maximum = (1 << buffer.LumaBitDepth) - 1; float scale = maximum / ushort.MaxValue; - for (int y = 0; y < image.Height; y++) + for (int y = 0; y < sourceRectangle.Height; y++) { - ReadOnlySpan source = image.PixelBuffer.DangerousGetRowSpan(y); + ReadOnlySpan source = image.PixelBuffer + .DangerousGetRowSpan(sourceRectangle.Y + y) + .Slice(sourceRectangle.X, sourceRectangle.Width); + PixelOperations.Instance.ToRgba64(configuration, source, packed); ExtractAlpha(packed, alpha); HeifSampleConversion.WriteSamples( diff --git a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs index 522b2d0a5c..472abc44a3 100644 --- a/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs +++ b/src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs @@ -3,8 +3,6 @@ using System.Buffers; using System.Runtime.InteropServices; -using SixLabors.ImageSharp.Advanced; -using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; @@ -15,6 +13,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Components; /// internal static class HeifPlanarColorConverter { + /// + /// The number of planar color components retained for each source row. + /// + private const int ColorComponentCount = 3; + + /// + /// The number of source rows consumed together by vertically subsampled chroma. + /// + private const int VerticallySubsampledRowCount = 2; + /// /// The largest value represented by an eight-bit packed RGB component. /// @@ -26,7 +34,20 @@ internal static class HeifPlanarColorConverter private const float UShortMaximum = ushort.MaxValue; /// - /// Converts native unsigned 16-bit component storage to packed pixels and selects eligible exact integer kernels. + /// Converts complete native unsigned 16-bit component planes to packed pixels. + /// + public static void ConvertToRgb( + Configuration configuration, + TBuffer buffer, + ImageFrame image, + in HeifColorConversionParameters parameters, + HeifColorConversionMode mode) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + => ConvertToRgb(configuration, buffer, image, in parameters, mode, 0, 0); + + /// + /// Converts a region of native unsigned 16-bit component storage to packed pixels and selects eligible exact integer kernels. /// /// The destination pixel type. /// The codec adapter exposing the native component planes. @@ -43,8 +64,8 @@ internal static class HeifPlanarColorConverter ImageFrame image, in HeifColorConversionParameters parameters, HeifColorConversionMode mode, - int sourceX = 0, - int sourceY = 0) + int sourceX, + int sourceY) where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer { @@ -74,7 +95,22 @@ internal static class HeifPlanarColorConverter } /// - /// Converts native component planes to packed pixels. + /// Converts complete native component planes to packed pixels. + /// + public static void ConvertToRgb( + Configuration configuration, + TBuffer buffer, + ImageFrame image, + in HeifColorConversionParameters parameters, + HeifColorConversionMode mode) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TLoader : struct, IHeifSampleConverter + => ConvertToRgb(configuration, buffer, image, in parameters, mode, 0, 0); + + /// + /// Converts a region of native component planes to packed pixels. /// /// The destination pixel type. /// The codec adapter exposing the native component planes. @@ -93,8 +129,8 @@ internal static class HeifPlanarColorConverter ImageFrame image, in HeifColorConversionParameters parameters, HeifColorConversionMode mode, - int sourceX = 0, - int sourceY = 0) + int sourceX, + int sourceY) where TPixel : unmanaged, IPixel where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged @@ -139,12 +175,48 @@ internal static class HeifPlanarColorConverter where TBuffer : struct, IHeifPlanarSampleBuffer where TSample : unmanaged where TStorer : struct, IHeifSampleConverter + { + Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); + ConvertFromRgb( + configuration, + image, + sourceRectangle, + buffer, + in parameters, + mode); + } + + /// + /// Converts a rectangular packed-pixel region to native component planes. + /// + /// The source pixel type. + /// The codec adapter exposing the native component planes. + /// The native unsigned sample storage type. + /// The SIMD narrowing and storage operations for the sample type. + /// The configuration used for allocation and pixel conversion. + /// The source image frame. + /// The source region mapped to the complete destination buffer. + /// The destination component-plane buffer. + /// The resolved H.273 conversion parameters. + /// The resolved H.273 conversion mode. + public static void ConvertFromRgb( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + TBuffer buffer, + in HeifColorConversionParameters parameters, + HeifColorConversionMode mode) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter { HeifColorConverterBase colorConverter = HeifColorConverterBase.Create(mode, in parameters, buffer.IsMonochrome); RgbToYuvRowConverter converter = new( configuration, buffer, image, + sourceRectangle, colorConverter, in parameters); @@ -156,10 +228,60 @@ internal static class HeifPlanarColorConverter return; } - using IMemoryOwner packedOwner = configuration.MemoryAllocator.Allocate(image.Width); - converter.Convert(packedOwner.GetSpan()[..image.Width], components); + using IMemoryOwner packedOwner = configuration.MemoryAllocator.Allocate(sourceRectangle.Width); + converter.Convert(packedOwner.GetSpan()[..sourceRectangle.Width], components); } + /// + /// Converts a rectangular packed-pixel region using a retained color converter and caller-owned row storage. + /// + /// The source pixel type. + /// The codec adapter exposing the native component planes. + /// The native unsigned sample storage type. + /// The SIMD narrowing and storage operations for the sample type. + /// The configuration used for pixel conversion. + /// The source image frame. + /// The source region mapped to the complete destination buffer. + /// The destination component-plane buffer. + /// The resolved H.273 conversion parameters. + /// The retained converter matching . + /// The reusable high-bit-depth packed RGB row, or an empty span for eight-bit input. + /// The reusable planar component rows. + public static void ConvertFromRgb( + Configuration configuration, + ImageFrame image, + Rectangle sourceRectangle, + TBuffer buffer, + in HeifColorConversionParameters parameters, + HeifColorConverterBase colorConverter, + Span packed, + Span components) + where TPixel : unmanaged, IPixel + where TBuffer : struct, IHeifPlanarSampleBuffer + where TSample : unmanaged + where TStorer : struct, IHeifSampleConverter + { + RgbToYuvRowConverter converter = new( + configuration, + buffer, + image, + sourceRectangle, + colorConverter, + in parameters); + + converter.Convert(packed, components); + } + + /// + /// Gets the planar float storage required to convert one row, or one vertically subsampled row pair. + /// + /// The source-row width. + /// Whether only luma is written. + /// The vertical chroma-subsampling shift. + /// The required number of float elements. + public static int GetRgbToYuvComponentBufferLength(int width, bool isMonochrome, int subsamplingY) + => width * ColorComponentCount * (subsamplingY == 0 || isMonochrome ? 1 : VerticallySubsampledRowCount); + /// /// Resolves the two chroma rows and quarter-sample weight surrounding a luma row. /// @@ -484,6 +606,11 @@ internal static class HeifPlanarColorConverter /// private readonly ImageFrame image; + /// + /// The source region mapped to the complete destination planes. + /// + private readonly Rectangle sourceRectangle; + /// /// The selected H.273 color converter. /// @@ -535,18 +662,21 @@ internal static class HeifPlanarColorConverter /// The configuration used for pixel conversion. /// The codec adapter exposing the destination component planes. /// The source image frame. + /// The source region mapped to the complete destination planes. /// The selected H.273 color converter. /// The resolved H.273 component ranges. public RgbToYuvRowConverter( Configuration configuration, TBuffer buffer, ImageFrame image, + Rectangle sourceRectangle, HeifColorConverterBase colorConverter, in HeifColorConversionParameters parameters) { this.configuration = configuration; this.buffer = buffer; this.image = image; + this.sourceRectangle = sourceRectangle; this.colorConverter = colorConverter; this.lumaMaximum = parameters.LumaSampleMaximum; this.chromaMaximum = parameters.ChromaSampleMaximum; @@ -566,7 +696,8 @@ internal static class HeifPlanarColorConverter /// /// Gets the number of float elements required by the reusable component buffer. /// - public readonly int ComponentBufferLength => this.image.Width * (this.subsamplingY == 0 || this.isMonochrome ? 3 : 6); + public readonly int ComponentBufferLength + => GetRgbToYuvComponentBufferLength(this.sourceRectangle.Width, this.isMonochrome, this.subsamplingY); /// /// Converts every packed source row to the destination component planes. @@ -575,13 +706,13 @@ internal static class HeifPlanarColorConverter /// The reusable planar component buffer. public void Convert(Span packed, Span components) { - int width = this.image.Width; + int width = this.sourceRectangle.Width; Span luma0 = components[..width]; Span blue0 = components.Slice(width, width); Span red0 = components.Slice(width * 2, width); if (this.subsamplingY == 0) { - for (int y = 0; y < this.image.Height; y++) + for (int y = 0; y < this.sourceRectangle.Height; y++) { this.ConvertSourceRow(y, packed, luma0, blue0, red0); HeifSampleConversion.WriteSamples( @@ -603,7 +734,7 @@ internal static class HeifPlanarColorConverter Span luma1 = this.isMonochrome ? luma0 : components.Slice(width * 3, width); Span blue1 = this.isMonochrome ? blue0 : components.Slice(width * 4, width); Span red1 = this.isMonochrome ? red0 : components.Slice(width * 5, width); - int chromaHeight = (this.image.Height + 1) >> 1; + int chromaHeight = (this.sourceRectangle.Height + 1) >> 1; for (int destinationY = 0; destinationY < chromaHeight; destinationY++) { // A vertically subsampled chroma row is owned by one two-row luma cell. Processing that cell as a @@ -617,7 +748,7 @@ internal static class HeifPlanarColorConverter this.colorConverter.LumaBias, this.lumaMaximum); - bool hasSecondRow = sourceY + 1 < this.image.Height; + bool hasSecondRow = sourceY + 1 < this.sourceRectangle.Height; if (hasSecondRow) { this.ConvertSourceRow(sourceY + 1, packed, luma1, blue1, red1); @@ -659,7 +790,10 @@ internal static class HeifPlanarColorConverter /// The destination red-difference or third component values. private void ConvertSourceRow(int y, Span packed, Span luma, Span chromaBlue, Span chromaRed) { - ReadOnlySpan source = this.image.PixelBuffer.DangerousGetRowSpan(y); + ReadOnlySpan source = this.image.PixelBuffer + .DangerousGetRowSpan(this.sourceRectangle.Y + y) + .Slice(this.sourceRectangle.X, this.sourceRectangle.Width); + if (this.UsesByteInput) { // JPEG's planar unpack contract reaches the existing pixel-specific SIMD implementation before diff --git a/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs index 8f10ef45ab..996893c1d6 100644 --- a/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs @@ -3,7 +3,6 @@ using System.Buffers; using System.Buffers.Binary; -using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Memory; @@ -19,6 +18,31 @@ namespace SixLabors.ImageSharp.Formats.Heif; internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IHeifAlphaItemDecoder where TPixel : unmanaged, IPixel { + /// + /// The image-grid descriptor version defined by HEIF. + /// + private const byte GridDescriptorVersion = 0; + + /// + /// The descriptor flag that selects 32-bit output dimensions instead of 16-bit dimensions. + /// + private const byte LargeDimensionsFlag = 1; + + /// + /// The descriptor length when output dimensions use 16-bit fields. + /// + private const int ShortGridDescriptorLength = 8; + + /// + /// The descriptor length when output dimensions use 32-bit fields. + /// + private const int LongGridDescriptorLength = 12; + + /// + /// The minimum width and height of the first cell in a MIAF image grid. + /// + private const int MinimumGridCellDimension = 64; + /// /// The item definitions available to the grid, indexed by item identifier. /// @@ -122,9 +146,16 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH ref av1GridConfiguration, cancellationToken); - if (tile.Width != tileWidth || tile.Height != tileHeight) + Size copySize = GetGridTileCopySize( + descriptor, + tileWidth, + tileHeight, + tileIndex); + + if (!IsGridTileExtentValid(tile.Size, copySize, tileWidth, tileHeight)) { - throw new InvalidImageContentException("The HEIF image grid contains tiles with mismatched dimensions."); + throw new InvalidImageContentException( + $"HEIF image grid tile {item.Id} has dimensions {tile.Size}, which cannot cover its {copySize} grid region."); } CopyGridTile(tile, result, descriptor, tileIndex, tileWidth, tileHeight); @@ -165,16 +196,13 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH tileWidth = tile.Width; tileHeight = tile.Height; - if (((long)tileWidth * descriptor.Columns) < descriptor.OutputSize.Width || - ((long)tileHeight * descriptor.Rows) < descriptor.OutputSize.Height) - { - throw new InvalidImageContentException("The HEIF image grid tiles do not cover the output canvas."); - } - - if (((long)tileWidth * (descriptor.Columns - 1)) >= descriptor.OutputSize.Width || - ((long)tileHeight * (descriptor.Rows - 1)) >= descriptor.OutputSize.Height) + ValidateGridCoverage(descriptor, tileWidth, tileHeight); + ValidateGridDimensions(descriptor, tile.Size, av1GridConfiguration); + Size copySize = GetGridTileCopySize(descriptor, tileWidth, tileHeight, 0); + if (!IsGridTileExtentValid(tile.Size, copySize, tileWidth, tileHeight)) { - throw new InvalidImageContentException("The HEIF image grid edge tiles do not overlap the output canvas."); + throw new InvalidImageContentException( + $"HEIF image grid tile {item.Id} has dimensions {tile.Size}, which cannot cover its {copySize} grid region."); } Image result = new( @@ -279,12 +307,20 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH IReadOnlyList linked = this.GetLinkedTileIds(gridItem, descriptor); Heif4CharCode tileType = default; Av1CodecConfiguration? av1GridConfiguration = null; - Size tileSize = default; + HeifItem firstItem = this.items[linked[0]]; + if (firstItem.Extent == default) + { + throw new InvalidImageContentException($"HEIF alpha grid tile {firstItem.Id} has no spatial extent."); + } + + Size tileSize = firstItem.Extent; + ValidateGridCoverage(descriptor, tileSize.Width, tileSize.Height); // Validate the complete grid before mutating the color frame. IgnoreImageData can then omit a failed alpha // grid without leaving a partially composed prefix in the returned image. - foreach (uint id in linked) + for (int tileIndex = 0; tileIndex < linked.Count; tileIndex++) { + uint id = linked[tileIndex]; HeifItem item = this.items[id]; ValidateTileConfiguration(item, ref tileType, ref av1GridConfiguration); if (HeifCompressionFactory.GetDecoder(item.Type) is not IHeifAlphaItemDecoder) @@ -297,29 +333,23 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH throw new InvalidImageContentException($"HEIF alpha grid tile {item.Id} has no spatial extent."); } - if (tileSize == default) - { - tileSize = item.Extent; - } - else if (item.Extent != tileSize) + Size copySize = GetGridTileCopySize( + descriptor, + tileSize.Width, + tileSize.Height, + tileIndex); + + if (!IsGridTileExtentValid(item.Extent, copySize, tileSize.Width, tileSize.Height)) { - throw new InvalidImageContentException("The HEIF alpha grid contains tiles with mismatched dimensions."); + throw new InvalidImageContentException( + $"HEIF alpha grid tile {item.Id} has dimensions {item.Extent}, which cannot cover its {copySize} grid region."); } } + ValidateGridDimensions(descriptor, tileSize, av1GridConfiguration); + int gridWidth = descriptor.OutputSize.Width; int gridHeight = descriptor.OutputSize.Height; - if (((long)tileSize.Width * descriptor.Columns) < gridWidth || ((long)tileSize.Height * descriptor.Rows) < gridHeight) - { - throw new InvalidImageContentException("The HEIF alpha grid tiles do not cover the output canvas."); - } - - if (((long)tileSize.Width * (descriptor.Columns - 1)) >= gridWidth || - ((long)tileSize.Height * (descriptor.Rows - 1)) >= gridHeight) - { - throw new InvalidImageContentException("The HEIF alpha grid edge tiles do not overlap the output canvas."); - } - if (descriptor.OutputSize != outputSize || destinationRectangle.Size != outputSize) { throw new InvalidImageContentException("The HEIF alpha grid dimensions do not match the color grid dimensions."); @@ -340,22 +370,109 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH int row = tileIndex / descriptor.Columns; int destinationX = destinationRectangle.X + (column * tileSize.Width); int destinationY = destinationRectangle.Y + (row * tileSize.Height); - int copyWidth = Math.Min(tileSize.Width, destinationRectangle.Right - destinationX); - int copyHeight = Math.Min(tileSize.Height, destinationRectangle.Bottom - destinationY); - Rectangle tileDestination = new(destinationX, destinationY, copyWidth, copyHeight); + Size copySize = GetGridTileCopySize( + descriptor, + tileSize.Width, + tileSize.Height, + tileIndex); + + Rectangle tileDestination = new(destinationX, destinationY, copySize.Width, copySize.Height); decoder.DecodeAlphaItemData( options, item, itemMemory.GetSpan(), destination, - tileSize, + item.Extent, tileDestination, premultiplied, cancellationToken); } } + /// + /// Validates that the first cell dimensions cover the grid while leaving a nonempty final row and column. + /// + private static void ValidateGridCoverage(in GridDescriptor descriptor, int tileWidth, int tileHeight) + { + if (((long)tileWidth * descriptor.Columns) < descriptor.OutputSize.Width || + ((long)tileHeight * descriptor.Rows) < descriptor.OutputSize.Height) + { + throw new InvalidImageContentException("The HEIF image grid tiles do not cover the output canvas."); + } + + if (((long)tileWidth * (descriptor.Columns - 1)) >= descriptor.OutputSize.Width || + ((long)tileHeight * (descriptor.Rows - 1)) >= descriptor.OutputSize.Height) + { + throw new InvalidImageContentException("The HEIF image grid edge tiles do not overlap the output canvas."); + } + } + + /// + /// Gets the portion of one cell that overlaps the output canvas. + /// + private static Size GetGridTileCopySize( + in GridDescriptor descriptor, + int tileWidth, + int tileHeight, + int tileIndex) + { + int column = tileIndex % descriptor.Columns; + int row = tileIndex / descriptor.Columns; + int copyWidth = column == descriptor.Columns - 1 + ? descriptor.OutputSize.Width - (tileWidth * (descriptor.Columns - 1)) + : tileWidth; + + int copyHeight = row == descriptor.Rows - 1 + ? descriptor.OutputSize.Height - (tileHeight * (descriptor.Rows - 1)) + : tileHeight; + + return new Size(copyWidth, copyHeight); + } + + /// + /// Determines whether a cell can cover its output region without exceeding the first cell's dimensions. + /// + private static bool IsGridTileExtentValid(Size extent, Size copySize, int tileWidth, int tileHeight) + => extent.Width >= copySize.Width + && extent.Width <= tileWidth + && extent.Height >= copySize.Height + && extent.Height <= tileHeight; + + /// + /// Validates the MIAF cell-size and chroma-alignment rules established by the first grid cell. + /// + private static void ValidateGridDimensions( + in GridDescriptor descriptor, + Size tileSize, + Av1CodecConfiguration? av1GridConfiguration) + { + if (tileSize.Width < MinimumGridCellDimension || tileSize.Height < MinimumGridCellDimension) + { + throw new InvalidImageContentException( + $"HEIF image grid cells must be at least {MinimumGridCellDimension} samples wide and high."); + } + + if (av1GridConfiguration is null || av1GridConfiguration.IsMonochrome) + { + return; + } + + if (av1GridConfiguration.ChromaSubsamplingX && + (((descriptor.OutputSize.Width & 1) != 0) || ((tileSize.Width & 1) != 0))) + { + throw new InvalidImageContentException( + "HEIF image grid widths must be even when AV1 chroma is horizontally subsampled."); + } + + if (av1GridConfiguration.ChromaSubsamplingY && + (((descriptor.OutputSize.Height & 1) != 0) || ((tileSize.Height & 1) != 0))) + { + throw new InvalidImageContentException( + "HEIF image grid heights must be even when AV1 chroma is vertically subsampled."); + } + } + /// /// Parses and validates the fixed HEIF image-grid descriptor fields used by both color and alpha composition. /// @@ -363,38 +480,43 @@ internal sealed class GridHeifItemDecoder : IHeifItemDecoder, IH /// The validated row, column, and output dimensions. private static GridDescriptor ParseGridDescriptor(ReadOnlySpan data) { - if (data.Length < 8) + if (data.Length < ShortGridDescriptorLength) { throw new InvalidImageContentException("The HEIF image grid descriptor is truncated."); } - byte version = data[0]; - if (version != 0) + int offset = 0; + byte version = data[offset++]; + if (version != GridDescriptorVersion) { throw new InvalidImageContentException($"The HEIF image grid descriptor has unsupported version {version}."); } - bool usesLargeDimensions = (data[1] & 1) != 0; - int descriptorLength = usesLargeDimensions ? 12 : 8; + byte flags = data[offset++]; + bool usesLargeDimensions = (flags & LargeDimensionsFlag) != 0; + int descriptorLength = usesLargeDimensions ? LongGridDescriptorLength : ShortGridDescriptorLength; if (data.Length != descriptorLength) { throw new InvalidImageContentException("The HEIF image grid descriptor has an invalid length."); } + int rows = data[offset++] + 1; + int columns = data[offset++] + 1; uint outputWidth = usesLargeDimensions - ? BinaryPrimitives.ReadUInt32BigEndian(data[4..]) - : BinaryPrimitives.ReadUInt16BigEndian(data[4..]); + ? BinaryPrimitives.ReadUInt32BigEndian(data[offset..]) + : BinaryPrimitives.ReadUInt16BigEndian(data[offset..]); + offset += usesLargeDimensions ? sizeof(uint) : sizeof(ushort); uint outputHeight = usesLargeDimensions - ? BinaryPrimitives.ReadUInt32BigEndian(data[8..]) - : BinaryPrimitives.ReadUInt16BigEndian(data[6..]); + ? BinaryPrimitives.ReadUInt32BigEndian(data[offset..]) + : BinaryPrimitives.ReadUInt16BigEndian(data[offset..]); if (outputWidth is 0 or > int.MaxValue || outputHeight is 0 or > int.MaxValue) { throw new InvalidImageContentException("The HEIF image grid descriptor has invalid output dimensions."); } - return new GridDescriptor(data[2] + 1, data[3] + 1, new Size((int)outputWidth, (int)outputHeight)); + return new GridDescriptor(rows, columns, new Size((int)outputWidth, (int)outputHeight)); } /// diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 574294dfdf..468b83ecc2 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -16,6 +16,7 @@ using SixLabors.ImageSharp.Metadata.Profiles.Icc; using SixLabors.ImageSharp.Metadata.Profiles.Xmp; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; +using SixLabors.ImageSharp.Processing.Processors.Transforms; namespace SixLabors.ImageSharp.Formats.Heif; @@ -266,23 +267,32 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// The bounded selected image-sequence model. private HeifSequence ParseImageSequence(BufferedReadStream stream) { + this.items.Clear(); + this.itemLinks.Clear(); + this.itemDataOffset = -1; + this.itemDataLength = 0; + HeifSequence? sequence = null; while (stream.Position < stream.Length) { long boxLength = HeifBoxReader.ReadHeader(stream, stream.Length, this.boxHeaderScratch, out Heif4CharCode boxType, true); - if (boxType == Heif4CharCode.Moov) + switch (boxType) { - if (sequence is not null) - { - throw new InvalidImageContentException("The HEIF image sequence contains more than one movie box."); - } + case Heif4CharCode.Meta: + this.ParseMetadata(stream, boxLength); + break; + case Heif4CharCode.Moov: + if (sequence is not null) + { + throw new InvalidImageContentException("The HEIF image sequence contains more than one movie box."); + } - sequence = this.sequenceParser.Parse(stream, boxLength, this.fileStartOffset); - } - else - { - // Sequence samples use absolute file offsets, so unrelated top-level payloads never need buffering. - HeifBoxReader.Skip(stream, boxLength); + sequence = this.sequenceParser.Parse(stream, boxLength, this.fileStartOffset); + break; + default: + // Sequence samples and image items use file-relative offsets, so payload boxes never need buffering. + HeifBoxReader.Skip(stream, boxLength); + break; } } @@ -297,9 +307,18 @@ internal sealed class HeifDecoderCore : ImageDecoderCore private ImageInfo IdentifyImageSequence(HeifSequence sequence) { HeifSequenceTrack colorTrack = sequence.ColorTrack; - this.UpdateSequenceMetadata(this.metadata, sequence); - ImageFrameMetadata[] frameMetadata = CreateSequenceFrameMetadata(colorTrack); - this.Dimensions = GetSequencePresentationExtent(colorTrack); + HeifItem primaryItem = this.FindSequencePrimaryItem(); + bool animateRootFrame = IsPrimaryItemFirstSequenceSample(primaryItem, colorTrack); + Size sequenceExtent = GetSequencePresentationExtent(colorTrack); + this.Dimensions = animateRootFrame ? sequenceExtent : GetPresentationExtent(primaryItem); + if (!animateRootFrame && this.Dimensions != sequenceExtent) + { + throw new InvalidImageContentException("The primary image and image sequence have different presentation dimensions."); + } + + this.UpdateMetadata(this.metadata, primaryItem); + this.UpdateSequenceMetadata(this.metadata, sequence, animateRootFrame); + ImageFrameMetadata[] frameMetadata = CreateSequenceFrameMetadata(colorTrack, animateRootFrame); return new ImageInfo(this.Dimensions, this.metadata, frameMetadata); } @@ -318,170 +337,241 @@ internal sealed class HeifDecoderCore : ImageDecoderCore where TPixel : unmanaged, IPixel { HeifSequenceTrack colorTrack = sequence.ColorTrack; - this.UpdateSequenceMetadata(this.metadata, sequence); - ImageFrame[] colorFrames = this.DecodeVisibleSequenceFrames( - stream, - colorTrack, - cancellationToken, - out int[] sampleIndices); + HeifItem primaryItem = this.FindSequencePrimaryItem(); + bool animateRootFrame = IsPrimaryItemFirstSequenceSample(primaryItem, colorTrack); + this.UpdateSequenceMetadata(this.metadata, sequence, animateRootFrame); + Size codedSize = new(colorTrack.CodedWidth, colorTrack.CodedHeight); + Rectangle sourceRectangle = colorTrack.CleanAperture is not null + ? colorTrack.CleanAperture.Value.ToRectangle(codedSize) + : new Rectangle(Point.Empty, codedSize); + + // HEIF stores counter-clockwise quarter turns; ImageSharp's exact modes are clockwise. + RotateMode rotation = colorTrack.RotationAngle switch + { + 1 => RotateMode.Rotate270, + 2 => RotateMode.Rotate180, + 3 => RotateMode.Rotate90, + _ => RotateMode.None + }; + + Size presentationSize = rotation is RotateMode.Rotate90 or RotateMode.Rotate270 + ? new Size(sourceRectangle.Height, sourceRectangle.Width) + : sourceRectangle.Size; + + // The returned image owns every presented frame from the outset. A separate primary becomes its root; + // otherwise the first successfully decoded timed sample fills the root allocated here. + Image image = animateRootFrame + ? new Image( + this.configuration, + presentationSize.Width, + presentationSize.Height, + this.metadata) + : this.DecodePrimaryItem(stream, cancellationToken); - Image? image = null; try { - // Each codec frame owns its pixel buffer. The multi-frame image adopts those buffers directly instead - // of cloning a complete decoded frame on every append. - image = new Image(this.configuration, this.metadata, colorFrames); + if (image.Size != presentationSize) + { + throw new InvalidImageContentException( + "The primary image and image sequence have different presentation dimensions."); + } + + using Av1Decoder colorDecoder = new(this.configuration); + HeifSequenceTrack? alphaTrack = sequence.AlphaTrack; - if (alphaTrack is not null) + (HeifSequenceTrack Track, Av1Decoder Decoder)? alphaState = alphaTrack is null + ? null + : (alphaTrack, new Av1Decoder(this.configuration)); + + using Av1Decoder? alphaDecoder = alphaState?.Decoder; + + // Quarter turns need source and destination frames with opposite dimensions. Reuse one source frame + // across the sequence, then rotate each completed color-and-alpha sample into its final owned frame. + using ImageFrame? rotationSource = rotation == RotateMode.None + ? null + : new ImageFrame( + this.configuration, + sourceRectangle.Width, + sourceRectangle.Height); + + int decodedFrameCount = 0; + for (int sampleIndex = 0; sampleIndex < colorTrack.Samples.Length; sampleIndex++) { - int frameIndex = 0; - using Av1Decoder alphaDecoder = new(this.configuration); - for (int sampleIndex = 0; sampleIndex < alphaTrack.Samples.Length; sampleIndex++) + cancellationToken.ThrowIfCancellationRequested(); + HeifSequenceSample colorSample = colorTrack.Samples[sampleIndex]; + if (colorSample.IsHidden) { - cancellationToken.ThrowIfCancellationRequested(); - HeifSequenceSample alphaSample = alphaTrack.Samples[sampleIndex]; - if (alphaSample.IsHidden || - frameIndex >= colorFrames.Length || - sampleIndices[frameIndex] != sampleIndex) + this.ExecuteImageDataSegmentAction( + () => this.DecodeSequenceReference(stream, colorTrack, colorSample, colorDecoder)); + + if (alphaState is not null) { + (HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value; + HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex]; this.ExecuteImageDataSegmentAction( - () => this.DecodeSequenceReference(stream, alphaTrack, alphaSample, alphaDecoder)); - - continue; + () => this.DecodeSequenceReference( + stream, + currentAlphaState.Track, + alphaSample, + currentAlphaState.Decoder)); } - this.ExecuteImageDataSegmentAction( - () => this.DecodeSequenceAlphaFrame( - stream, - alphaTrack, - alphaSample, - alphaDecoder, - colorFrames[frameIndex], - colorTrack.IsPremultiplied)); + continue; + } - frameIndex++; + bool appendedDestination = false; + ImageFrame decodedFrame; + if (rotationSource is not null) + { + decodedFrame = rotationSource; + } + else if (animateRootFrame && decodedFrameCount == 0) + { + decodedFrame = image.Frames.RootFrame; + } + else + { + decodedFrame = image.Frames.CreateFrame(); + appendedDestination = true; } - } - ApplyPresentationTransforms( - image, - colorTrack.CleanAperture, - colorTrack.RotationAngle, - colorTrack.MirrorAxis); + bool colorDecoded = false; + this.ExecuteImageDataSegmentAction( + () => + { + this.DecodeSequenceFrame( + stream, + colorTrack, + colorSample, + colorDecoder, + sourceRectangle, + decodedFrame); - if (!this.Options.SkipMetadata) - { - image.Metadata.CicpProfile ??= image.Frames.RootFrame.Metadata.CicpProfile?.DeepClone(); - _ = this.TryConvertIccProfile(image); - } - else - { - foreach (ImageFrame frame in image.Frames) + colorDecoded = true; + }); + + if (!colorDecoded) { - frame.Metadata.CicpProfile = null; + if (alphaState is not null) + { + (HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value; + HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex]; + this.ExecuteImageDataSegmentAction( + () => this.DecodeSequenceReference( + stream, + currentAlphaState.Track, + alphaSample, + currentAlphaState.Decoder)); + } + + if (appendedDestination) + { + image.Frames.RemoveFrame(image.Frames.Count - 1); + } + + continue; } - } - this.Dimensions = image.Size; - return image; - } - catch - { - if (image is not null) - { - image.Dispose(); - } - else - { - // Ownership transfers to Image only after its constructor validates every decoded frame. - foreach (ImageFrame frame in colorFrames) + bool alphaDecoded = true; + if (alphaState is not null) { - frame.Dispose(); + alphaDecoded = false; + (HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value; + HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex]; + this.ExecuteImageDataSegmentAction( + () => + { + this.DecodeSequenceAlphaFrame( + stream, + currentAlphaState.Track, + alphaSample, + currentAlphaState.Decoder, + sourceRectangle, + decodedFrame, + colorTrack.IsPremultiplied); + + alphaDecoded = true; + }); } - } - throw; - } - } + if (!alphaDecoded) + { + if (appendedDestination) + { + image.Frames.RemoveFrame(image.Frames.Count - 1); + } - /// - /// Decodes visible samples while preserving their source indices for frame-aligned alpha lookup. - /// - /// The destination pixel format. - /// The complete seekable HEIF stream. - /// The selected coded-image track. - /// The token used to cancel work between coded samples. - /// Receives the decode-order sample index for each returned visible frame. - /// The exact array of successfully decoded visible frames. - private ImageFrame[] DecodeVisibleSequenceFrames( - BufferedReadStream stream, - HeifSequenceTrack track, - CancellationToken cancellationToken, - out int[] sampleIndices) - where TPixel : unmanaged, IPixel - { - int visibleFrameCount = 0; - foreach (HeifSequenceSample sample in track.Samples) - { - visibleFrameCount += sample.IsHidden ? 0 : 1; - } + continue; + } - ImageFrame[] frames = new ImageFrame[visibleFrameCount]; - sampleIndices = new int[visibleFrameCount]; - int decodedFrameCount = 0; - using Av1Decoder decoder = new(this.configuration); - try - { - for (int sampleIndex = 0; sampleIndex < track.Samples.Length; sampleIndex++) - { - HeifSequenceSample sample = track.Samples[sampleIndex]; - if (sample.IsHidden) + ImageFrame presentedFrame = decodedFrame; + if (rotationSource is not null) { - this.ExecuteImageDataSegmentAction( - () => this.DecodeSequenceReference(stream, track, sample, decoder)); + presentedFrame = animateRootFrame && decodedFrameCount == 0 + ? image.Frames.RootFrame + : image.Frames.CreateFrame(); - continue; + RotateProcessor.ApplyQuarterTurn( + rotation, + rotationSource, + presentedFrame, + this.configuration); + + presentedFrame.Metadata.CicpProfile = rotationSource.Metadata.CicpProfile; } - cancellationToken.ThrowIfCancellationRequested(); - ImageFrame? frame = null; - this.ExecuteImageDataSegmentAction( - () => frame = this.DecodeSequenceFrame(stream, track, sample, decoder)); + if (colorTrack.MirrorAxis is not null) + { + // Axis zero reflects top-to-bottom around the horizontal axis; axis one reflects left-to-right. + FlipMode flip = colorTrack.MirrorAxis.Value == 0 ? FlipMode.Vertical : FlipMode.Horizontal; + FlipProcessor.Apply(flip, presentedFrame, this.configuration); + } + + presentedFrame.Metadata.GetHeifMetadata().FrameDelay = new Rational( + colorSample.Duration, + colorTrack.MediaTimescale); - if (frame is null) + if (!animateRootFrame && !this.Options.SkipMetadata) { - continue; + presentedFrame.Metadata.IccProfile = colorTrack.IccProfile; + _ = this.TryConvertIccProfile(presentedFrame); } - frame.Metadata.GetHeifMetadata().FrameDelay = new Rational(sample.Duration, track.MediaTimescale); - frames[decodedFrameCount] = frame; - sampleIndices[decodedFrameCount] = sampleIndex; decodedFrameCount++; } if (decodedFrameCount == 0) { - throw new InvalidImageContentException("The HEIF image sequence contains no decodable visible samples."); + throw new InvalidImageContentException( + "The HEIF image sequence contains no decodable visible samples."); } - if (decodedFrameCount != frames.Length) + if (!this.Options.SkipMetadata) + { + if (animateRootFrame) + { + image.Metadata.CicpProfile ??= image.Frames.RootFrame.Metadata.CicpProfile?.DeepClone(); + _ = this.TryConvertIccProfile(image); + } + } + else { - // Compaction occurs only in IgnoreImageData mode after a recoverable coded-sample failure. - Array.Resize(ref frames, decodedFrameCount); - Array.Resize(ref sampleIndices, decodedFrameCount); + foreach (ImageFrame frame in image.Frames) + { + frame.Metadata.CicpProfile = null; + } } - return frames; + HeifMetadata resultMetadata = image.Metadata.GetHeifMetadata(); + resultMetadata.RepeatCount = colorTrack.RepeatCount; + resultMetadata.AnimateRootFrame = animateRootFrame; + resultMetadata.HasAlpha |= alphaState is not null; + this.Dimensions = image.Size; + return image; } catch { - // Frames are independently allocated before the final Image adopts them. Retain ownership until this - // method returns so a later sample failure cannot leak the successfully decoded prefix. - for (int frameIndex = 0; frameIndex < decodedFrameCount; frameIndex++) - { - frames[frameIndex].Dispose(); - } - + image.Dispose(); throw; } } @@ -494,12 +584,15 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// The track supplying the codec configuration and color description. /// The validated sample range. /// The decoder retaining earlier sequence references. - /// The independently owned decoded frame. - private ImageFrame DecodeSequenceFrame( + /// The clean-aperture region mapped to the destination frame. + /// The caller-owned frame receiving the presented sample. + private void DecodeSequenceFrame( BufferedReadStream stream, HeifSequenceTrack track, HeifSequenceSample sample, - Av1Decoder decoder) + Av1Decoder decoder, + Rectangle sourceRectangle, + ImageFrame destination) where TPixel : unmanaged, IPixel { if (track.CodecType != Heif4CharCode.Av01) @@ -513,18 +606,13 @@ internal sealed class HeifDecoderCore : ImageDecoderCore using IMemoryOwner sampleOwner = this.ReadSequenceSample(stream, track, sample); Span sampleData = sampleOwner.GetSpan()[..sample.Length]; - ImageFrame frame = decoder.DecodeSequenceFrame( + decoder.DecodeSequenceFrame( sampleData, track.CicpProfile, - codecConfiguration); - - if (frame.Width != track.CodedWidth || frame.Height != track.CodedHeight) - { - frame.Dispose(); - throw new InvalidImageContentException("The decoded image-sequence sample dimensions do not match its visual sample entry."); - } - - return frame; + codecConfiguration, + new Size(track.CodedWidth, track.CodedHeight), + sourceRectangle, + destination); } /// @@ -535,6 +623,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// The alpha track supplying the codec configuration and color description. /// The validated alpha sample range. /// The decoder retaining earlier alpha-sequence references. + /// The clean-aperture region mapped to the destination frame. /// The decoded color frame receiving alpha values. /// Whether stored color samples must be converted to unassociated alpha. private void DecodeSequenceAlphaFrame( @@ -542,6 +631,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore HeifSequenceTrack track, HeifSequenceSample sample, Av1Decoder decoder, + Rectangle sourceRectangle, ImageFrame destination, bool premultiplied) where TPixel : unmanaged, IPixel @@ -566,6 +656,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore track.CicpProfile, codecConfiguration, new Size(track.CodedWidth, track.CodedHeight), + sourceRectangle, destination, destination.Size, destination.Bounds, @@ -642,28 +733,33 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// /// The image metadata receiving the sequence description. /// The parsed selected image sequence. - private void UpdateSequenceMetadata(ImageMetadata metadata, HeifSequence sequence) + /// Whether the primary image is also the first timed sample. + private void UpdateSequenceMetadata(ImageMetadata metadata, HeifSequence sequence, bool animateRootFrame) { HeifSequenceTrack colorTrack = sequence.ColorTrack; HeifMetadata heifMetadata = metadata.GetHeifMetadata(); heifMetadata.RepeatCount = colorTrack.RepeatCount; - heifMetadata.AnimateRootFrame = true; - heifMetadata.HasAlpha = sequence.AlphaTrack is not null; + heifMetadata.AnimateRootFrame = animateRootFrame; + heifMetadata.HasAlpha |= sequence.AlphaTrack is not null; switch (colorTrack.CodecType) { case Heif4CharCode.Av01: Av1CodecConfiguration av1Configuration = colorTrack.Av1CodecConfiguration ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); - heifMetadata.CompressionMethod = HeifCompressionMethod.Av1; - heifMetadata.BitDepth = av1Configuration.BitDepth; - heifMetadata.IsMonochrome = av1Configuration.IsMonochrome; + if (animateRootFrame) + { + heifMetadata.CompressionMethod = HeifCompressionMethod.Av1; + heifMetadata.BitDepth = av1Configuration.BitDepth; + heifMetadata.IsMonochrome = av1Configuration.IsMonochrome; + } + break; default: throw new InvalidImageContentException($"The image-sequence sample entry '{colorTrack.CodecType}' is not supported."); } - if (this.Options.SkipMetadata) + if (this.Options.SkipMetadata || !animateRootFrame) { return; } @@ -696,8 +792,11 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// Creates one HEIF frame-metadata entry for each visible retained sequence sample. /// /// The selected color track supplying sample durations. + /// Whether the first sequence sample occupies the root-frame slot. /// The exact visible-frame metadata array in presentation order. - private static ImageFrameMetadata[] CreateSequenceFrameMetadata(HeifSequenceTrack track) + private static ImageFrameMetadata[] CreateSequenceFrameMetadata( + HeifSequenceTrack track, + bool animateRootFrame) { int visibleFrameCount = 0; foreach (HeifSequenceSample sample in track.Samples) @@ -705,8 +804,14 @@ internal sealed class HeifDecoderCore : ImageDecoderCore visibleFrameCount += sample.IsHidden ? 0 : 1; } - ImageFrameMetadata[] result = new ImageFrameMetadata[visibleFrameCount]; - int frameIndex = 0; + int firstSequenceFrameIndex = animateRootFrame ? 0 : 1; + ImageFrameMetadata[] result = new ImageFrameMetadata[visibleFrameCount + firstSequenceFrameIndex]; + if (!animateRootFrame) + { + result[0] = new ImageFrameMetadata(); + } + + int frameIndex = firstSequenceFrameIndex; foreach (HeifSequenceSample sample in track.Samples) { if (sample.IsHidden) @@ -722,6 +827,39 @@ internal sealed class HeifDecoderCore : ImageDecoderCore return result; } + /// + /// Gets the primary image item required alongside an AVIF image sequence. + /// + private HeifItem FindSequencePrimaryItem() + => this.FindItemById(this.primaryItem) + ?? throw new InvalidImageContentException("The HEIF image sequence contains no primary image item."); + + /// + /// Determines whether the primary image item reuses the first presented sequence sample. + /// + private static bool IsPrimaryItemFirstSequenceSample(HeifItem primaryItem, HeifSequenceTrack colorTrack) + { + if (primaryItem.DataLocations.Count != 1) + { + return false; + } + + HeifLocation location = primaryItem.DataLocations[0]; + foreach (HeifSequenceSample sample in colorTrack.Samples) + { + if (sample.IsHidden) + { + continue; + } + + return location.Origin == HeifLocationOffsetOrigin.FileOffset + && location.BaseOffset + location.Offset == sample.Offset + && location.Length == sample.Length; + } + + return false; + } + /// /// Updates identification metadata from the primary item or its decodable thumbnail fallback. /// diff --git a/src/ImageSharp/Formats/Heif/HeifEncoder.cs b/src/ImageSharp/Formats/Heif/HeifEncoder.cs index db26aec078..1b6472ee5a 100644 --- a/src/ImageSharp/Formats/Heif/HeifEncoder.cs +++ b/src/ImageSharp/Formats/Heif/HeifEncoder.cs @@ -92,20 +92,22 @@ public sealed class HeifEncoder : AnimatedImageEncoder /// /// Gets a value indicating whether the primary and auxiliary alpha images are encoded without loss. When /// , and do not affect the encoded image. - /// Legacy JPEG image items do not support lossless encoding. The default is . + /// This option has no effect on legacy JPEG image items. The default is . /// public bool Lossless { get; init; } /// /// Gets the encoded precision of each image component, or to use the HEIF metadata bit /// depth. Metadata that does not specify a bit depth defaults to . Legacy JPEG - /// image items support only . + /// image items are always encoded with . /// public HeifBitDepth? BitDepth { get; init; } /// /// Gets the encoded chroma sampling, or to use - /// for lossy encoding and for lossless encoding. + /// for lossy encoding and for lossless encoding. Oversized still + /// images use when a subsampled AVIF grid cannot represent an odd + /// output dimension. /// public HeifChromaSubsampling? ChromaSubsampling { get; init; } diff --git a/src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs b/src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs index 1bf61d1904..7af697ec86 100644 --- a/src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs +++ b/src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs @@ -16,16 +16,59 @@ namespace SixLabors.ImageSharp.Formats.Heif; internal sealed partial class HeifEncoderCore { + /// + /// The millisecond media timescale used when every frame delay can be represented exactly. + /// private const uint DefaultSequenceTimescale = 1000; + + /// + /// The microsecond fallback used when the exact common frame-delay timescale exceeds 32 bits. + /// private const uint FallbackSequenceTimescale = 1000000; + + /// + /// The identity value for signed 16.16 movie and track matrix entries. + /// private const uint UnityFixed16Point16 = 1U << 16; + + /// + /// The identity value for the signed 2.30 homogeneous movie and track matrix entry. + /// private const uint UnityFixed2Point30 = 1U << 30; + + /// + /// The identity value for unsigned 8.8 track volume. + /// private const ushort UnityFixed8Point8 = 1 << 8; + + /// + /// The packed ISO 639-2/T language code for undetermined content. + /// private const ushort PackedUndeterminedLanguage = 0x55C4; + + /// + /// The coding-constraints flag stating that every reference picture is intra. + /// private const uint AllReferencePicturesIntraMask = 1U << 31; + + /// + /// The coding-constraints flag stating that intra prediction is used. + /// private const uint IntraPicturePredictionUsedMask = 1U << 30; + + /// + /// The conventional 72-dpi horizontal and vertical resolution stored as unsigned 16.16. + /// private const uint DefaultVisualSampleResolution = 72U << 16; + + /// + /// The fixed visual-sample-entry compressor-name field length. + /// private const int VisualSampleCompressorNameLength = 32; + + /// + /// The visual-sample-entry depth used for color pictures. + /// private const ushort VisualSampleDepth = 24; private Av1EncodingSettings ResolveAv1Encoding(Image image) @@ -48,6 +91,18 @@ internal sealed partial class HeifEncoderCore HeifChromaSubsampling chromaSubsampling = this.encoder.ChromaSubsampling ?? (metadata.IsMonochrome ? HeifChromaSubsampling.Monochrome : defaultChromaSubsampling); + if (this.encoder.ChromaSubsampling is null + && image.Frames.Count == 1 + && (image.Width > Av1Constants.MaxFrameDimension || image.Height > Av1Constants.MaxFrameDimension) + && ((chromaSubsampling == HeifChromaSubsampling.Yuv420 + && (((image.Width & 1) != 0) || ((image.Height & 1) != 0))) + || (chromaSubsampling == HeifChromaSubsampling.Yuv422 && (image.Width & 1) != 0))) + { + // A derived grid requires even output dimensions on every subsampled axis. When sampling was not + // explicitly requested, retain the complete image dimensions by selecting full-resolution chroma. + chromaSubsampling = HeifChromaSubsampling.Yuv444; + } + (bool isMonochrome, bool subsamplingX, bool subsamplingY) = chromaSubsampling switch { HeifChromaSubsampling.Monochrome => (true, true, true), @@ -143,14 +198,10 @@ internal sealed partial class HeifEncoderCore ChunkedMemoryStream stream, Av1EncodingSettings settings, Memory samples, + int firstFrameIndex, CancellationToken cancellationToken) where TPixel : unmanaged, IPixel { - if (image.Width > ushort.MaxValue || image.Height > ushort.MaxValue) - { - throw new NotSupportedException("AV1 image-sequence dimensions cannot exceed 65535 pixels."); - } - byte[]? exifData = null; uint tiffHeaderOffset = 0; byte[]? xmpData = null; @@ -164,47 +215,58 @@ internal sealed partial class HeifEncoderCore } } - int frameCount = image.Frames.Count; - uint timescale = GetSequenceTimescale(image); + int frameCount = image.Frames.Count - firstFrameIndex; + uint timescale = GetSequenceTimescale(image, firstFrameIndex); // The container needs only offset, length, and duration after each frame is streamed. Color and alpha // share one allocator-owned table, with each track occupying one contiguous slice until moov is written. Span colorSamples = samples.Span[..frameCount]; - ImageFrame rootFrame = image.Frames.RootFrame; - uint duration = GetSequenceSampleDuration(rootFrame.Metadata.GetHeifMetadata().FrameDelay, timescale); - cancellationToken.ThrowIfCancellationRequested(); - long colorOffset = stream.Length; - ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode( + ImageFrame firstFrame = image.Frames[firstFrameIndex]; + ObuSequenceHeader colorHeader; + bool colorUsesInterPrediction = settings.ColorQIndex != 0; + using (Av1FrameEncoder.SequenceEncoder colorEncoder = Av1FrameEncoder.CreateColorSequenceEncoder( this.configuration, - rootFrame, - stream, + image.Width, + image.Height, settings.ColorConfig, settings.ColorQIndex, - this.encoder.Effort); - - colorSamples[0] = new HeifSequenceSampleInfo( - colorOffset, - checked((int)(stream.Length - colorOffset)), - duration); - - for (int frameIndex = 1; frameIndex < frameCount; frameIndex++) + this.encoder.Effort)) { cancellationToken.ThrowIfCancellationRequested(); - ImageFrame frame = image.Frames[frameIndex]; - duration = GetSequenceSampleDuration(frame.Metadata.GetHeifMetadata().FrameDelay, timescale); - colorOffset = stream.Length; - _ = Av1FrameEncoder.Encode( - this.configuration, - frame, - stream, - settings.ColorConfig, - settings.ColorQIndex, - this.encoder.Effort); + long colorOffset = stream.Length; + colorEncoder.EncodeKeyFrame(firstFrame, stream); + colorHeader = colorEncoder.SequenceHeader; - colorSamples[frameIndex] = new HeifSequenceSampleInfo( + colorSamples[0] = new HeifSequenceSampleInfo( colorOffset, checked((int)(stream.Length - colorOffset)), - duration); + GetSequenceSampleDuration(firstFrame.Metadata.GetHeifMetadata().FrameDelay, timescale), + isSyncSample: true); + + for (int sampleIndex = 1; sampleIndex < frameCount; sampleIndex++) + { + cancellationToken.ThrowIfCancellationRequested(); + int frameIndex = firstFrameIndex + sampleIndex; + ImageFrame frame = image.Frames[frameIndex]; + uint duration = GetSequenceSampleDuration(frame.Metadata.GetHeifMetadata().FrameDelay, timescale); + colorOffset = stream.Length; + if (colorUsesInterPrediction) + { + colorEncoder.EncodeInterFrame(frame, stream); + } + else + { + // Lossless AV1 requires 4x4 transforms. Until the inter path supports that reversible size, + // continuation samples remain independent key frames instead of weakening losslessness. + colorEncoder.EncodeKeyFrame(frame, stream); + } + + colorSamples[sampleIndex] = new HeifSequenceSampleInfo( + colorOffset, + checked((int)(stream.Length - colorOffset)), + duration, + isSyncSample: !colorUsesInterPrediction); + } } HeifSequenceTrackEncoding colorTrack = new( @@ -217,37 +279,47 @@ internal sealed partial class HeifEncoderCore { Memory alphaSampleMemory = samples.Slice(frameCount, frameCount); Span alphaSamples = alphaSampleMemory.Span; - cancellationToken.ThrowIfCancellationRequested(); - long alphaOffset = stream.Length; - ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha( + ObuSequenceHeader alphaHeader; + bool alphaUsesInterPrediction = settings.AlphaQIndex != 0; + using (Av1FrameEncoder.SequenceEncoder alphaEncoder = Av1FrameEncoder.CreateAlphaSequenceEncoder( this.configuration, - rootFrame, - stream, + image.Width, + image.Height, settings.AlphaConfig, settings.AlphaQIndex, - this.encoder.Effort); - - alphaSamples[0] = new HeifSequenceSampleInfo( - alphaOffset, - checked((int)(stream.Length - alphaOffset)), - colorSamples[0].Duration); - - for (int frameIndex = 1; frameIndex < frameCount; frameIndex++) + this.encoder.Effort)) { cancellationToken.ThrowIfCancellationRequested(); - alphaOffset = stream.Length; - _ = Av1FrameEncoder.EncodeAlpha( - this.configuration, - image.Frames[frameIndex], - stream, - settings.AlphaConfig, - settings.AlphaQIndex, - this.encoder.Effort); - - alphaSamples[frameIndex] = new HeifSequenceSampleInfo( + long alphaOffset = stream.Length; + alphaEncoder.EncodeKeyFrame(firstFrame, stream); + alphaHeader = alphaEncoder.SequenceHeader; + + alphaSamples[0] = new HeifSequenceSampleInfo( alphaOffset, checked((int)(stream.Length - alphaOffset)), - colorSamples[frameIndex].Duration); + colorSamples[0].Duration, + isSyncSample: true); + + for (int sampleIndex = 1; sampleIndex < frameCount; sampleIndex++) + { + cancellationToken.ThrowIfCancellationRequested(); + int frameIndex = firstFrameIndex + sampleIndex; + alphaOffset = stream.Length; + if (alphaUsesInterPrediction) + { + alphaEncoder.EncodeInterFrame(image.Frames[frameIndex], stream); + } + else + { + alphaEncoder.EncodeKeyFrame(image.Frames[frameIndex], stream); + } + + alphaSamples[sampleIndex] = new HeifSequenceSampleInfo( + alphaOffset, + checked((int)(stream.Length - alphaOffset)), + colorSamples[sampleIndex].Duration, + isSyncSample: !alphaUsesInterPrediction); + } } alphaTrack = new HeifSequenceTrackEncoding( @@ -266,7 +338,7 @@ internal sealed partial class HeifEncoderCore return new HeifSequenceEncoding( image.Width, image.Height, - image.Metadata.GetHeifMetadata().RepeatCount, + this.encoder.RepeatCount ?? image.Metadata.GetHeifMetadata().RepeatCount, timescale, colorTrack, alphaTrack, @@ -279,7 +351,7 @@ internal sealed partial class HeifEncoderCore private int WriteSequenceFileTypeBox(Stream stream) { - Span buffer = stackalloc byte[32]; + Span buffer = stackalloc byte[44]; int bytesWritten = WriteBoxHeader(buffer, Heif4CharCode.Ftyp); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis); bytesWritten += sizeof(uint); @@ -287,18 +359,24 @@ internal sealed partial class HeifEncoderCore bytesWritten += sizeof(uint); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avif); bytesWritten += sizeof(uint); + BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avio); + bytesWritten += sizeof(uint); + BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis); + bytesWritten += sizeof(uint); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Msf1); bytesWritten += sizeof(uint); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Iso8); bytesWritten += sizeof(uint); - BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avio); + BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Mif1); + bytesWritten += sizeof(uint); + BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Miaf); bytesWritten += sizeof(uint); BinaryPrimitives.WriteUInt32BigEndian(buffer, (uint)bytesWritten); stream.Write(buffer[..bytesWritten]); return bytesWritten; } - private void WriteSequenceMovieBox(HeifSequenceEncoding sequence, int fileTypeLength, Stream stream) + private void WriteSequenceMovieBox(HeifSequenceEncoding sequence, int precedingBoxLength, Stream stream) { int movieLength = GetSequenceMovieBoxLength(sequence); using IMemoryOwner movieOwner = this.configuration.MemoryAllocator.Allocate(movieLength); @@ -343,7 +421,7 @@ internal sealed partial class HeifEncoderCore } EndSequenceBox(memory, movieStart, offset); - ulong mediaDataOffset = checked((ulong)fileTypeLength + (uint)offset + 8U); + ulong mediaDataOffset = checked((ulong)precedingBoxLength + (uint)offset + 8U); BinaryPrimitives.WriteUInt64BigEndian( memory[colorChunkOffsetPosition..], checked(mediaDataOffset + (ulong)sequence.ColorTrack.Samples[0].Offset)); @@ -377,7 +455,8 @@ internal sealed partial class HeifEncoderCore const int chunkOffsetBoxLength = 24; const int syncSampleBoxFixedLength = 16; const int timingRunLength = 8; - const int sampleSizeAndSyncEntryLength = 8; + const int sampleSizeEntryLength = sizeof(uint); + const int syncSampleEntryLength = sizeof(uint); const int sampleTableFixedLength = sampleTableBoxHeaderLength + sampleDescriptionBoxLength @@ -398,10 +477,12 @@ internal sealed partial class HeifEncoderCore int repeatBoxLength = sequence.RepeatCount == 1 ? 0 : editListBoxLength; int colorRunCount = GetSequenceTimingRunCount(sequence.ColorTrack.Samples); + int colorSyncSampleCount = GetSequenceSyncSampleCount(sequence.ColorTrack.Samples); long colorSampleTableLength = (long)sampleTableFixedLength + (colorRunCount * timingRunLength) - + (sequence.ColorTrack.Samples.Length * sampleSizeAndSyncEntryLength) + + (sequence.ColorTrack.Samples.Length * sampleSizeEntryLength) + + (colorSyncSampleCount * syncSampleEntryLength) + colorInformationBoxLength; if (!sequence.IccProfileData.IsEmpty) @@ -436,6 +517,7 @@ internal sealed partial class HeifEncoderCore { HeifSequenceTrackEncoding alphaTrack = sequence.AlphaTrack.GetValueOrDefault(); int alphaRunCount = GetSequenceTimingRunCount(alphaTrack.Samples); + int alphaSyncSampleCount = GetSequenceSyncSampleCount(alphaTrack.Samples); int auxiliaryTypeBoxLength = FullBoxHeaderLength + Encoding.UTF8.GetByteCount(HeifConstants.AlphaAuxiliaryType) @@ -444,7 +526,8 @@ internal sealed partial class HeifEncoderCore long alphaSampleTableLength = (long)sampleTableFixedLength + (alphaRunCount * timingRunLength) - + (alphaTrack.Samples.Length * sampleSizeAndSyncEntryLength) + + (alphaTrack.Samples.Length * sampleSizeEntryLength) + + (alphaSyncSampleCount * syncSampleEntryLength) + auxiliaryTypeBoxLength; alphaTrackLength = @@ -774,13 +857,19 @@ internal sealed partial class HeifEncoderCore WriteSequenceUInt64(memory, ref offset, 0); EndSequenceBox(memory, chunkOffsetsStart, offset); - // The current bounded sequence encoder emits independent all-intra pictures; every sample is seekable. + int syncSampleCount = GetSequenceSyncSampleCount(track.Samples); int syncSamplesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stss); WriteSequenceFullBoxHeader(memory, ref offset, 0, 0); - WriteSequenceUInt32(memory, ref offset, (uint)track.Samples.Length); - for (uint sampleIndex = 1; sampleIndex <= track.Samples.Length; sampleIndex++) + WriteSequenceUInt32(memory, ref offset, (uint)syncSampleCount); + uint sampleNumber = 1; + foreach (HeifSequenceSampleInfo sample in track.Samples) { - WriteSequenceUInt32(memory, ref offset, sampleIndex); + if (sample.IsSyncSample) + { + WriteSequenceUInt32(memory, ref offset, sampleNumber); + } + + sampleNumber++; } EndSequenceBox(memory, syncSamplesStart, offset); @@ -839,9 +928,15 @@ internal sealed partial class HeifEncoderCore int codingConstraintsStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Ccst); WriteSequenceFullBoxHeader(memory, ref offset, 0, 0); - // Every emitted sequence sample is independently decodable, while intra prediction remains available inside - // each picture. No inter-picture reference slot is therefore advertised. - WriteSequenceUInt32(memory, ref offset, AllReferencePicturesIntraMask | IntraPicturePredictionUsedMask); + uint codingConstraints = IntraPicturePredictionUsedMask; + if (GetSequenceSyncSampleCount(track.Samples) == track.Samples.Length) + { + codingConstraints |= AllReferencePicturesIntraMask; + } + + // Sync samples are key frames in this encoder. The all-intra flag is therefore valid when every sample + // is independently decodable, including lossless sequences that deliberately avoid inter transforms. + WriteSequenceUInt32(memory, ref offset, codingConstraints); EndSequenceBox(memory, codingConstraintsStart, offset); EndSequenceBox(memory, sampleEntryStart, offset); EndSequenceBox(memory, descriptionStart, offset); @@ -892,6 +987,20 @@ internal sealed partial class HeifEncoderCore return runCount; } + private static int GetSequenceSyncSampleCount(ReadOnlySpan samples) + { + int count = 0; + foreach (HeifSequenceSampleInfo sample in samples) + { + if (sample.IsSyncSample) + { + count++; + } + } + + return count; + } + private static uint GetSequenceSampleDuration(Rational delay, uint timescale) { // HEIF metadata uses either a zero numerator or a zero denominator for an unspecified duration. @@ -905,12 +1014,13 @@ internal sealed partial class HeifEncoderCore return checked((uint)Math.Max(1UL, scaledDuration / delay.Denominator)); } - private static uint GetSequenceTimescale(Image image) + private static uint GetSequenceTimescale(Image image, int firstFrameIndex) where TPixel : unmanaged, IPixel { uint timescale = DefaultSequenceTimescale; - foreach (ImageFrame frame in image.Frames) + for (int frameIndex = firstFrameIndex; frameIndex < image.Frames.Count; frameIndex++) { + ImageFrame frame = image.Frames[frameIndex]; Rational delay = frame.Metadata.GetHeifMetadata().FrameDelay; if (delay.Numerator == 0 || delay.Denominator == 0) { @@ -1070,11 +1180,12 @@ internal sealed partial class HeifEncoderCore private readonly struct HeifSequenceSampleInfo { - public HeifSequenceSampleInfo(long offset, int length, uint duration) + public HeifSequenceSampleInfo(long offset, int length, uint duration, bool isSyncSample) { this.Offset = offset; this.Length = length; this.Duration = duration; + this.IsSyncSample = isSyncSample; } public long Offset { get; } @@ -1082,6 +1193,8 @@ internal sealed partial class HeifEncoderCore public int Length { get; } public uint Duration { get; } + + public bool IsSyncSample { get; } } private readonly struct HeifSequenceEncoding diff --git a/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs b/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs index bc41d6ed34..8a1e875eed 100644 --- a/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifEncoderCore.cs @@ -10,7 +10,6 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Jpeg; using SixLabors.ImageSharp.IO; -using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.Metadata.Profiles.Icc; @@ -46,6 +45,32 @@ internal sealed partial class HeifEncoderCore private const int MaximumCompactPropertyIndex = 0x7F; private const ushort EssentialPropertyFlag = 0x8000; private const byte CompactEssentialPropertyFlag = 0x80; + private const uint HiddenImageItemFlag = 1; + + /// + /// The version defined for the AVIF grid item payload. + /// + private const byte GridDescriptorVersion = 0; + + /// + /// The largest row or column count representable by a grid descriptor. + /// + private const int MaximumGridAxisCellCount = byte.MaxValue + 1; + + /// + /// The minimum width and height permitted for the first cell of an AVIF grid. + /// + private const int MinimumGridCellDimension = 64; + + /// + /// The grid descriptor length when output dimensions use 32-bit fields. + /// + private const int LongGridDescriptorLength = 12; + + /// + /// Selects 32-bit output dimensions in a grid descriptor. + /// + private const byte LargeGridDimensionsFlag = 1; /// /// The global configuration. @@ -81,24 +106,93 @@ internal sealed partial class HeifEncoderCore Guard.NotNull(image, nameof(image)); Guard.NotNull(stream, nameof(stream)); + switch (this.encoder.CompressionMethod) + { + case HeifCompressionMethod.LegacyJpeg: + break; + case HeifCompressionMethod.Av1: + if (image.Frames.Count > 1) + { + if (image.Width > ushort.MaxValue || image.Height > ushort.MaxValue) + { + throw new NotSupportedException("AV1 image-sequence dimensions cannot exceed 65535 pixels."); + } + } + + break; + default: + throw new NotSupportedException($"HEIF compression method '{this.encoder.CompressionMethod}' is not supported."); + } + using ChunkedMemoryStream compressedPixels = new(this.configuration.MemoryAllocator); if (this.encoder.CompressionMethod == HeifCompressionMethod.Av1 && image.Frames.Count > 1) { Av1EncodingSettings settings = this.ResolveAv1Encoding(image); - int sampleCount = image.Frames.Count * (settings.HasAlpha ? 2 : 1); + bool animateRootFrame = this.encoder.AnimateRootFrame + ?? image.Metadata.GetHeifMetadata().AnimateRootFrame; + + int firstFrameIndex = animateRootFrame ? 0 : 1; + int sequenceFrameCount = image.Frames.Count - firstFrameIndex; + int sampleCount = sequenceFrameCount * (settings.HasAlpha ? 2 : 1); using IMemoryOwner samplesOwner = this.configuration.MemoryAllocator.Allocate(sampleCount); + List sequenceItems = new(); + List sequenceLinks = new(); + if (!animateRootFrame) + { + Av1ImageItemEncoding primaryImage = this.CompressAv1ImageItem( + image.Frames.RootFrame, + compressedPixels, + settings, + cancellationToken); + + this.WriteAv1ImageItems( + image, + compressedPixels, + settings, + primaryImage, + sequenceItems, + sequenceLinks); + } + Memory samples = samplesOwner.Memory[..sampleCount]; HeifSequenceEncoding sequence = this.CompressAv1Sequence( image, compressedPixels, settings, samples, + firstFrameIndex, cancellationToken); + if (animateRootFrame) + { + HeifSequenceSampleInfo colorSample = sequence.ColorTrack.Samples[0]; + HeifSequenceTrackEncoding? alphaTrack = sequence.AlphaTrack; + Av1ImageItemEncoding primaryImage = new( + sequence.ColorTrack.Configuration, + colorSample.Offset, + colorSample.Length, + alphaTrack?.Configuration, + alphaTrack?.Samples[0].Offset ?? 0, + alphaTrack?.Samples[0].Length ?? 0); + + // The primary image item and the first track sample describe the same sync sample. Sharing its + // extent matches libavif and avoids encoding or storing the root frame twice. + this.WriteAv1ImageItems( + image, + compressedPixels, + settings, + primaryImage, + sequenceItems, + sequenceLinks); + } + int fileTypeLength = this.WriteSequenceFileTypeBox(stream); - this.WriteSequenceMovieBox(sequence, fileTypeLength, stream); + int metadataLength = GetMetadataBoxLength(sequenceItems, sequenceLinks); + int movieLength = GetSequenceMovieBoxLength(sequence); + this.WriteMetadataBox(sequenceItems, sequenceLinks, fileTypeLength, movieLength, stream); + this.WriteSequenceMovieBox(sequence, fileTypeLength + metadataLength, stream); this.WriteMediaDataBox(compressedPixels, stream); stream.Flush(); return; @@ -115,13 +209,11 @@ internal sealed partial class HeifEncoderCore case HeifCompressionMethod.Av1: this.CompressAv1Pixels(image, compressedPixels, items, links, cancellationToken); break; - default: - throw new NotSupportedException($"HEIF compression method '{this.encoder.CompressionMethod}' is not supported."); } // Write out the generated header and pixels. long metadataBoxOffset = this.WriteFileTypeBox(stream); - this.WriteMetadataBox(items, links, metadataBoxOffset, stream); + this.WriteMetadataBox(items, links, metadataBoxOffset, 0, stream); this.WriteMediaDataBox(compressedPixels, stream); stream.Flush(); } @@ -229,8 +321,14 @@ internal sealed partial class HeifEncoderCore /// The declared image and metadata items. /// The typed relationships between items. /// The metadata box offset from the start of the encoded file. + /// The number of bytes between this box and the media-data box. /// The destination stream positioned after the file-type box. - private void WriteMetadataBox(List items, List links, long metadataBoxOffset, Stream stream) + private void WriteMetadataBox( + List items, + List links, + long metadataBoxOffset, + int followingBoxLength, + Stream stream) { int metadataLength = GetMetadataBoxLength(items, links); using IMemoryOwner metadataOwner = this.configuration.MemoryAllocator.Allocate(metadataLength); @@ -254,7 +352,7 @@ internal sealed partial class HeifEncoderCore bytesWritten += WriteItemLocationBox(memory, bytesWritten, items, 0); // The mdat payload immediately follows the completed meta box and its own eight-byte header. - long mediaDataOffset = checked(metadataBoxOffset + bytesWritten + BasicBoxHeaderLength); + long mediaDataOffset = checked(metadataBoxOffset + bytesWritten + followingBoxLength + BasicBoxHeaderLength); WriteItemLocationBox(memory, itemLocationOffset, items, mediaDataOffset); buffer = memory[..bytesWritten]; @@ -315,13 +413,21 @@ internal sealed partial class HeifEncoderCore { long propertyCount = 0; long associationItemCount = 0; + long associationPropertyCount = 0; long propertyBytes = 0; foreach (HeifItem item in items) { - int itemPropertyCount = GetPropertyCount(item); - propertyCount += itemPropertyCount; + HeifItem propertyItem = item.PropertySource ?? item; + int itemPropertyCount = GetPropertyCount(propertyItem); associationItemCount += itemPropertyCount == 0 ? 0 : 1; + associationPropertyCount += itemPropertyCount; + + if (item.PropertySource is not null) + { + continue; + } + propertyCount += itemPropertyCount; propertyBytes += item.Extent == default ? 0 : SpatialExtentPropertyBoxLength; if (item.ChannelBitDepths is not null) { @@ -348,7 +454,7 @@ internal sealed partial class HeifEncoderCore long length = ItemPropertiesBoxFixedLength + propertyBytes + (associationItemCount * PropertyAssociationEntryFixedLength) - + (propertyCount * associationSize); + + (associationPropertyCount * associationSize); return checked((int)length); } @@ -425,7 +531,12 @@ internal sealed partial class HeifEncoderCore foreach (HeifItem item in items) { int itemLengthOffset = bytesWritten; - bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Infe, 2, 0); + bytesWritten += WriteBoxHeader( + buffer[bytesWritten..], + Heif4CharCode.Infe, + 2, + item.IsHidden ? HiddenImageItemFlag : 0); + BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], (ushort)item.Id); bytesWritten += 2; BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], 0); @@ -499,17 +610,26 @@ internal sealed partial class HeifEncoderCore // ipco order defines the one-based property indices written later in ipma. int ipcoLengthOffset = bytesWritten; bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipco); + ushort nextPropertyIndex = 1; foreach (HeifItem item in items) { + if (item.PropertySource is not null) + { + continue; + } + + item.FirstPropertyIndex = nextPropertyIndex; if (item.Extent != default) { bytesWritten += WriteSpatialExtentPropertyBox(memory, memoryOffset + bytesWritten, item); + nextPropertyIndex++; } byte[]? channelBitDepths = item.ChannelBitDepths; if (channelBitDepths is not null) { bytesWritten += WritePixelInformationPropertyBox(memory, memoryOffset + bytesWritten, channelBitDepths); + nextPropertyIndex++; } else { @@ -521,6 +641,8 @@ internal sealed partial class HeifEncoderCore memoryOffset + bytesWritten, item.ChannelCount, uniformChannelBitDepth.Value); + + nextPropertyIndex++; } } @@ -528,39 +650,43 @@ internal sealed partial class HeifEncoderCore if (codecConfiguration is not null) { bytesWritten += WriteAv1CodecConfigurationPropertyBox(memory, memoryOffset + bytesWritten, codecConfiguration); + nextPropertyIndex++; } string? auxiliaryType = item.AuxiliaryType; if (auxiliaryType is not null) { bytesWritten += WriteAuxiliaryTypePropertyBox(memory, memoryOffset + bytesWritten, auxiliaryType); + nextPropertyIndex++; } IccProfile? iccProfile = item.IccProfile; if (iccProfile is not null) { bytesWritten += WriteIccColorInformationPropertyBox(memory, memoryOffset + bytesWritten, item.GetIccProfileDataForWriting()); + nextPropertyIndex++; } CicpProfile? cicpProfile = item.CicpProfile; if (cicpProfile is not null) { bytesWritten += WriteColorInformationPropertyBox(memory, memoryOffset + bytesWritten, cicpProfile); + nextPropertyIndex++; } } BinaryPrimitives.WriteUInt32BigEndian(buffer[ipcoLengthOffset..], (uint)(bytesWritten - ipcoLengthOffset)); - int propertyCount = 0; + int propertyCount = nextPropertyIndex - 1; int associationItemCount = 0; foreach (HeifItem item in items) { - int itemPropertyCount = GetPropertyCount(item); + HeifItem propertyItem = item.PropertySource ?? item; + int itemPropertyCount = GetPropertyCount(propertyItem); if (itemPropertyCount == 0) { continue; } - propertyCount += itemPropertyCount; associationItemCount++; } @@ -571,10 +697,10 @@ internal sealed partial class HeifEncoderCore bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipma, 0, largePropertyIndex ? 1U : 0U); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)associationItemCount); bytesWritten += 4; - ushort propertyIndex = 1; foreach (HeifItem item in items) { - int itemPropertyCount = GetPropertyCount(item); + HeifItem propertyItem = item.PropertySource ?? item; + int itemPropertyCount = GetPropertyCount(propertyItem); if (itemPropertyCount == 0) { continue; @@ -584,32 +710,33 @@ internal sealed partial class HeifEncoderCore bytesWritten += 2; buffer[bytesWritten++] = (byte)itemPropertyCount; - if (item.Extent != default) + ushort propertyIndex = propertyItem.FirstPropertyIndex; + if (propertyItem.Extent != default) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); } - if (item.ChannelBitDepths is not null || item.UniformChannelBitDepth is not null) + if (propertyItem.ChannelBitDepths is not null || propertyItem.UniformChannelBitDepth is not null) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); } - if (item.Av1CodecConfiguration is not null) + if (propertyItem.Av1CodecConfiguration is not null) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, true); } - if (item.AuxiliaryType is not null) + if (propertyItem.AuxiliaryType is not null) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); } - if (item.IccProfile is not null) + if (propertyItem.IccProfile is not null) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); } - if (item.CicpProfile is not null) + if (propertyItem.CicpProfile is not null) { WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); } @@ -918,68 +1045,396 @@ internal sealed partial class HeifEncoderCore CancellationToken cancellationToken) where TPixel : unmanaged, IPixel { - byte[]? exifData = null; - uint tiffHeaderOffset = 0; - byte[]? xmpData = null; + Av1EncodingSettings settings = this.ResolveAv1Encoding(image); + if (image.Width > Av1Constants.MaxFrameDimension || image.Height > Av1Constants.MaxFrameDimension) + { + this.CompressAv1GridPixels(image, stream, settings, items, links, cancellationToken); + return; + } + + Av1ImageItemEncoding encoding = this.CompressAv1ImageItem( + image.Frames.RootFrame, + stream, + settings, + cancellationToken); + + this.WriteAv1ImageItems(image, stream, settings, encoding, items, links); + } + + /// + /// Encodes a still image as independently coded AV1 cells referenced by one derived grid item. + /// + private void CompressAv1GridPixels( + Image image, + ChunkedMemoryStream stream, + Av1EncodingSettings settings, + List items, + List links, + CancellationToken cancellationToken) + where TPixel : unmanaged, IPixel + { + bool isSubsampledX = !settings.ColorConfig.IsMonochrome && settings.ColorConfig.SubSamplingX; + bool isSubsampledY = !settings.ColorConfig.IsMonochrome && settings.ColorConfig.SubSamplingY; + if ((isSubsampledX && (image.Width & 1) != 0) || (isSubsampledY && (image.Height & 1) != 0)) + { + throw new NotSupportedException("AVIF grid output dimensions must be even along each subsampled chroma axis."); + } + + int columns = GetGridCellCount(image.Width, Av1Constants.MaxFrameDimension); + int rows = GetGridCellCount(image.Height, Av1Constants.MaxFrameDimension); + if (columns > MaximumGridAxisCellCount || rows > MaximumGridAxisCellCount) + { + throw new NotSupportedException( + $"AVIF grids support at most {MaximumGridAxisCellCount} columns and rows."); + } + + int cellWidth = GetGridCellSize(image.Width, columns, isSubsampledX); + int cellHeight = GetGridCellSize(image.Height, rows, isSubsampledY); + Size encodedCellSize = new( + Math.Max(cellWidth, MinimumGridCellDimension), + Math.Max(cellHeight, MinimumGridCellDimension)); + + long cellCount = (long)columns * rows; + long itemCount = 1 + cellCount; + if (settings.HasAlpha) + { + itemCount += 1 + cellCount; + } + if (!this.encoder.SkipMetadata) { - exifData = GetExifData(image.Metadata, out tiffHeaderOffset); - byte[]? sourceXmpData = image.Metadata.XmpProfile?.Data; - if (sourceXmpData is not null && sourceXmpData.Length > 0) + itemCount += image.Metadata.ExifProfile is null ? 0 : 1; + itemCount += image.Metadata.XmpProfile is null ? 0 : 1; + } + + if (itemCount > ushort.MaxValue) + { + throw new NotSupportedException( + $"The encoded AVIF grid requires {itemCount} items, but this container supports at most {ushort.MaxValue}."); + } + + byte channelBitDepth = (byte)settings.BitDepth; + long descriptorOffset = stream.Length; + int descriptorLength = WriteGridDescriptor(stream, rows, columns, image.Size); + HeifItem colorGrid = new(Heif4CharCode.Grid, 1) + { + ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3, + UniformChannelBitDepth = channelBitDepth, + BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3), + IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile, + CicpProfile = settings.ColorProfile + }; + + colorGrid.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + descriptorOffset, + descriptorLength)); + + colorGrid.SetExtent(image.Size); + items.Add(colorGrid); + HeifItemLink colorGridLink = new(Heif4CharCode.Dimg, colorGrid.Id); + links.Add(colorGridLink); + HeifItem? colorPropertySource = null; + ImageFrame rootFrame = image.Frames.RootFrame; + for (int row = 0; row < rows; row++) + { + int y = row * cellHeight; + int height = Math.Min(cellHeight, image.Height - y); + for (int column = 0; column < columns; column++) { - xmpData = sourceXmpData; + cancellationToken.ThrowIfCancellationRequested(); + int x = column * cellWidth; + int width = Math.Min(cellWidth, image.Width - x); + Rectangle sourceRectangle = new(x, y, width, height); + long colorOffset = stream.Length; + ObuSequenceHeader colorHeader = Av1FrameEncoder.EncodeGridCell( + this.configuration, + rootFrame, + sourceRectangle, + encodedCellSize, + stream, + settings.ColorConfig, + settings.ColorQIndex, + this.encoder.Effort); + + long colorLength = stream.Length - colorOffset; + HeifItem colorCell = new(Heif4CharCode.Av01, (uint)items.Count + 1) + { + IsHidden = true, + ChannelCount = colorGrid.ChannelCount, + UniformChannelBitDepth = channelBitDepth, + BitsPerPixel = colorGrid.BitsPerPixel, + Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader), + IccProfile = colorGrid.IccProfile, + CicpProfile = settings.ColorProfile + }; + + colorCell.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + colorOffset, + colorLength)); + + colorCell.SetExtent(encodedCellSize); + ShareGridCellProperties(colorCell, ref colorPropertySource); + + items.Add(colorCell); + colorGridLink.DestinationIds.Add(colorCell.Id); } } - Av1EncodingSettings settings = this.ResolveAv1Encoding(image); + if (settings.HasAlpha) + { + descriptorOffset = stream.Length; + descriptorLength = WriteGridDescriptor(stream, rows, columns, image.Size); + HeifItem alphaGrid = new(Heif4CharCode.Grid, (uint)items.Count + 1) + { + ChannelCount = 1, + UniformChannelBitDepth = channelBitDepth, + BitsPerPixel = channelBitDepth, + AuxiliaryType = HeifConstants.AlphaAuxiliaryType + }; + + alphaGrid.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + descriptorOffset, + descriptorLength)); + + alphaGrid.SetExtent(image.Size); + items.Add(alphaGrid); + HeifItemLink alphaGridLink = new(Heif4CharCode.Dimg, alphaGrid.Id); + links.Add(alphaGridLink); + HeifItemLink alphaLink = new(Heif4CharCode.Auxl, alphaGrid.Id); + alphaLink.DestinationIds.Add(colorGrid.Id); + links.Add(alphaLink); + HeifItem? alphaPropertySource = null; + for (int row = 0; row < rows; row++) + { + int y = row * cellHeight; + int height = Math.Min(cellHeight, image.Height - y); + for (int column = 0; column < columns; column++) + { + cancellationToken.ThrowIfCancellationRequested(); + int x = column * cellWidth; + int width = Math.Min(cellWidth, image.Width - x); + Rectangle sourceRectangle = new(x, y, width, height); + long alphaOffset = stream.Length; + ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlphaGridCell( + this.configuration, + rootFrame, + sourceRectangle, + encodedCellSize, + stream, + settings.AlphaConfig, + settings.AlphaQIndex, + this.encoder.Effort); + + long alphaLength = stream.Length - alphaOffset; + HeifItem alphaCell = new(Heif4CharCode.Av01, (uint)items.Count + 1) + { + IsHidden = true, + ChannelCount = 1, + UniformChannelBitDepth = channelBitDepth, + BitsPerPixel = channelBitDepth, + Av1CodecConfiguration = new Av1CodecConfiguration(alphaHeader), + AuxiliaryType = HeifConstants.AlphaAuxiliaryType + }; + + alphaCell.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + alphaOffset, + alphaLength)); + + alphaCell.SetExtent(encodedCellSize); + ShareGridCellProperties(alphaCell, ref alphaPropertySource); + + items.Add(alphaCell); + alphaGridLink.DestinationIds.Add(alphaCell.Id); + } + } + } + + this.WriteMetadataItems(image, stream, colorGrid, items, links); + } + + /// + /// Gets the minimum number of independently coded cells needed along one grid axis. + /// + /// The complete output dimension along the axis. + /// The largest permitted nominal cell dimension. + private static int GetGridCellCount(int dimension, int maximumCellDimension) + => (int)(((long)dimension + maximumCellDimension - 1) / maximumCellDimension); + + /// + /// Gets the nominal cell size while preserving chroma alignment for every non-edge cell. + /// + private static int GetGridCellSize(int dimension, int cellCount, bool isSubsampled) + { + int cellSize = (int)(((long)dimension + cellCount - 1) / cellCount); + if (isSubsampled && (cellSize & 1) != 0) + { + cellSize++; + } + + return cellSize; + } + + /// + /// Writes the fixed grid item payload and returns its exact length. + /// + private static int WriteGridDescriptor(Stream stream, int rows, int columns, Size outputSize) + { + bool usesLargeDimensions = outputSize.Width > ushort.MaxValue || outputSize.Height > ushort.MaxValue; + Span descriptor = stackalloc byte[LongGridDescriptorLength]; + int descriptorLength = 0; + descriptor[descriptorLength++] = GridDescriptorVersion; + descriptor[descriptorLength++] = usesLargeDimensions ? LargeGridDimensionsFlag : (byte)0; + descriptor[descriptorLength++] = (byte)(rows - 1); + descriptor[descriptorLength++] = (byte)(columns - 1); + if (usesLargeDimensions) + { + BinaryPrimitives.WriteUInt32BigEndian(descriptor[descriptorLength..], (uint)outputSize.Width); + descriptorLength += sizeof(uint); + BinaryPrimitives.WriteUInt32BigEndian(descriptor[descriptorLength..], (uint)outputSize.Height); + descriptorLength += sizeof(uint); + } + else + { + BinaryPrimitives.WriteUInt16BigEndian(descriptor[descriptorLength..], (ushort)outputSize.Width); + descriptorLength += sizeof(ushort); + BinaryPrimitives.WriteUInt16BigEndian(descriptor[descriptorLength..], (ushort)outputSize.Height); + descriptorLength += sizeof(ushort); + } + + stream.Write(descriptor[..descriptorLength]); + return descriptorLength; + } + + /// + /// Reuses the common property set emitted for the first cell in one grid plane. + /// + private static void ShareGridCellProperties(HeifItem item, ref HeifItem? source) + { + if (source is null) + { + source = item; + return; + } + + // Every cell in one plane is coded to the same extent and configuration so current AVIF readers can + // share one property set. Only the source rectangle differs for cells clipped by the output canvas. + item.PropertySource = source; + } + + /// + /// Encodes one frame as the color and optional alpha payloads used by a primary AV1 image item. + /// + private Av1ImageItemEncoding CompressAv1ImageItem( + ImageFrame frame, + ChunkedMemoryStream stream, + Av1EncodingSettings settings, + CancellationToken cancellationToken) + where TPixel : unmanaged, IPixel + { cancellationToken.ThrowIfCancellationRequested(); + long colorOffset = stream.Length; ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode( this.configuration, - image.Frames.RootFrame, + frame, stream, settings.ColorConfig, settings.ColorQIndex, this.encoder.Effort); - long colorLength = stream.Length; + long colorLength = stream.Length - colorOffset; + Av1CodecConfiguration? alphaConfiguration = null; + long alphaOffset = 0; + long alphaLength = 0; + + if (settings.HasAlpha) + { + cancellationToken.ThrowIfCancellationRequested(); + alphaOffset = stream.Length; + ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha( + this.configuration, + frame, + stream, + settings.AlphaConfig, + settings.AlphaQIndex, + this.encoder.Effort); + + alphaLength = stream.Length - alphaOffset; + alphaConfiguration = new Av1CodecConfiguration(alphaHeader); + } + + return new Av1ImageItemEncoding( + new Av1CodecConfiguration(colorHeader), + colorOffset, + colorLength, + alphaConfiguration, + alphaOffset, + alphaLength); + } + + /// + /// Declares a primary AV1 image item over existing payload extents and appends its associated metadata payloads. + /// + private void WriteAv1ImageItems( + Image image, + ChunkedMemoryStream stream, + Av1EncodingSettings settings, + Av1ImageItemEncoding encoding, + List items, + List links) + where TPixel : unmanaged, IPixel + { byte channelBitDepth = (byte)settings.BitDepth; HeifItem colorItem = new(Heif4CharCode.Av01, 1) { ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3, UniformChannelBitDepth = channelBitDepth, BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3), - Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader), + Av1CodecConfiguration = encoding.ColorConfiguration, IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile, CicpProfile = settings.ColorProfile }; - colorItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, 0L, colorLength)); + colorItem.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + encoding.ColorOffset, + encoding.ColorLength)); + colorItem.SetExtent(image.Size); items.Add(colorItem); - if (settings.HasAlpha) + Av1CodecConfiguration? alphaConfiguration = encoding.AlphaConfiguration; + if (alphaConfiguration is not null) { - cancellationToken.ThrowIfCancellationRequested(); - long alphaOffset = stream.Length; - ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha( - this.configuration, - image.Frames.RootFrame, - stream, - settings.AlphaConfig, - settings.AlphaQIndex, - this.encoder.Effort); - - long alphaLength = stream.Length - alphaOffset; HeifItem alphaItem = new(Heif4CharCode.Av01, 2) { ChannelCount = 1, UniformChannelBitDepth = channelBitDepth, BitsPerPixel = channelBitDepth, - Av1CodecConfiguration = new Av1CodecConfiguration(alphaHeader), + Av1CodecConfiguration = alphaConfiguration, AuxiliaryType = HeifConstants.AlphaAuxiliaryType }; - alphaItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, alphaOffset, alphaLength)); + alphaItem.DataLocations.Add( + new HeifLocation( + HeifLocationOffsetOrigin.FileOffset, + 0L, + encoding.AlphaOffset, + encoding.AlphaLength)); + alphaItem.SetExtent(image.Size); items.Add(alphaItem); HeifItemLink alphaLink = new(Heif4CharCode.Auxl, alphaItem.Id); @@ -987,15 +1442,30 @@ internal sealed partial class HeifEncoderCore links.Add(alphaLink); } + this.WriteMetadataItems(image, stream, colorItem, items, links); + } + + /// + /// Appends Exif and XMP payload items associated with the primary presentation item. + /// + private void WriteMetadataItems( + Image image, + ChunkedMemoryStream stream, + HeifItem primaryItem, + List items, + List links) + where TPixel : unmanaged, IPixel + { if (this.encoder.SkipMetadata) { return; } + byte[]? exifData = GetExifData(image.Metadata, out uint tiffHeaderOffset); if (exifData is not null) { long exifOffset = stream.Length; - Span offsetBuffer = stackalloc byte[4]; + Span offsetBuffer = stackalloc byte[sizeof(uint)]; BinaryPrimitives.WriteUInt32BigEndian(offsetBuffer, tiffHeaderOffset); stream.Write(offsetBuffer); stream.Write(exifData); @@ -1010,15 +1480,16 @@ internal sealed partial class HeifEncoderCore HeifLocationOffsetOrigin.FileOffset, 0L, exifOffset, - 4L + exifData.Length)); + sizeof(uint) + (long)exifData.Length)); items.Add(exifItem); HeifItemLink exifLink = new(Heif4CharCode.Cdsc, exifItem.Id); - exifLink.DestinationIds.Add(colorItem.Id); + exifLink.DestinationIds.Add(primaryItem.Id); links.Add(exifLink); } - if (xmpData is not null) + byte[]? xmpData = image.Metadata.XmpProfile?.Data; + if (xmpData is not null && xmpData.Length > 0) { long xmpOffset = stream.Length; stream.Write(xmpData); @@ -1037,7 +1508,7 @@ internal sealed partial class HeifEncoderCore items.Add(xmpItem); HeifItemLink xmpLink = new(Heif4CharCode.Cdsc, xmpItem.Id); - xmpLink.DestinationIds.Add(colorItem.Id); + xmpLink.DestinationIds.Add(primaryItem.Id); links.Add(xmpLink); } } @@ -1093,16 +1564,6 @@ internal sealed partial class HeifEncoderCore CancellationToken cancellationToken) where TPixel : unmanaged, IPixel { - if (this.encoder.Lossless) - { - throw new NotSupportedException("Legacy JPEG image items do not support lossless encoding."); - } - - if (this.encoder.BitDepth is not null and not HeifBitDepth.Bit8) - { - throw new NotSupportedException("Legacy JPEG image items support only 8-bit component encoding."); - } - JpegColorType colorType = this.encoder.ChromaSubsampling switch { null or HeifChromaSubsampling.Yuv420 => JpegColorType.YCbCrRatio420, @@ -1125,4 +1586,38 @@ internal sealed partial class HeifEncoderCore // cannot return while its pooled item payload is still being produced. image.SaveAsJpegAsync(stream, encoder, cancellationToken).GetAwaiter().GetResult(); } + + /// + /// Describes the already-written color and optional alpha extents backing one AV1 image item. + /// + private readonly struct Av1ImageItemEncoding + { + public Av1ImageItemEncoding( + Av1CodecConfiguration colorConfiguration, + long colorOffset, + long colorLength, + Av1CodecConfiguration? alphaConfiguration, + long alphaOffset, + long alphaLength) + { + this.ColorConfiguration = colorConfiguration; + this.ColorOffset = colorOffset; + this.ColorLength = colorLength; + this.AlphaConfiguration = alphaConfiguration; + this.AlphaOffset = alphaOffset; + this.AlphaLength = alphaLength; + } + + public Av1CodecConfiguration ColorConfiguration { get; } + + public long ColorOffset { get; } + + public long ColorLength { get; } + + public Av1CodecConfiguration? AlphaConfiguration { get; } + + public long AlphaOffset { get; } + + public long AlphaLength { get; } + } } diff --git a/src/ImageSharp/Formats/Heif/HeifItem.cs b/src/ImageSharp/Formats/Heif/HeifItem.cs index 2c8d26de19..e1c1ec4c3a 100644 --- a/src/ImageSharp/Formats/Heif/HeifItem.cs +++ b/src/ImageSharp/Formats/Heif/HeifItem.cs @@ -28,6 +28,21 @@ internal sealed class HeifItem(Heif4CharCode type, uint id) /// public Heif4CharCode Type { get; } = type; + /// + /// Gets or sets a value indicating whether this item is excluded from primary-item discovery. + /// + public bool IsHidden { get; set; } + + /// + /// Gets or sets an earlier item whose identical property associations are reused by this item. + /// + public HeifItem? PropertySource { get; set; } + + /// + /// Gets or sets the first one-based property index assigned while writing the property container. + /// + public ushort FirstPropertyIndex { get; set; } + /// /// Gets or sets the name of this item. /// diff --git a/src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs b/src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs index 86ba2f0f9a..d4999220f1 100644 --- a/src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs +++ b/src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs @@ -60,15 +60,23 @@ internal class FlipProcessor : ImageProcessor /// protected override void OnFrameApply(ImageFrame source) + => Apply(this.definition.FlipMode, source, this.Configuration); + + /// + /// Applies an exact axis-aligned reflection to an existing frame. + /// + /// The reflection direction. + /// The frame modified in place. + /// The configuration controlling row parallelism and scratch allocation. + internal static void Apply(FlipMode flipMode, ImageFrame source, Configuration configuration) { - switch (this.definition.FlipMode) + switch (flipMode) { - // No default needed as we have already set the pixels. case FlipMode.Vertical: - FlipX(source.PixelBuffer, this.Configuration); + FlipX(source.PixelBuffer, configuration); break; case FlipMode.Horizontal: - FlipY(source, this.Configuration); + FlipY(source, configuration); break; } } diff --git a/src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs b/src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs index e9d0ecf57d..ed376872a6 100644 --- a/src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs +++ b/src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs @@ -103,25 +103,52 @@ internal class RotateProcessor : AffineTransformProcessor if (MathF.Abs(degrees - 90) < Constants.Epsilon) { - Rotate90(source, destination, configuration); + ApplyQuarterTurn(RotateMode.Rotate90, source, destination, configuration); return true; } if (MathF.Abs(degrees - 180) < Constants.Epsilon) { - Rotate180(source, destination, configuration); + ApplyQuarterTurn(RotateMode.Rotate180, source, destination, configuration); return true; } if (MathF.Abs(degrees - 270) < Constants.Epsilon) { - Rotate270(source, destination, configuration); + ApplyQuarterTurn(RotateMode.Rotate270, source, destination, configuration); return true; } return false; } + /// + /// Applies an exact quarter-turn rotation between already allocated frames. + /// + /// The clockwise quarter-turn rotation. + /// The source frame. + /// The destination frame with the rotated dimensions. + /// The configuration controlling row parallelism. + internal static void ApplyQuarterTurn( + RotateMode rotation, + ImageFrame source, + ImageFrame destination, + Configuration configuration) + { + switch (rotation) + { + case RotateMode.Rotate90: + Rotate90(source, destination, configuration); + break; + case RotateMode.Rotate180: + Rotate180(source, destination, configuration); + break; + case RotateMode.Rotate270: + Rotate270(source, destination, configuration); + break; + } + } + /// /// Rotates the image 180 degrees clockwise at the centre point. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs index cc8aa52584..78d50182a7 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs @@ -204,6 +204,26 @@ public class Av1BitStreamTests Assert.Equal(values, actuals); } + [Fact] + public void SignedReferenceSubexponentialMatchesFiniteRecentering() + { + const int ValueMagnitude = 5; + const int GroupBitCount = 3; + byte[] buffer = new byte[2]; + Av1BitStreamWriter writer = new(buffer); + writer.WriteSignedReferenceSubexponential(-4, ValueMagnitude, GroupBitCount, -4); + writer.WriteSignedReferenceSubexponential(4, ValueMagnitude, GroupBitCount, -4); + writer.WriteSignedReferenceSubexponential(0, ValueMagnitude, GroupBitCount, 0); + writer.Flush(); + + Assert.Equal([0x1e, 0x00], buffer); + + Av1BitStreamReader reader = new(buffer); + Assert.Equal(-4, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, -4)); + Assert.Equal(4, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, -4)); + Assert.Equal(0, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, 0)); + } + [Theory] [InlineData(3)] [InlineData(4)] diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs index 9802a563a1..9af860135f 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs @@ -5,8 +5,10 @@ using System.Buffers; using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; @@ -133,14 +135,16 @@ public class Av1CoefficientsEntropyTests TilesInfo = new ObuTileGroupHeader() }, FrameHeader = new ObuFrameHeader(), - PreviousQIndex = [] + PreviousQIndex = Memory.Empty }, SegmentationNeighborMap = Memory.Empty, ModeInfoGrid = grid, ModeInfoAllocation = allocation, ModeInfoStride = 4, Disallow4x4AllFrames = disallow4x4, - CdefPreset = [] + CdefPreset = Memory.Empty, + TileDataOffsets = Memory.Empty, + TileDataLengths = Memory.Empty }; Point position = new(column, row); @@ -218,6 +222,137 @@ public class Av1CoefficientsEntropyTests Assert.Equal(8, Unsafe.SizeOf()); } + /// + /// Verifies that segment, reference, filter, and flag updates preserve adjacent packed values. + /// + [Fact] + public void EncoderBlockModeInfoPackedFieldsRemainIndependent() + { + for (int segment = 0; segment < 8; segment++) + { + for (int reference = 0; reference < 8; reference++) + { + for (int vertical = 0; vertical < 4; vertical++) + { + for (int horizontal = 0; horizontal < 4; horizontal++) + { + Av1EncoderBlockModeInfo modeInfo = new() + { + Skip = true, + SkipMode = true, + UseIntraBlockCopy = true, + SegmentId = segment, + ReferenceFrame = (Av1ReferenceFrameType)reference, + VerticalInterpolationFilter = (Av1InterpolationFilter)vertical, + HorizontalInterpolationFilter = (Av1InterpolationFilter)horizontal + }; + + Assert.Equal(segment, modeInfo.SegmentId); + Assert.Equal((Av1ReferenceFrameType)reference, modeInfo.ReferenceFrame); + Assert.Equal((Av1InterpolationFilter)vertical, modeInfo.VerticalInterpolationFilter); + Assert.Equal((Av1InterpolationFilter)horizontal, modeInfo.HorizontalInterpolationFilter); + Assert.True(modeInfo.Skip); + Assert.True(modeInfo.SkipMode); + Assert.True(modeInfo.UseIntraBlockCopy); + + // Overwrite every bit in each shared region after the adjacent value has been populated. + modeInfo.SegmentId = segment ^ 7; + Assert.Equal((Av1ReferenceFrameType)reference, modeInfo.ReferenceFrame); + modeInfo.ReferenceFrame = (Av1ReferenceFrameType)(reference ^ 7); + Assert.Equal(segment ^ 7, modeInfo.SegmentId); + modeInfo.Skip = false; + modeInfo.SkipMode = false; + modeInfo.UseIntraBlockCopy = false; + Assert.Equal((Av1InterpolationFilter)vertical, modeInfo.VerticalInterpolationFilter); + Assert.Equal((Av1InterpolationFilter)horizontal, modeInfo.HorizontalInterpolationFilter); + modeInfo.VerticalInterpolationFilter = (Av1InterpolationFilter)(vertical ^ 3); + modeInfo.HorizontalInterpolationFilter = (Av1InterpolationFilter)(horizontal ^ 3); + Assert.False(modeInfo.Skip); + Assert.False(modeInfo.SkipMode); + Assert.False(modeInfo.UseIntraBlockCopy); + } + } + } + } + } + + /// + /// Verifies both filter directions for matching references, mismatches, and unavailable tile neighbors. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void EncoderInterpolationContextUsesTileNeighbors(bool aboveAvailable, bool leftAvailable) + { + // Rows are the above filter and columns the left filter; index three means a nonmatching reference. + ReadOnlySpan expectedContexts = [0, 3, 3, 0, 3, 1, 3, 1, 3, 3, 2, 2, 0, 1, 2, 3]; + Av1MacroBlockModeInfo[] allocation = new Av1MacroBlockModeInfo[3]; + int[] grid = new int[9]; + grid[1] = 0; + grid[3] = 1; + grid[4] = 2; + Av1MacroBlockD macroBlock = CreateMacroBlock(); + macroBlock.ModeInfoStride = 3; + macroBlock.IsUpAvailable = aboveAvailable; + macroBlock.IsLeftAvailable = leftAvailable; + macroBlock.SetModeInfoGrid(grid, allocation, 4); + Av1EncoderBlockModeInfo current = new() { ReferenceFrame = Av1ReferenceFrameType.Last }; + for (int above = 0; above < 4; above++) + { + for (int left = 0; left < 4; left++) + { + allocation[0].Block.ReferenceFrame = above == 3 ? Av1ReferenceFrameType.Golden : Av1ReferenceFrameType.Last; + allocation[0].Block.VerticalInterpolationFilter = (Av1InterpolationFilter)(above % 3); + allocation[0].Block.HorizontalInterpolationFilter = (Av1InterpolationFilter)((above + 1) % 3); + allocation[1].Block.ReferenceFrame = left == 3 ? Av1ReferenceFrameType.Golden : Av1ReferenceFrameType.Last; + allocation[1].Block.VerticalInterpolationFilter = (Av1InterpolationFilter)(left % 3); + allocation[1].Block.HorizontalInterpolationFilter = (Av1InterpolationFilter)((left + 1) % 3); + int aboveVertical = aboveAvailable ? above : 3; + int leftVertical = leftAvailable ? left : 3; + int aboveHorizontal = aboveVertical == 3 ? 3 : (above + 1) % 3; + int leftHorizontal = leftVertical == 3 ? 3 : (left + 1) % 3; + + Assert.Equal(expectedContexts[(aboveVertical * 4) + leftVertical], Av1SymbolContextHelper.GetSwitchableInterpolationContext(current, macroBlock, 0)); + Assert.Equal(8 + expectedContexts[(aboveHorizontal * 4) + leftHorizontal], Av1SymbolContextHelper.GetSwitchableInterpolationContext(current, macroBlock, 1)); + } + } + } + + /// + /// Verifies which inter modes signal filters and distinguishes residual skip from compound skip mode. + /// + [Theory] + [InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.GlobalMotionVector, false)] + [InlineData(Av1GlobalMotionType.Translation, Av1PredictionMode.GlobalMotionVector, true)] + [InlineData(Av1GlobalMotionType.RotationZoom, Av1PredictionMode.GlobalMotionVector, false)] + [InlineData(Av1GlobalMotionType.Affine, Av1PredictionMode.GlobalMotionVector, false)] + [InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.NewMotionVector, true)] + [InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.NearestMotionVector, true)] + public void EncoderInterpolationSyntaxMatchesModeEligibility(int globalType, int predictionMode, bool expected) + { + ObuFrameHeader frameHeader = new() { InterpolationFilter = Av1InterpolationFilter.Switchable }; + frameHeader.GetGlobalMotionParameters()[0].Type = (Av1GlobalMotionType)globalType; + Av1EncoderBlockModeInfo modeInfo = new() + { + BlockSize = Av1BlockSize.Block8x8, + ReferenceFrame = Av1ReferenceFrameType.Last, + Mode = (Av1PredictionMode)predictionMode + }; + + Assert.Equal(expected, Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo)); + modeInfo.Skip = true; + Assert.Equal(expected, Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo)); + modeInfo.BlockSize = Av1BlockSize.Block4x8; + Assert.True(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo)); + modeInfo.SkipMode = true; + Assert.False(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo)); + modeInfo.SkipMode = false; + frameHeader.InterpolationFilter = Av1InterpolationFilter.Regular; + Assert.False(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo)); + } + [Fact] public void EncoderSuperblockWorkspaceUsesOneExactSizeOwner() { @@ -231,9 +366,9 @@ public class Av1CoefficientsEntropyTests { allocation = Assert.Single(allocator.AllocationLog); Assert.Empty(allocator.ReturnLog); - Assert.Equal(typeof(Av1EncoderBlockStruct), allocation.ElementType); + Assert.Equal(typeof(byte), allocation.ElementType); Assert.Equal(AllocationOptions.None, allocation.AllocationOptions); - Assert.Equal(Av1EncoderSuperblockWorkspace.StorageLength, allocation.Length); + Assert.Equal(Av1EncoderSuperblockWorkspace.StorageByteLength, allocation.Length); Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumFinalBlockCount, workspace.FinalBlocks.Length); Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumPartitionCount, workspace.PartitionTypes.Length); Assert.Equal(Av1EncoderBlockStruct.StorageSize, Unsafe.SizeOf()); @@ -295,39 +430,34 @@ public class Av1CoefficientsEntropyTests } [Fact] - public void EncoderPaletteMapsUseOneLazyExactSizeOwner() + public void EncoderSuperblockWorkspaceExposesReusablePaletteMaps() { TestMemoryAllocator allocator = new(); allocator.EnableNonThreadSafeLogging(); Configuration configuration = Configuration.Default.Clone(); configuration.MemoryAllocator = allocator; - TestMemoryAllocator.AllocationRequest[] allocations; + TestMemoryAllocator.AllocationRequest allocation; using (Av1EncoderSuperblockWorkspace workspace = new(configuration)) { - Assert.Single(allocator.AllocationLog); + allocation = Assert.Single(allocator.AllocationLog); Assert.Empty(allocator.ReturnLog); Av1EncoderPaletteMapBuffer maps = workspace.GetPaletteMaps(); Assert.Same(maps, workspace.GetPaletteMaps()); - allocations = allocator.AllocationLog.ToArray(); - Assert.Equal(2, allocations.Length); - Assert.Equal(typeof(byte), allocations[1].ElementType); - Assert.Equal(Av1EncoderPaletteMapBuffer.StorageLength, allocations[1].Length); - Assert.Equal(AllocationOptions.None, allocations[1].AllocationOptions); + Assert.Single(allocator.AllocationLog); Buffer2DRegion luma = maps.GetMap(Av1PlaneType.Y, 64, 64); Buffer2DRegion chroma = maps.GetMap(Av1PlaneType.Uv, 32, 32); luma.DangerousGetRowSpan(0)[0] = 3; chroma.DangerousGetRowSpan(0)[0] = 5; - Assert.Equal(3, luma.DangerousGetRowSpan(0)[0]); - Assert.Equal(5, chroma.DangerousGetRowSpan(0)[0]); + Av1EncoderPaletteMapBuffer retainedMaps = workspace.GetPaletteMaps(); + Assert.Equal(3, retainedMaps.GetMap(Av1PlaneType.Y, 64, 64).DangerousGetRowSpan(0)[0]); + Assert.Equal(5, retainedMaps.GetMap(Av1PlaneType.Uv, 32, 32).DangerousGetRowSpan(0)[0]); } - Assert.Equal(2, allocator.ReturnLog.Count); - Assert.Equal( - allocations.Select(x => x.AllocationId).Order(), - allocator.ReturnLog.Select(x => x.AllocationId).Order()); + TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog); + Assert.Equal(allocation.AllocationId, returned.AllocationId); } [Fact] @@ -841,7 +971,7 @@ public class Av1CoefficientsEntropyTests skip: false, modeInfoPosition: new Point(20, 4)); - Assert.Equal(new[] { -1, 3, -1, -1 }, picture.CdefPreset[0]); + Assert.True(picture.CdefPreset.Span.SequenceEqual([-1, 3, -1, -1])); } [Fact] @@ -1431,13 +1561,15 @@ public class Av1CoefficientsEntropyTests TilesInfo = tiles }, FrameHeader = frameHeader, - PreviousQIndex = [] + PreviousQIndex = Memory.Empty }, SegmentationNeighborMap = Memory.Empty, ModeInfoGrid = modeInfoGrid, ModeInfoAllocation = modeInfoAllocation, ModeInfoStride = modeInfoColumnCount, - CdefPreset = [[-1, -1, -1, -1]] + CdefPreset = new int[] { -1, -1, -1, -1 }, + TileDataOffsets = Memory.Empty, + TileDataLengths = Memory.Empty }; } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs index 79e49ca68f..44df111e1f 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs @@ -3,9 +3,11 @@ using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats.Heif.Av1; +using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Components; @@ -26,6 +28,42 @@ public class Av1EncoderFrameTests private const int Yuv422 = (int)Av1ColorFormat.Yuv422; private const int Yuv444 = (int)Av1ColorFormat.Yuv444; + [Fact] + public void EncodeUsesMultipleTilesWhenSingleTileWidthLimitIsExceeded() + { + const int Width = Av1Constants.MaxTileWidth + 1; + const int SuperblockSize = 1 << (Av1Constants.MaxSuperBlockSizeLog2 - 1); + const int Height = SuperblockSize; + int superblockColumns = (Width + SuperblockSize - 1) / SuperblockSize; + int secondTileStart = ((superblockColumns + 1) / 2) * SuperblockSize; + using Image source = new(Width, Height, new L8(128)); + source[0, 0] = new L8(1); + source[secondTileStart - 1, 0] = new L8(17); + source[secondTileStart, 0] = new L8(241); + source[Width - 1, Height - 1] = new L8(255); + using MemoryStream stream = new(); + ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv400); + + Av1FrameEncoder.Encode( + Configuration.Default, + source.Frames.RootFrame, + stream, + colorConfig, + qIndex: 0, + effort: 0); + + using Av1Decoder decoder = new(Configuration.Default); + using Image decoded = decoder.Decode(stream.ToArray()); + + Assert.Equal(2, decoder.FrameHeader.TilesInfo.TileColumnCount); + Assert.Equal(1, decoder.FrameHeader.TilesInfo.TileRowCount); + Assert.Equal(source.Size, decoded.Size); + Assert.Equal(source[0, 0], decoded[0, 0]); + Assert.Equal(source[secondTileStart - 1, 0], decoded[secondTileStart - 1, 0]); + Assert.Equal(source[secondTileStart, 0], decoded[secondTileStart, 0]); + Assert.Equal(source[Width - 1, Height - 1], decoded[Width - 1, Height - 1]); + } + [Theory] [InlineData(8, 8, false, EightBit, Yuv400)] [InlineData(8, 8, true, EightBit, Yuv400)] @@ -153,6 +191,250 @@ public class Av1EncoderFrameTests } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public void EncodeSequenceFrameWritesNonReducedHeaderConsumedByProductionDecoder(bool encodeAlpha) + { + const int Width = 16; + const int Height = 16; + using Image source = new(Width, Height, new Rgba32(48, 96, 192)); + using MemoryStream stream = new(); + ObuColorConfig colorConfig = encodeAlpha + ? CreateColorConfig(Av1BitDepth.EightBit) + : CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420); + + using Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha + ? Av1FrameEncoder.CreateAlphaSequenceEncoder( + Configuration.Default, + Width, + Height, + colorConfig, + qIndex: 37, + effort: 5) + : Av1FrameEncoder.CreateColorSequenceEncoder( + Configuration.Default, + Width, + Height, + colorConfig, + qIndex: 37, + effort: 5); + + encoder.EncodeKeyFrame(source.Frames.RootFrame, stream); + ObuSequenceHeader encodedHeader = encoder.SequenceHeader; + byte[] payload = stream.ToArray(); + using Av1Decoder decoder = new(Configuration.Default); + using Image decoded = decoder.Decode(payload); + ObuSequenceHeader decodedHeader = decoder.SequenceHeader; + + Assert.False(encodedHeader.IsStillPicture); + Assert.False(encodedHeader.IsReducedStillPictureHeader); + Assert.Equal(encodeAlpha, encodedHeader.ColorConfig.IsMonochrome); + Assert.NotNull(decodedHeader); + Assert.False(decodedHeader.IsStillPicture); + Assert.False(decodedHeader.IsReducedStillPictureHeader); + Assert.Equal(new Size(Width, Height), decoded.Size); + } + + /// + /// Verifies retained reference reconstruction and effort-dependent filter signaling through production sequence decoding. + /// + [Theory] + [InlineData(5, false, false)] + [InlineData(7, false, false)] + [InlineData(8, true, false)] + [InlineData(9, true, true)] + public void SequenceEncoderUsesRetainedReconstructionForInterFrame(int effort, bool switchableFilters, bool dualFilters) + { + const int Width = 16; + const int Height = 16; + Rgba32 sourceColor = new(48, 96, 192); + using Image source = new(Width, Height, sourceColor); + using MemoryStream firstSample = new(); + using MemoryStream secondSample = new(); + ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420); + using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( + Configuration.Default, + Width, + Height, + colorConfig, + qIndex: 37, + effort); + + encoder.EncodeKeyFrame(source.Frames.RootFrame, firstSample); + encoder.EncodeInterFrame(source.Frames.RootFrame, secondSample); + + // Retain the exact two-sample elementary stream for independent reference-decoder acceptance. + string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.SequenceEncoderUsesRetainedReconstructionForInterFrame)); + using (FileStream output = File.Create(Path.Combine(outputDirectory, $"effort-{effort}.obu"))) + { + firstSample.Position = 0; + firstSample.CopyTo(output); + secondSample.Position = 0; + secondSample.CopyTo(output); + } + + using Av1Decoder decoder = new(Configuration.Default); + using ImageFrame decodedFirst = decoder.DecodeSequenceFrame( + firstSample.ToArray(), + null, + null); + + using ImageFrame decodedSecond = decoder.DecodeSequenceFrame( + secondSample.ToArray(), + null, + null); + + ObuFrameHeader frameHeader = decoder.FrameHeader; + Assert.Equal(ObuFrameType.InterFrame, frameHeader.FrameType); + Assert.False(frameHeader.SegmentationParameters.Enabled); + Assert.False(frameHeader.AllowScreenContentTools); + Assert.False(frameHeader.ForceIntegerMotionVector); + Assert.Equal(effort >= 8, frameHeader.AllowHighPrecisionMotionVector); + Assert.Equal(37, frameHeader.QuantizationParameters.BaseQIndex); + Assert.Equal(switchableFilters ? Av1InterpolationFilter.Switchable : Av1InterpolationFilter.Regular, frameHeader.InterpolationFilter); + Assert.Equal(dualFilters, decoder.SequenceHeader.EnableDualFilter); + for (int y = 0; y < Height; y++) + { + Assert.Equal( + decodedFirst.PixelBuffer.DangerousGetRowSpan(y), + decodedSecond.PixelBuffer.DangerousGetRowSpan(y)); + } + } + + [Fact] + public void SequenceEncoderWritesSelectedGlobalTranslation() + { + const int Width = 64; + const int Height = 64; + const int HorizontalOffset = 4; + using Image first = new(Width, Height); + using Image second = new(Width, Height); + for (int y = 0; y < Height; y++) + { + Span firstRow = first.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < Width; x++) + { + byte value = (byte)(((x * 37) + (y * 53) + ((x * y) * 11)) & byte.MaxValue); + firstRow[x] = new Rgba32(value, value, value); + } + } + + for (int y = 0; y < Height; y++) + { + ReadOnlySpan firstRow = first.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + Span secondRow = second.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + for (int x = 0; x < Width; x++) + { + secondRow[x] = firstRow[Math.Min(x + HorizontalOffset, Width - 1)]; + } + } + + using MemoryStream firstSample = new(); + using MemoryStream secondSample = new(); + ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420); + using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( + Configuration.Default, + Width, + Height, + colorConfig, + qIndex: 4, + effort: 6); + + encoder.EncodeKeyFrame(first.Frames.RootFrame, firstSample); + encoder.EncodeInterFrame(second.Frames.RootFrame, secondSample); + + using Av1Decoder decoder = new(Configuration.Default); + using ImageFrame decodedFirst = decoder.DecodeSequenceFrame( + firstSample.ToArray(), + null, + null); + + using ImageFrame decodedSecond = decoder.DecodeSequenceFrame( + secondSample.ToArray(), + null, + null); + + ObuFrameHeader frameHeader = decoder.FrameHeader; + Av1GlobalMotionParameters globalMotion = frameHeader.GetGlobalMotionParameters()[0]; + Av1MotionVector vector = globalMotion.GetMotionVector( + frameHeader.AllowHighPrecisionMotionVector, + Av1BlockSize.Block8x8, + default, + frameHeader.ForceIntegerMotionVector); + + Assert.Equal(Av1GlobalMotionType.RotationZoom, globalMotion.Type); + Assert.False(frameHeader.AllowScreenContentTools); + Assert.False(frameHeader.ForceIntegerMotionVector); + Assert.Equal(0, vector.Row); + Assert.Equal(HorizontalOffset * 8, vector.Column); + Assert.Equal(first.Size, decodedFirst.Size); + Assert.Equal(second.Size, decodedSecond.Size); + } + + [Theory] + [InlineData(false, EightBit, Yuv420, 384)] + [InlineData(false, TwelveBit, Yuv444, 288)] + [InlineData(true, EightBit, Yuv400, 192)] + [InlineData(true, TwelveBit, Yuv400, 192)] + public void SequenceEncoderReusesAllocatorOwnedRowStorage( + bool encodeAlpha, + int bitDepthValue, + int colorFormatValue, + int expectedRowStorageLength) + { + const int Width = 64; + const int Height = 64; + Av1BitDepth bitDepth = (Av1BitDepth)bitDepthValue; + Av1ColorFormat colorFormat = (Av1ColorFormat)colorFormatValue; + using Image source = new( + Width, + Height, + new Rgba64(ushort.MaxValue, 32768, 16384, 49152)); + + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + ObuColorConfig colorConfig = CreateColorConfig(bitDepth, colorFormat); + TestMemoryAllocator.AllocationRequest rowStorage; + int allocationCount; + using (Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha + ? Av1FrameEncoder.CreateAlphaSequenceEncoder( + configuration, + Width, + Height, + colorConfig, + qIndex: 37, + effort: 6) + : Av1FrameEncoder.CreateColorSequenceEncoder( + configuration, + Width, + Height, + colorConfig, + qIndex: 37, + effort: 6)) + { + rowStorage = Assert.Single( + allocator.AllocationLog, + allocation => allocation.ElementType == typeof(float)); + + allocationCount = allocator.AllocationLog.Count; + using MemoryStream output = new(256 * 1024); + encoder.EncodeKeyFrame(source.Frames.RootFrame, output); + encoder.EncodeInterFrame(source.Frames.RootFrame, output); + + // Fixed sequence geometry lets libaom retain its frame-sized compressor data. The ImageSharp + // sequence encoder must likewise perform every sample conversion and coding pass without another rent. + Assert.Equal(allocationCount, allocator.AllocationLog.Count); + } + + Assert.Equal(expectedRowStorageLength, rowStorage.Length); + Assert.Contains( + allocator.ReturnLog, + returned => returned.AllocationId == rowStorage.AllocationId); + } + [Theory] [InlineData(TenBit)] [InlineData(TwelveBit)] diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs index 3c4d6ce847..726a7c3b80 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs @@ -54,10 +54,13 @@ public class Av1EncoderModeInfoBufferTests public unsafe void PictureBufferPacksAllPictureStateIntoTwoAllocatorOwners( bool allowScreenContentTools, bool allowIntraBlockCopy, - int expectedStateStorageLength) + int expectedContextStorageLength) { const int Width = 16; const int Height = 16; + + // Four CDEF presets, the preceding quantizer, and two payload bounds follow the context regions. + const int TileStateStorageLength = 7 * sizeof(int); TestMemoryAllocator allocator = new(); allocator.EnableNonThreadSafeLogging(); Configuration configuration = Configuration.Default.Clone(); @@ -108,7 +111,7 @@ public class Av1EncoderModeInfoBufferTests Assert.Equal(6_144, allocations[0].Length); Assert.Equal(AllocationOptions.Clean, allocations[0].AllocationOptions); Assert.Equal(typeof(byte), allocations[1].ElementType); - Assert.Equal(expectedStateStorageLength, allocations[1].Length); + Assert.Equal(expectedContextStorageLength + TileStateStorageLength, allocations[1].Length); Assert.Equal(AllocationOptions.Clean, allocations[1].AllocationOptions); Assert.Empty(allocator.ReturnLog); @@ -122,6 +125,29 @@ public class Av1EncoderModeInfoBufferTests Assert.Equal(16, picture.CbDcSignLevelCoefficientNeighbors[0].Top.Length); Assert.Equal(32, picture.TransformFunctionContexts[0].Left.Length); Assert.Equal(32, picture.TransformFunctionContexts[0].Top.Length); + Assert.Equal(4, picture.CdefPreset.Length); + Assert.Equal(1, picture.Parent.PreviousQIndex.Length); + Assert.Equal(1, picture.TileDataOffsets.Length); + Assert.Equal(1, picture.TileDataLengths.Length); + + // Exact offsets prove that all four typed views occupy the trailing region of the same owner, + // without gaps, overlapping fields, or a separate allocation hidden behind a memory manager. + fixed (byte* state = picture.SegmentationNeighborMap.Span) + { + fixed (int* cdef = picture.CdefPreset.Span, + quantizer = picture.Parent.PreviousQIndex.Span, + offsets = picture.TileDataOffsets.Span, + lengths = picture.TileDataLengths.Span) + { + Assert.Equal((nuint)0, (nuint)cdef % (nuint)sizeof(int)); + Assert.Equal(expectedContextStorageLength, (byte*)cdef - state); + Assert.Equal(4, quantizer - cdef); + Assert.Equal(1, offsets - quantizer); + Assert.Equal(1, lengths - offsets); + Assert.Equal(allocations[1].Length, (byte*)(lengths + 1) - state); + } + } + if (allowScreenContentTools) { Av1NeighborArrayUnit paletteContext = Assert.Single(picture.PaletteContexts); @@ -176,6 +202,157 @@ public class Av1EncoderModeInfoBufferTests allocator.ReturnLog.Select(x => x.AllocationId).Order()); } + [Fact] + public void InterPictureBufferExposesPackedMotionVectorStorage() + { + const int Width = 16; + const int Height = 16; + ObuColorConfig colorConfig = new() + { + IsMonochrome = false, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + }; + + ObuTileGroupHeader tiles = new() + { + TileColumnCount = 1, + TileRowCount = 1 + }; + + tiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2; + tiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2; + ObuSequenceHeader sequenceHeader = new() { ColorConfig = colorConfig }; + ObuFrameHeader frameHeader = new() + { + FrameType = ObuFrameType.InterFrame, + ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2, + ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2, + TilesInfo = tiles + }; + + using Av1EncoderPictureBuffer buffer = new( + Configuration.Default, + sequenceHeader, + frameHeader, + Width, + Height, + disallow4x4AllFrames: true); + + Av1PictureControlSet picture = buffer.Picture; + Assert.Equal(256, picture.DisplacementVectors.Length); + Assert.Equal(0, picture.IntraBlockCopySearch.OriginWidth); + + Point position = new(2, 2); + Av1MotionVector vector = new(-32, 40); + picture.MapModeInfoBlock(position, Av1BlockSize.Block8x8); + picture.SetDisplacementVector(position, vector); + Assert.Equal(vector, picture.GetDisplacementVector(new Point(3, 3))); + } + + [Fact] + public void PictureBufferResetReusesStorageAndRestoresFrameState() + { + const int Width = 16; + const int Height = 16; + const int InitialQIndex = 37; + const int NextQIndex = 91; + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + ObuColorConfig colorConfig = new() + { + IsMonochrome = false, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + }; + + ObuTileGroupHeader initialTiles = new() + { + TileColumnCount = 1, + TileRowCount = 1 + }; + + initialTiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2; + initialTiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2; + ObuSequenceHeader sequenceHeader = new() + { + Use128x128Superblock = true, + ColorConfig = colorConfig + }; + + ObuFrameHeader initialFrameHeader = new() + { + FrameType = ObuFrameType.KeyFrame, + ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2, + ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2, + TilesInfo = initialTiles + }; + + initialFrameHeader.QuantizationParameters.BaseQIndex = InitialQIndex; + using Av1EncoderPictureBuffer buffer = new( + configuration, + sequenceHeader, + initialFrameHeader, + Width, + Height, + disallow4x4AllFrames: true, + allocateScreenContentState: true, + allocateMotionVectorState: true, + allocateIntraBlockCopySearch: true); + + Av1PictureControlSet picture = buffer.Picture; + int allocationCount = allocator.AllocationLog.Count; + picture.ModeInfoGrid.Span[0] = 7; + picture.ModeInfoAllocation.Span[0].Block.Mode = Av1PredictionMode.Paeth; + picture.SegmentationNeighborMap.Span[0] = 3; + picture.PartitionContexts[0].Left[0] = new Av1PartitionContext(5, 7); + picture.TransformFunctionContexts[0].Top[0] = 8; + picture.PaletteContexts[0].Left[0].PaletteSizes[0] = 2; + picture.DisplacementVectors.Span[0] = new Av1EncoderDisplacementVector { Row = -8, Column = 16 }; + picture.CdefPreset.Span[0] = 2; + picture.Parent.PreviousQIndex.Span[0] = InitialQIndex + 1; + picture.TileDataOffsets.Span[0] = 11; + picture.TileDataLengths.Span[0] = 13; + ObuTileGroupHeader nextTiles = new() + { + TileColumnCount = 1, + TileRowCount = 1 + }; + + nextTiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2; + nextTiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2; + ObuFrameHeader nextFrameHeader = new() + { + FrameType = ObuFrameType.InterFrame, + ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2, + ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2, + TilesInfo = nextTiles + }; + + nextFrameHeader.QuantizationParameters.BaseQIndex = NextQIndex; + buffer.Reset(nextFrameHeader); + + Assert.Equal(allocationCount, allocator.AllocationLog.Count); + Assert.Empty(allocator.ReturnLog); + Assert.Equal(0, picture.ModeInfoGrid.Span[0]); + Assert.Equal(Av1PredictionMode.DC, picture.ModeInfoAllocation.Span[0].Block.Mode); + Assert.Equal(0, picture.SegmentationNeighborMap.Span[0]); + Assert.Equal(default, picture.PartitionContexts[0].Left[0]); + Assert.Equal(Av1Constants.MaxTransformSize, picture.TransformFunctionContexts[0].Top[0]); + Assert.Equal(0, picture.PaletteContexts[0].Left[0].PaletteSizes[0]); + Assert.Equal(default, picture.DisplacementVectors.Span[0]); + Assert.Equal(-1, picture.CdefPreset.Span[0]); + Assert.Equal(NextQIndex, picture.Parent.PreviousQIndex.Span[0]); + Assert.Equal(0, picture.TileDataOffsets.Span[0]); + Assert.Equal(0, picture.TileDataLengths.Span[0]); + Assert.Same(nextFrameHeader, picture.Parent.FrameHeader); + Assert.Same(nextTiles, picture.Parent.Common.TilesInfo); + } + [Theory] [InlineData(false, 2, 3, 98)] [InlineData(true, 2, 2, 17)] @@ -248,14 +425,16 @@ public class Av1EncoderModeInfoBufferTests TilesInfo = tiles }, FrameHeader = frameHeader, - PreviousQIndex = [] + PreviousQIndex = Memory.Empty }, SegmentationNeighborMap = Memory.Empty, ModeInfoGrid = buffer.Grid, ModeInfoAllocation = buffer.Allocation, ModeInfoStride = buffer.ModeInfoStride, Disallow4x4AllFrames = buffer.Disallow4x4AllFrames, - CdefPreset = [] + CdefPreset = Memory.Empty, + TileDataOffsets = Memory.Empty, + TileDataLengths = Memory.Empty }; } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs index 31b73786f1..c27fa959c3 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs @@ -155,6 +155,27 @@ public class Av1EntropyTests encoder.GetTransformBlockSkipCost(true, TransformSize, SkipContext)); } + [Fact] + public void SymbolEncoderResetRestoresNormativeDistributionState() + { + const byte TopContext = 0; + const byte LeftContext = 0; + const Av1PredictionMode LumaMode = Av1PredictionMode.DC; + using Av1SymbolEncoder encoder = new(Configuration.Default, 64, BaseQIndex, updateCdf: true); + int initialCost = encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext); + + for (int i = 0; i < 8; i++) + { + encoder.WriteLumaMode(LumaMode, TopContext, LeftContext); + } + + Assert.NotEqual(initialCost, encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext)); + + encoder.Reset(); + + Assert.Equal(initialCost, encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext)); + } + [Fact] public void BlockSkipDecisionUsesAdaptedRatesForEmptyTransforms() { @@ -246,7 +267,8 @@ public class Av1EntropyTests macroBlock, Av1BlockSize.Block8x8, Mode, - angleDelta)); + angleDelta, + isIntraFrame: true)); } /// @@ -698,6 +720,48 @@ public class Av1EntropyTests long expected) => Assert.Equal(expected, Av1RateDistortion.GetCost(rateMultiplier, rate, distortion)); + /// + /// Verifies fixed curve samples, interpolation, error categories, native quantizer normalization, and skip selection. + /// + [Theory] + [InlineData(Av1BlockSize.Block4x4, 16L, 16, 8, Av1BitDepth.EightBit, 1, 13243, 17L)] + [InlineData(Av1BlockSize.Block8x8, 64L, 64, 8, Av1BitDepth.EightBit, 1, 45715, 66L)] + [InlineData(Av1BlockSize.Block16x16, 256L, 256, 8, Av1BitDepth.EightBit, 1, 154928, 265L)] + [InlineData(Av1BlockSize.Block32x32, 1024L, 1024, 8, Av1BitDepth.EightBit, 1, 410224, 1061L)] + [InlineData(Av1BlockSize.Block8x8, 96L, 64, 8, Av1BitDepth.EightBit, 1, 53253, 71L)] + [InlineData(Av1BlockSize.Block8x8, 1024L, 64, 8, Av1BitDepth.EightBit, 1, 96672, 95L)] + [InlineData(Av1BlockSize.Block8x8, 1056L, 64, 8, Av1BitDepth.EightBit, 1, 97253, 95L)] + [InlineData(Av1BlockSize.Block8x8, 64L, 64, 32, Av1BitDepth.TenBit, 1, 45715, 66L)] + [InlineData(Av1BlockSize.Block8x8, 64L, 64, 128, Av1BitDepth.TwelveBit, 1, 45715, 66L)] + [InlineData(Av1BlockSize.Block8x8, 64L, 64, 8, Av1BitDepth.EightBit, 1000000, 0, 1024L)] + [InlineData(Av1BlockSize.Block8x8, 0L, 64, 8, Av1BitDepth.EightBit, 1, 0, 0L)] + [InlineData(Av1BlockSize.Block128x128, 1L, 16384, 21387, Av1BitDepth.TwelveBit, 1, 0, 16L)] + public void PredictionErrorModelMatchesReferenceCurveSamples( + int blockSize, + long squaredError, + int sampleCount, + int acQuantizer, + int bitDepth, + int rateMultiplier, + int expectedRate, + long expectedDistortion) + { + // Expectations come from the published curve samples and cubic polynomial, not from an encode/decode + // round trip. Unit normalized error and unit quantizer hit rate column 31 in each block-size category. + Av1RateDistortion.ModelPredictionError( + (Av1BlockSize)blockSize, + squaredError, + sampleCount, + acQuantizer, + (Av1BitDepth)bitDepth, + rateMultiplier, + out int rate, + out long distortion); + + Assert.Equal(expectedRate, rate); + Assert.Equal(expectedDistortion, distortion); + } + [Theory] [InlineData(1, 8191, 100, 100)] [InlineData(1, 8192, 100, 101)] @@ -744,6 +808,19 @@ public class Av1EntropyTests int expected) => Assert.Equal(expected, Av1RateDistortion.GetKeyFrameRateMultiplier(qIndex, (Av1BitDepth)bitDepth)); + [Theory] + [InlineData(0, 0, 51)] + [InlineData(0, 1, 3)] + [InlineData(0, 2, 1)] + [InlineData(255, 0, 9_288_598)] + [InlineData(255, 1, 20_049_918)] + [InlineData(255, 2, 63_036_850)] + public void InterFrameRateMultiplierMatchesCurrentLibaom( + int qIndex, + int bitDepth, + int expected) + => Assert.Equal(expected, Av1RateDistortion.GetInterFrameRateMultiplier(qIndex, (Av1BitDepth)bitDepth)); + [Fact] public void SymbolWriterMatchesCurrentLibaomCarryRegression() { @@ -782,7 +859,7 @@ public class Av1EntropyTests } [Fact] - public void SymbolWriterExposesExistingOutputAllocationWithoutCopy() + public void SymbolWriterResetReusesExistingOutputAllocation() { const int bufferLength = 257; TestMemoryAllocator allocator = new(); @@ -805,14 +882,46 @@ public class Av1EntropyTests Assert.Equal(63, encoded.Span[0]); Assert.Single(allocator.AllocationLog); Assert.Empty(allocator.ReturnLog); + + int firstLength = length; + writer.Reset(firstLength); + writer.WriteBoolean(false, 16_384); + writer.WriteBoolean(false, 16_384); + writer.WriteBoolean(true, 512); + writer.WriteBoolean(false, 8_192); + encoded = writer.Exit(out length); + + Assert.Equal(2, length); + Assert.Equal(length, encoded.Length); + Assert.Equal(63, encoded.Span[0]); + ReadOnlyMemory output = writer.GetOutput(firstLength + length); + Assert.True(output.Span[..firstLength].SequenceEqual(output.Span[firstLength..])); + Assert.Single(allocator.AllocationLog); + Assert.Empty(allocator.ReturnLog); + + writer.Reset(); + writer.WriteBoolean(false, 16_384); + writer.WriteBoolean(false, 16_384); + writer.WriteBoolean(true, 512); + writer.WriteBoolean(false, 8_192); + encoded = writer.Exit(out length); + + Assert.Equal(2, length); + Assert.Equal(length, encoded.Length); + Assert.Equal(63, encoded.Span[0]); + Assert.Single(allocator.AllocationLog); + Assert.Empty(allocator.ReturnLog); } TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog); Assert.Equal(allocation.AllocationId, returned.AllocationId); } + /// + /// Verifies bounded coefficient scratch is rented at construction and reused by costing, coding, and frame resets. + /// [Fact] - public void SymbolEncoderRentsCoefficientScratchOnlyForNonzeroBlocks() + public void SymbolEncoderReusesConstructorOwnedCoefficientScratchAcrossFrames() { TestMemoryAllocator allocator = new(); allocator.EnableNonThreadSafeLogging(); @@ -822,54 +931,10 @@ public class Av1EntropyTests using (Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex, updateCdf: true)) { - TestMemoryAllocator.AllocationRequest outputScratch = Assert.Single(allocator.AllocationLog); - Assert.Equal(typeof(byte), outputScratch.ElementType); - - int emptyContext = encoder.WriteCoefficients( - Av1TransformSize.Size4x4, - Av1TransformType.DctDct, - Av1PredictionMode.DC, - coefficients, - Av1ComponentType.Luminance, - default, - 0, - false, - Av1FilterIntraMode.DC, - usesInterTransformSet: false); - - Assert.Equal(0, emptyContext); - Assert.Single(allocator.AllocationLog); - - coefficients[0] = 1; - _ = encoder.GetCoefficientCost( - Av1TransformSize.Size4x4, - Av1TransformType.DctDct, - Av1PredictionMode.DC, - coefficients, - Av1ComponentType.Luminance, - default, - 1, - false, - Av1FilterIntraMode.DC, - usesInterTransformSet: false); - Assert.Equal(3, allocator.AllocationLog.Count); - - encoder.WriteCoefficients( - Av1TransformSize.Size4x4, - Av1TransformType.DctDct, - Av1PredictionMode.DC, - coefficients, - Av1ComponentType.Luminance, - default, - 1, - false, - Av1FilterIntraMode.DC, - usesInterTransformSet: false); - - Assert.Equal(3, allocator.AllocationLog.Count); - TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1]; - TestMemoryAllocator.AllocationRequest contextScratch = allocator.AllocationLog[2]; + TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[0]; + TestMemoryAllocator.AllocationRequest contextScratch = allocator.AllocationLog[1]; + TestMemoryAllocator.AllocationRequest outputScratch = allocator.AllocationLog[2]; int maximumTransformDimension = Av1Constants.MaxTransformSize / 2; int expectedLevelLength = (Av1Constants.TransformPadHorizontal + maximumTransformDimension) * @@ -880,6 +945,59 @@ public class Av1EntropyTests Assert.Equal(AllocationOptions.Clean, levelScratch.AllocationOptions); Assert.Equal(typeof(sbyte), contextScratch.ElementType); Assert.Equal(maximumTransformDimension * maximumTransformDimension, contextScratch.Length); + Assert.Equal(typeof(byte), outputScratch.ElementType); + Assert.Equal(64, outputScratch.Length); + + // Exercise both an empty and a coded transform on each side of a frame reset. The second pass must + // reuse every constructor-owned buffer even after nonzero levels and probability updates exist. + for (int frame = 0; frame < 2; frame++) + { + coefficients.Clear(); + int emptyContext = encoder.WriteCoefficients( + Av1TransformSize.Size4x4, + Av1TransformType.DctDct, + Av1PredictionMode.DC, + coefficients, + Av1ComponentType.Luminance, + default, + 0, + false, + Av1FilterIntraMode.DC, + usesInterTransformSet: false); + + Assert.Equal(0, emptyContext); + Assert.Equal(3, allocator.AllocationLog.Count); + + coefficients[0] = 1; + _ = encoder.GetCoefficientCost( + Av1TransformSize.Size4x4, + Av1TransformType.DctDct, + Av1PredictionMode.DC, + coefficients, + Av1ComponentType.Luminance, + default, + 1, + false, + Av1FilterIntraMode.DC, + usesInterTransformSet: false); + + encoder.WriteCoefficients( + Av1TransformSize.Size4x4, + Av1TransformType.DctDct, + Av1PredictionMode.DC, + coefficients, + Av1ComponentType.Luminance, + default, + 1, + false, + Av1FilterIntraMode.DC, + usesInterTransformSet: false); + + encoder.Exit(out _); + encoder.Reset(); + Assert.Equal(3, allocator.AllocationLog.Count); + Assert.Empty(allocator.ReturnLog); + } } Assert.Equal(3, allocator.ReturnLog.Count); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs index 72dec76cee..cfeadfcee0 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs @@ -114,6 +114,50 @@ public class Av1InterpolationFilterEntropyTests } } + /// + /// Verifies the encoder's context selection, read-only costing, and adaptive output against independently seeded distributions. + /// + [Theory] + [MemberData(nameof(GetContexts))] + public void EncoderUsesRequestedContextAndLiveCosts(int context) + { + ReadOnlySpan filters = + [ + Av1InterpolationFilter.Sharp, + Av1InterpolationFilter.Smooth, + Av1InterpolationFilter.Regular, + Av1InterpolationFilter.Sharp, + Av1InterpolationFilter.Regular, + Av1InterpolationFilter.Smooth, + ]; + + // The constructor converts forward thresholds to inverse CDF storage. Supply the published forward + // values directly, independently of the production context factory. + Av1Distribution distribution = new( + ForwardThresholds[context * 2], + ForwardThresholds[(context * 2) + 1]); + + using Av1SymbolWriter expectedWriter = new(Configuration.Default, 64, updateCdf: true); + using Av1SymbolEncoder encoder = new(Configuration.Default, 64, qIndex: 0, updateCdf: true); + foreach (Av1InterpolationFilter filter in filters) + { + int expectedCost = Av1ProbabilityCost.GetSymbolCost(distribution, (int)filter); + Assert.Equal(expectedCost, encoder.GetSwitchableInterpolationFilterCost(filter, context)); + Assert.Equal(expectedCost, encoder.GetSwitchableInterpolationFilterCost(filter, context)); + expectedWriter.WriteSymbol((int)filter, distribution); + encoder.WriteSwitchableInterpolationFilter(filter, context); + } + + using IMemoryOwner expected = expectedWriter.Exit(); + using IMemoryOwner actual = encoder.Exit(); + Assert.Equal(expected.Memory.Span, actual.Memory.Span); + Av1SymbolDecoder decoder = new(Configuration.Default, actual.Memory.Span, 0, updateCdf: true); + foreach (Av1InterpolationFilter filter in filters) + { + Assert.Equal(filter, decoder.ReadSwitchableInterpolationFilter(context)); + } + } + /// /// Verifies all sixteen combinations of reference type, direction, and contributing neighbor filter state. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs index 4acb8d8510..e51fd4b901 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs @@ -8,12 +8,15 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; +using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Tests.Memory; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -23,6 +26,168 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1IntraSuperblockEncoderTests { + /// + /// Verifies non-regular filter selection, retained reconstruction, and allocation-free inter tile coding. + /// + [Theory] + [InlineData((int)Av1InterpolationFilter.Smooth, false)] + [InlineData((int)Av1InterpolationFilter.Sharp, false)] + [InlineData((int)Av1InterpolationFilter.Smooth, true)] + [InlineData((int)Av1InterpolationFilter.Sharp, true)] + public void ProductionTileSelectsNonRegularInterpolation(int filterValue, bool dualFilter) + { + const int Width = 32; + const int Height = 8; + const int TargetColumn = 8; + const int BlockWidth = 8; + const int QIndex = 37; + const int TileBufferLength = 4096; + Av1InterpolationFilter filter = (Av1InterpolationFilter)filterValue; + int effort = dualFilter ? 9 : 8; + ReadOnlySpan referencePeriod = [128, 184, 208, 184, 128, 72, 48, 72]; + + // These are fixed half-sample responses of the reference's eight-tap smooth and sharp kernels. + // The horizontal pass rounds first by three bits and then by four; edge samples are replicated. + // Keeping the results literal avoids using the predictor under test to manufacture its own target. + ReadOnlySpan targetRow = filter == Av1InterpolationFilter.Smooth + ? [159, 189, 190, 154, 102, 66, 66, 102, 154, 190, 190, 154, 102, 66, 66, 102, + 154, 190, 190, 154, 102, 66, 66, 102, 154, 190, 190, 154, 102, 67, 61, 69] + : [153, 204, 200, 158, 98, 55, 55, 98, 158, 202, 202, 158, 98, 55, 55, 98, + 158, 202, 202, 158, 98, 55, 55, 98, 158, 202, 202, 158, 100, 53, 59, 75]; + + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + ObuColorConfig colorConfig = new() + { + IsMonochrome = true, + ColorRange = true, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + }; + + using Image referenceImage = new(Width, Height); + using Av1EncoderFrameBuffer reference = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); + using Av1EncoderFrameBuffer source = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); + using Av1EncoderFrameBuffer reconstruction = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); + for (int y = 0; y < Height; y++) + { + Span pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); + Span referenceRow = reference.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y); + for (int x = 0; x < Width; x++) + { + byte sample = referencePeriod[x % referencePeriod.Length]; + referenceRow[x] = sample; + pixels[x] = new L8(sample); + } + + targetRow.CopyTo(source.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y)); + } + + reference.Frame.ExtendBorders(); + source.Frame.ExtendBorders(); + ClearPlane(reconstruction.Luma); + + // A lossless key frame gives an independent decoder exactly the reference samples used by tile search. + using MemoryStream firstSample = new(); + using Av1FrameEncoder.SequenceEncoder keyEncoder = Av1FrameEncoder.CreateColorSequenceEncoder( + configuration, + Width, + Height, + colorConfig, + qIndex: 0, + effort); + + keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample); + ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader; + using Av1EncoderModeInfoBuffer modeInfo = new(configuration, Width, Height, disallow4x4AllFrames: true); + Av1PictureControlSet template = CreatePicture(modeInfo, colorConfig, use128x128Superblock: false, QIndex); + ObuFrameHeader frameHeader = template.Parent.FrameHeader; + frameHeader.FrameType = ObuFrameType.InterFrame; + frameHeader.ShowFrame = true; + frameHeader.ErrorResilientMode = true; + frameHeader.RefreshFrameFlags = byte.MaxValue; + frameHeader.DisableFrameEndUpdateCdf = true; + frameHeader.ReferenceMode = ObuReferenceMode.SingleReference; + frameHeader.InterpolationFilter = Av1InterpolationFilter.Switchable; + frameHeader.AllowHighPrecisionMotionVector = true; + frameHeader.TransformMode = Av1TransformMode.Select; + frameHeader.FrameSize.FrameWidth = Width; + frameHeader.FrameSize.FrameHeight = Height; + frameHeader.FrameSize.SuperResolutionUpscaledWidth = Width; + frameHeader.FrameSize.RenderWidth = Width; + frameHeader.FrameSize.RenderHeight = Height; + frameHeader.TilesInfo.HasUniformTileSpacing = true; + Av1QuantizationLookup.UpdateFrameQuantizationState(frameHeader); + + using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true); + using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height); + using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); + using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); + using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true); + Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace); + int allocationCount = allocator.AllocationLog.Count; + Av1TileEncoder tileWriter = new( + symbolEncoder, + source.Frame, + reference.Frame, + reconstruction.Frame, + picture.Picture, + coefficients, + tileWorkspace, + blockWorkspace, + effort); + + Assert.Equal(allocationCount, allocator.AllocationLog.Count); + Point targetPosition = new(TargetColumn >> Av1Constants.ModeInfoSizeLog2, 0); + ref Av1MacroBlockModeInfo targetMode = ref picture.Picture.GetMacroBlockModeInfo(targetPosition); + Assert.Equal(Av1ReferenceFrameType.Last, targetMode.Block.ReferenceFrame); + Assert.Equal(filter, targetMode.Block.HorizontalInterpolationFilter); + Assert.Equal(dualFilter ? Av1InterpolationFilter.Regular : filter, targetMode.Block.VerticalInterpolationFilter); + Assert.Equal(4, picture.Picture.GetDisplacementVector(targetPosition).Column); + Assert.Equal(0, picture.Picture.GetDisplacementVector(targetPosition).Row); + for (int y = 0; y < Height; y++) + { + Assert.Equal( + targetRow.Slice(TargetColumn, BlockWidth), + reconstruction.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y).Slice(TargetColumn, BlockWidth)); + } + + using MemoryStream secondSample = new(); + using ObuWriter obuWriter = new(configuration); + obuWriter.WriteFrame(secondSample, sequenceHeader, frameHeader, tileWriter); + using Av1Decoder decoder = new(configuration); + using Av1FrameBuffer decodedFirst = decoder.DecodeFrameBuffer(firstSample.ToArray(), null, null, out _); + using Av1FrameBuffer decodedSecond = decoder.DecodeFrameBuffer(secondSample.ToArray(), null, null, out _); + + // Preserve both the production stream and every managed reconstructed luma sample for exact libaom comparison. + string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.ProductionTileSelectsNonRegularInterpolation)); + string outputName = $"{filter}-{dualFilter}"; + using FileStream output = File.Create(Path.Combine(outputDirectory, outputName + ".obu")); + firstSample.Position = 0; + firstSample.CopyTo(output); + secondSample.Position = 0; + secondSample.CopyTo(output); + using FileStream rawOutput = File.Create(Path.Combine(outputDirectory, outputName + ".managed.yuv")); + for (int y = 0; y < Height; y++) + { + ReadOnlySpan expected = reference.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y); + ReadOnlySpan actual = decodedFirst.DeriveBlockPointer(Av1Plane.Y, new Point(0, y), 0, 0, out _)[..Width]; + Assert.Equal(expected, actual); + rawOutput.Write(actual); + } + + for (int y = 0; y < Height; y++) + { + ReadOnlySpan expected = reconstruction.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y); + ReadOnlySpan actual = decodedSecond.DeriveBlockPointer(Av1Plane.Y, new Point(0, y), 0, 0, out _)[..Width]; + Assert.Equal(expected, actual); + rawOutput.Write(actual); + } + } + /// /// Gets the normative eight-sample weights used to build independent smooth-mode fixtures. /// @@ -259,7 +424,7 @@ public class Av1IntraSuperblockEncoderTests tilePicture.Picture, 512); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( tileSymbolEncoder, source.Frame, tileReconstruction.Frame, @@ -413,7 +578,7 @@ public class Av1IntraSuperblockEncoderTests livePicture.Picture, 256); - Av1IntraTileWriter liveTileWriter = new( + Av1TileEncoder liveTileWriter = new( liveSymbolEncoder, source.Frame, liveReconstruction.Frame, @@ -541,7 +706,7 @@ public class Av1IntraSuperblockEncoderTests tilePicture.Picture, 256); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( tileSymbolEncoder, source.Frame, tileReconstruction.Frame, @@ -749,7 +914,7 @@ public class Av1IntraSuperblockEncoderTests 32, 224, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -770,7 +935,7 @@ public class Av1IntraSuperblockEncoderTests 512, 3584, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -791,7 +956,7 @@ public class Av1IntraSuperblockEncoderTests 48, 208, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -816,7 +981,7 @@ public class Av1IntraSuperblockEncoderTests 64, 192, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -837,7 +1002,7 @@ public class Av1IntraSuperblockEncoderTests 1024, 3072, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -946,7 +1111,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 256); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -1162,7 +1327,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 512); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -1279,7 +1444,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 512); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -1442,7 +1607,7 @@ public class Av1IntraSuperblockEncoderTests pilotPicture.Picture, TileBufferLength); - Av1IntraTileWriter pilotWriter = createWriter( + Av1TileEncoder pilotWriter = createWriter( pilotSymbolEncoder, pilotSource.Frame, pilotReconstruction.Frame, @@ -1549,7 +1714,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, TileBufferLength); - Av1IntraTileWriter tileWriter = createWriter( + Av1TileEncoder tileWriter = createWriter( symbolEncoder, source.Frame, reconstruction.Frame, @@ -1792,7 +1957,7 @@ public class Av1IntraSuperblockEncoderTests pilotPicture.Picture, TileBufferLength); - Av1IntraTileWriter pilotWriter = createWriter( + Av1TileEncoder pilotWriter = createWriter( pilotSymbolEncoder, pilotSource.Frame, pilotReconstruction.Frame, @@ -1954,7 +2119,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, TileBufferLength); - Av1IntraTileWriter tileWriter = createWriter( + Av1TileEncoder tileWriter = createWriter( symbolEncoder, source.Frame, reconstruction.Frame, @@ -2158,7 +2323,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 2048); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -2183,7 +2348,7 @@ public class Av1IntraSuperblockEncoderTests 8, static value => (byte)value, static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -2198,7 +2363,7 @@ public class Av1IntraSuperblockEncoderTests 12, static value => (ushort)(value << 4), static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => - new Av1IntraTileWriter( + new Av1TileEncoder( writer, source, reconstruction, @@ -2303,7 +2468,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, TileBufferLength); - Av1IntraTileWriter tileWriter = createTileWriter( + Av1TileEncoder tileWriter = createTileWriter( symbolEncoder, source.Frame, reconstruction.Frame, @@ -2436,7 +2601,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 4096); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -2550,7 +2715,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, 4096); - Av1IntraTileWriter tileWriter = new( + Av1TileEncoder tileWriter = new( symbolEncoder, source.Frame, reconstruction.Frame, @@ -2631,8 +2796,8 @@ public class Av1IntraSuperblockEncoderTests frameHeader.TilesInfo.HasUniformTileSpacing = true; using MemoryStream stream = new(); - new ObuWriter().WriteAll( - Configuration.Default, + using ObuWriter obuWriter = new(Configuration.Default); + obuWriter.WriteSequenceFrame( stream, sequenceHeader, frameHeader, @@ -2689,14 +2854,16 @@ public class Av1IntraSuperblockEncoderTests FrameSize = new ObuFrameSize() }, FrameHeader = frameHeader, - PreviousQIndex = [qIndex] + PreviousQIndex = new int[] { qIndex } }, SegmentationNeighborMap = new byte[modeInfo.ModeInfoColumnCount * modeInfo.ModeInfoRowCount], ModeInfoGrid = modeInfo.Grid, ModeInfoAllocation = modeInfo.Allocation, ModeInfoStride = modeInfo.ModeInfoStride, Disallow4x4AllFrames = modeInfo.Disallow4x4AllFrames, - CdefPreset = [[-1, -1, -1, -1]] + CdefPreset = new int[] { -1, -1, -1, -1 }, + TileDataOffsets = Memory.Empty, + TileDataLengths = Memory.Empty }; } @@ -2813,7 +2980,7 @@ public class Av1IntraSuperblockEncoderTests picture.Picture, TileBufferLength); - Av1IntraTileWriter tileWriter = createTileWriter( + Av1TileEncoder tileWriter = createTileWriter( symbolEncoder, source.Frame, reconstruction.Frame, @@ -3044,7 +3211,7 @@ public class Av1IntraSuperblockEncoderTests updateCdf: !frameHeader.DisableCdfUpdate); } - private delegate Av1IntraTileWriter TileWriterFactory( + private delegate Av1TileEncoder TileWriterFactory( Av1SymbolEncoder writer, Av1EncoderFrame source, Av1EncoderFrame reconstruction, @@ -3143,7 +3310,8 @@ public class Av1IntraSuperblockEncoderTests macroBlock, Av1BlockSize.Block8x8, Av1PredictionMode.DC, - 0); + 0, + isIntraFrame: true); modeInfo.Block = new Av1EncoderBlockModeInfo { diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs index c4818ccf1c..9040b66c10 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs @@ -112,6 +112,37 @@ public class Av1MotionVectorEntropyTests Assert.True(decoder.ReadDrl(1)); } + [Theory] + [InlineData((int)Av1MotionVectorPrecision.Integer, 16, -24)] + [InlineData((int)Av1MotionVectorPrecision.QuarterSample, 6, -10)] + [InlineData((int)Av1MotionVectorPrecision.EighthSample, 11, -17)] + public void WriteMotionVectorRoundTripsRequestedPrecision( + int precisionValue, + int rowDelta, + int columnDelta) + { + Av1MotionVectorPrecision precision = (Av1MotionVectorPrecision)precisionValue; + Av1MotionVector reference = new(27, -11); + Av1MotionVector value = reference + new Av1MotionVector(rowDelta, columnDelta); + Av1MotionVectorContext writerContext = new(); + Av1Distribution trailingDistribution = Av1DefaultDistributions.Drl[1]; + using Av1SymbolWriter writer = new(Configuration.Default, 32, updateCdf: true); + + writerContext.Write(writer, value, reference, precision); + writer.WriteSymbol(true, trailingDistribution); + + using IMemoryOwner encoded = writer.Exit(); + Av1FrameEntropyContext decoderContext = new(0); + Av1SymbolDecoder decoder = new( + Configuration.Default, + encoded.Memory.Span, + decoderContext, + updateCdf: true); + + Assert.Equal(value, decoder.ReadMotionVector(reference, precision)); + Assert.True(decoder.ReadDrl(1)); + } + /// /// Verifies that normal and displacement motion vectors never share adaptive distribution state. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs index 247e59cef2..64ac67a3e6 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs @@ -15,6 +15,128 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1ResidualBuilderTests { + /// + /// Verifies precision limits, independent strides, candidate order, and exact final-row bounds. + /// + [Theory] + [InlineData(255, false)] + [InlineData(255, true)] + public void ByteSearchMetricsMatchKnownMoments(int maximum, bool negative) + { + const int SourceStride = 13; + const int PredictionStride = 17; + const int SourceOffset = 1; + const int PredictionOffset = 3; + byte[] sourceBuffer = new byte[SourceOffset + (7 * SourceStride) + 8]; + byte[] predictionBuffer = new byte[PredictionOffset + (7 * PredictionStride) + 11]; + sourceBuffer.AsSpan().Fill((byte)maximum); + predictionBuffer.AsSpan().Fill((byte)maximum); + Span source = sourceBuffer.AsSpan(SourceOffset); + Span prediction = predictionBuffer.AsSpan(PredictionOffset); + int[] sums = [-1, 0, 0, 0, 0, -1]; + + // The last source row contains exactly eight samples, and the four prediction windows require + // exactly eleven. Distinct nonzero padding catches accidental participation of neighboring rows. + for (int row = 0; row < 8; row++) + { + source.Slice(row * SourceStride, 8).Fill((byte)(negative ? 0 : maximum)); + prediction.Slice(row * PredictionStride, 11).Fill((byte)(negative ? maximum : 0)); + } + + Assert.Equal(64 * maximum, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride)); + Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out int sum, out int squaredSum); + Assert.Equal((negative ? -64 : 64) * maximum, sum); + Assert.Equal(64 * maximum * maximum, squaredSum); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4)); + Assert.Equal(new[] { -1, 64 * maximum, 64 * maximum, 64 * maximum, 64 * maximum, -1 }, sums); + + // Source (2*y + x) and prediction (y + 2*x) give residual (y - x) * step, exercising both signs + // and distinct rows. Across the 8x8 square its signed sum is zero, absolute sum 168, and squared sum 672. + // Offsetting prediction by 1, 2, or 3 columns gives absolute sums 198, 276, and 386 respectively. + int step = maximum / 32; + for (int row = 0; row < 8; row++) + { + for (int column = 0; column < 8; column++) + { + source[(row * SourceStride) + column] = (byte)(((2 * row) + column) * step); + } + + for (int column = 0; column < 11; column++) + { + prediction[(row * PredictionStride) + column] = (byte)((row + (2 * column)) * step); + } + } + + Assert.Equal(168 * step, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride)); + Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out sum, out squaredSum); + Assert.Equal(0, sum); + Assert.Equal(672 * step * step, squaredSum); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4)); + Assert.Equal(new[] { -1, 168 * step, 198 * step, 276 * step, 386 * step, -1 }, sums); + } + + /// + /// Verifies precision limits, independent strides, candidate order, and exact final-row bounds. + /// + [Theory] + [InlineData(1023, false)] + [InlineData(1023, true)] + [InlineData(4095, false)] + [InlineData(4095, true)] + public void HighBitDepthSearchMetricsMatchKnownMoments(int maximum, bool negative) + { + const int SourceStride = 13; + const int PredictionStride = 17; + const int SourceOffset = 1; + const int PredictionOffset = 3; + ushort[] sourceBuffer = new ushort[SourceOffset + (7 * SourceStride) + 8]; + ushort[] predictionBuffer = new ushort[PredictionOffset + (7 * PredictionStride) + 11]; + sourceBuffer.AsSpan().Fill((ushort)maximum); + predictionBuffer.AsSpan().Fill((ushort)maximum); + Span source = sourceBuffer.AsSpan(SourceOffset); + Span prediction = predictionBuffer.AsSpan(PredictionOffset); + int[] sums = [-1, 0, 0, 0, 0, -1]; + + // The last source row contains exactly eight samples, and the four prediction windows require + // exactly eleven. Distinct nonzero padding catches accidental participation of neighboring rows. + for (int row = 0; row < 8; row++) + { + source.Slice(row * SourceStride, 8).Fill((ushort)(negative ? 0 : maximum)); + prediction.Slice(row * PredictionStride, 11).Fill((ushort)(negative ? maximum : 0)); + } + + Assert.Equal(64 * maximum, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride)); + Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out int sum, out int squaredSum); + Assert.Equal((negative ? -64 : 64) * maximum, sum); + Assert.Equal(64 * maximum * maximum, squaredSum); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4)); + Assert.Equal(new[] { -1, 64 * maximum, 64 * maximum, 64 * maximum, 64 * maximum, -1 }, sums); + + // Source (2*y + x) and prediction (y + 2*x) give residual (y - x) * step, exercising both signs + // and distinct rows. Across the 8x8 square its signed sum is zero, absolute sum 168, and squared sum 672. + // Offsetting prediction by 1, 2, or 3 columns gives absolute sums 198, 276, and 386 respectively. + int step = maximum / 32; + for (int row = 0; row < 8; row++) + { + for (int column = 0; column < 8; column++) + { + source[(row * SourceStride) + column] = (ushort)(((2 * row) + column) * step); + } + + for (int column = 0; column < 11; column++) + { + prediction[(row * PredictionStride) + column] = (ushort)((row + (2 * column)) * step); + } + } + + Assert.Equal(168 * step, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride)); + Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out sum, out squaredSum); + Assert.Equal(0, sum); + Assert.Equal(672 * step * step, squaredSum); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4)); + Assert.Equal(new[] { -1, 168 * step, 198 * step, 276 * step, 386 * step, -1 }, sums); + } + private const HwIntrinsics ResidualConfigurations = HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; @@ -133,6 +255,14 @@ public class Av1ResidualBuilderTests width, height); + int[] candidateSums = new int[4]; + _ = Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, width, prediction, width); + _ = Av1ResidualBuilder.SumAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, width, prediction, width, candidateSums); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width, candidateSums); + Av1ResidualBuilder.GetMoments8x8(source, width, prediction, width, out _, out _); + Av1ResidualBuilder.GetMoments8x8(highBitDepthSource, width, highBitDepthPrediction, width, out _, out _); + long sum = 0; long before = GC.GetAllocatedBytesForCurrentThread(); for (int iteration = 0; iteration < 32; iteration++) @@ -147,6 +277,16 @@ public class Av1ResidualBuilderTests width, width, height); + + sum += Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, width, prediction, width); + sum += Av1ResidualBuilder.SumAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width); + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, width, prediction, width, candidateSums); + sum += candidateSums[0]; + Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width, candidateSums); + sum += candidateSums[3]; + Av1ResidualBuilder.GetMoments8x8(source, width, prediction, width, out int byteSum, out int byteSquares); + Av1ResidualBuilder.GetMoments8x8(highBitDepthSource, width, highBitDepthPrediction, width, out int wordSum, out int wordSquares); + sum += byteSum + byteSquares + wordSum + wordSquares; } long allocated = GC.GetAllocatedBytesForCurrentThread() - before; diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs index 3eb5cb0537..a5efee5fae 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs @@ -14,6 +14,11 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; [Trait("Format", "Avif")] public class Av1SingleReferenceEntropyTests { + /// + /// The quantizer index used to initialize an otherwise unrelated tile entropy encoder. + /// + private const int BaseQIndex = 128; + /// /// Verifies all eighteen normative single-reference distributions against the reference decoder's forward Q15 defaults. /// @@ -121,6 +126,41 @@ public class Av1SingleReferenceEntropyTests } } + /// + /// Verifies that the production writer emits every single-reference branch consumed by the decoder. + /// + /// The encoded reference-frame label. + [Theory] + [InlineData((int)Av1ReferenceFrameType.Last)] + [InlineData((int)Av1ReferenceFrameType.Last2)] + [InlineData((int)Av1ReferenceFrameType.Last3)] + [InlineData((int)Av1ReferenceFrameType.Golden)] + [InlineData((int)Av1ReferenceFrameType.Backward)] + [InlineData((int)Av1ReferenceFrameType.Alternate2)] + [InlineData((int)Av1ReferenceFrameType.Alternate)] + public void SingleReferenceWriterRoundTripsEveryReference(int referenceFrameValue) + { + Av1ReferenceFrameType referenceFrame = (Av1ReferenceFrameType)referenceFrameValue; + InlineArray8 referenceCountStorage = default; + Span referenceCounts = referenceCountStorage; + referenceCounts[(int)Av1ReferenceFrameType.Last] = 5; + referenceCounts[(int)Av1ReferenceFrameType.Last2] = 1; + referenceCounts[(int)Av1ReferenceFrameType.Last3] = 2; + referenceCounts[(int)Av1ReferenceFrameType.Golden] = 2; + referenceCounts[(int)Av1ReferenceFrameType.Backward] = 3; + referenceCounts[(int)Av1ReferenceFrameType.Alternate2] = 3; + referenceCounts[(int)Av1ReferenceFrameType.Alternate] = 6; + + using Av1SymbolEncoder encoder = new(Configuration.Default, 8, BaseQIndex, updateCdf: true); + encoder.WriteSingleReference(referenceFrame, referenceCounts); + + using IMemoryOwner encoded = encoder.Exit(); + Av1SymbolDecoder decoder = new(Configuration.Default, encoded.Memory.Span, BaseQIndex, updateCdf: true); + Av1ReferenceFrameType decodedReference = ReadSingleReference(ref decoder, referenceCounts); + + Assert.Equal(referenceFrame, decodedReference); + } + /// /// Verifies one-pass neighbor collection, compound-neighbor votes, clearing, and intra-neighbor exclusion. /// @@ -314,6 +354,43 @@ public class Av1SingleReferenceEntropyTests _ => decoder.ReadSingleReferenceIsAlternate2(context), }; + /// + /// Reads one complete single-reference branch through the production semantic entry points. + /// + private static Av1ReferenceFrameType ReadSingleReference( + ref Av1SymbolDecoder decoder, + scoped ReadOnlySpan referenceCounts) + { + int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts); + if (decoder.ReadSingleReferenceIsBackward(context)) + { + context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts); + if (decoder.ReadSingleReferenceIsAlternate(context)) + { + return Av1ReferenceFrameType.Alternate; + } + + context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts); + return decoder.ReadSingleReferenceIsAlternate2(context) + ? Av1ReferenceFrameType.Alternate2 + : Av1ReferenceFrameType.Backward; + } + + context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts); + if (decoder.ReadSingleReferenceIsLast3OrGolden(context)) + { + context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts); + return decoder.ReadSingleReferenceIsGolden(context) + ? Av1ReferenceFrameType.Golden + : Av1ReferenceFrameType.Last3; + } + + context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts); + return decoder.ReadSingleReferenceIsLast2(context) + ? Av1ReferenceFrameType.Last2 + : Av1ReferenceFrameType.Last; + } + /// /// Creates decoded block-mode state with the requested primary and secondary reference labels. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs index 4fe759a339..7b08d44def 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs @@ -614,8 +614,8 @@ public class Av1TransformBlockEncoderTests Av1EncoderModeDecisionWorkspace modeWorkspace = workspace.GetModeDecisionWorkspace(); Av1EncoderPaletteWorkspace paletteWorkspace = modeWorkspace.Palette; - Av1EncoderIntraBlockCopyWorkspace intraBlockCopyWorkspace = - workspace.GetIntraBlockCopyWorkspace(); + Av1EncoderInterPredictionWorkspace intraBlockCopyWorkspace = + workspace.GetInterPredictionWorkspace(); Assert.Equal( (2 * Av1Constants.MaxTransformSize) + 1, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs index 360c9291f7..e4daab57b4 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs @@ -129,8 +129,8 @@ public class ObuFrameHeaderTests MemoryStream encoded = new(); // Act 2 - ObuWriter obuWriter = new(); - obuWriter.WriteAll(Configuration.Default, encoded, obuReader.SequenceHeader, obuReader.FrameHeader, tileStub); + using ObuWriter obuWriter = new(Configuration.Default); + obuWriter.WriteSequenceFrame(encoded, obuReader.SequenceHeader, obuReader.FrameHeader, tileStub); // Assert byte[] encodedArray = encoded.ToArray(); @@ -157,8 +157,8 @@ public class ObuFrameHeaderTests MemoryStream encoded = new(); // Act 2 - ObuWriter obuWriter = new(); - obuWriter.WriteAll(Configuration.Default, encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader, tileStub); + using ObuWriter obuWriter = new(Configuration.Default); + obuWriter.WriteSequenceFrame(encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader, tileStub); // Assign 2 Span encodedBuffer = encoded.ToArray(); @@ -456,8 +456,8 @@ public class ObuFrameHeaderTests tileStub.ReadTile(emptyTile, 0); using MemoryStream stream = new(); - ObuWriter writer = new(); - writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, tileStub); + using ObuWriter writer = new(Configuration.Default); + writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub); byte[] bitStream = stream.ToArray(); int sequenceObuOffset = DefaultTemporalDelimiterBitStream.Length; @@ -639,10 +639,9 @@ public class ObuFrameHeaderTests { // Arrange using MemoryStream stream = new(2); - ObuWriter obuWriter = new(); // Act - obuWriter.WriteAll(Configuration.Default, stream, null, null, null); + ObuWriter.WriteTemporalDelimiter(stream); byte[] actual = stream.GetBuffer(); // Assert @@ -655,10 +654,9 @@ public class ObuFrameHeaderTests // Arrange using MemoryStream stream = new(10); ObuSequenceHeader input = GetDefaultSequenceHeader(); - ObuWriter obuWriter = new(); // Act - obuWriter.WriteAll(Configuration.Default, stream, input, null, null); + ObuWriter.WriteSequenceHeader(Configuration.Default, stream, input); byte[] buffer = stream.GetBuffer(); // Assert @@ -667,6 +665,68 @@ public class ObuFrameHeaderTests Assert.Equal(DefaultSequenceHeaderBitStream, actual); } + [Fact] + public void WriteNonReducedSequenceHeaderPreservesTimingAndDecoderModel() + { + ObuSequenceHeader input = GetDefaultSequenceHeader(); + input.IsStillPicture = false; + input.IsReducedStillPictureHeader = false; + input.TimingInfoPresentFlag = true; + input.TimingInfo = new ObuTimingInfo + { + NumUnitsInDisplayTick = 1001, + TimeScale = 60000, + EqualPictureInterval = true, + NumTicksPerPicture = 2 + }; + + input.DecoderModelInfoPresentFlag = true; + input.DecoderModelInfo = new ObuDecoderModelInfo + { + BufferDelayLength = 12, + NumUnitsInDecodingTick = 1000, + BufferRemovalTimeLength = 10, + FramePresentationTimeLength = 9 + }; + + input.InitialDisplayDelayPresentFlag = true; + ObuOperatingPoint inputOperatingPoint = input.OperatingPoint[0]; + inputOperatingPoint.SequenceLevelIndex = Av1Constants.SequenceTierMinimumLevelIndex; + inputOperatingPoint.SequenceTier = 1; + inputOperatingPoint.IsDecoderModelInfoPresent = true; + inputOperatingPoint.DecoderBufferDelay = 137; + inputOperatingPoint.EncoderBufferDelay = 211; + inputOperatingPoint.LowDelayMode = true; + inputOperatingPoint.IsInitialDisplayDelayPresent = true; + inputOperatingPoint.InitialDisplayDelay = 4; + + using MemoryStream stream = new(); + ObuWriter.WriteSequenceHeader(Configuration.Default, stream, input); + byte[] bitStream = stream.ToArray(); + Av1BitStreamReader reader = new(bitStream); + ObuReader obuReader = new(); + obuReader.ReadAll(ref reader, bitStream.Length, () => new Av1TileDecoderStub()); + + ObuSequenceHeader output = obuReader.SequenceHeader; + ObuTimingInfo outputTiming = output.GetTimingInfo(); + ObuDecoderModelInfo outputDecoderModel = output.GetDecoderModelInfo(); + ObuOperatingPoint outputOperatingPoint = output.OperatingPoint[0]; + Assert.False(output.IsStillPicture); + Assert.False(output.IsReducedStillPictureHeader); + Assert.Equal(input.TimingInfo.NumUnitsInDisplayTick, outputTiming.NumUnitsInDisplayTick); + Assert.Equal(input.TimingInfo.TimeScale, outputTiming.TimeScale); + Assert.Equal(input.TimingInfo.NumTicksPerPicture, outputTiming.NumTicksPerPicture); + Assert.Equal(input.DecoderModelInfo.BufferDelayLength, outputDecoderModel.BufferDelayLength); + Assert.Equal(input.DecoderModelInfo.NumUnitsInDecodingTick, outputDecoderModel.NumUnitsInDecodingTick); + Assert.Equal(input.DecoderModelInfo.BufferRemovalTimeLength, outputDecoderModel.BufferRemovalTimeLength); + Assert.Equal(input.DecoderModelInfo.FramePresentationTimeLength, outputDecoderModel.FramePresentationTimeLength); + Assert.Equal(inputOperatingPoint.SequenceTier, outputOperatingPoint.SequenceTier); + Assert.Equal(inputOperatingPoint.DecoderBufferDelay, outputOperatingPoint.DecoderBufferDelay); + Assert.Equal(inputOperatingPoint.EncoderBufferDelay, outputOperatingPoint.EncoderBufferDelay); + Assert.Equal(inputOperatingPoint.LowDelayMode, outputOperatingPoint.LowDelayMode); + Assert.Equal(inputOperatingPoint.InitialDisplayDelay, outputOperatingPoint.InitialDisplayDelay); + } + /// /// Verifies that the combined frame OBU declares exactly the payload bytes emitted by the writer. /// @@ -680,10 +740,10 @@ public class ObuFrameHeaderTests Av1TileDecoderStub tileStub = new(); byte[] tileData = [0x80]; tileStub.ReadTile(tileData, 0); - ObuWriter obuWriter = new(); + using ObuWriter obuWriter = new(Configuration.Default); // Act - obuWriter.WriteAll(Configuration.Default, stream, sequenceInput, frameInput, tileStub); + obuWriter.WriteSequenceFrame(stream, sequenceInput, frameInput, tileStub); byte[] bitStream = stream.ToArray(); // Assert @@ -706,10 +766,10 @@ public class ObuFrameHeaderTests } /// - /// Verifies that the OBU writer streams an encoded tile from its owning buffer without renting a second payload-sized buffer. + /// Verifies that the OBU writer reuses header scratch and streams encoded tiles without a payload-sized copy. /// [Fact] - public void WriteFrameStreamsTilePayloadWithoutRentingPayloadCopy() + public void WriterReusesHeaderScratchAndStreamsTilePayloadWithoutCopy() { const int TilePayloadLength = 64 * 1024; @@ -725,8 +785,13 @@ public class ObuFrameHeaderTests tileStub.ReadTile(tileData, 0); using MemoryStream stream = new(); - ObuWriter writer = new(); - writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileStub); + using (ObuWriter writer = new(configuration)) + { + writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub); + + // A second complete write must reuse the same bounded header owner retained by the writer. + writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub); + } TestMemoryAllocator.AllocationRequest headerScratch = Assert.Single(allocator.AllocationLog); Assert.Equal(typeof(byte), headerScratch.ElementType); @@ -760,8 +825,8 @@ public class ObuFrameHeaderTests sourceTiles.ReadTile([0x80], 1); using MemoryStream stream = new(); - ObuWriter writer = new(); - writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, sourceTiles); + using ObuWriter writer = new(Configuration.Default); + writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, sourceTiles); byte[] bitStream = stream.ToArray(); Assert.Equal([0x00, 0x80, 0x80], bitStream[^3..]); @@ -936,8 +1001,8 @@ public class ObuFrameHeaderTests tileStub.ReadTile([0x80], 1); using MemoryStream stream = new(); - ObuWriter writer = new(); - writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, tileStub); + using ObuWriter writer = new(Configuration.Default); + writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub); return stream.ToArray(); } diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs index 224699905d..0bb6e69177 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs @@ -4,14 +4,13 @@ using System.Buffers; using System.Buffers.Binary; using System.Text; -using SixLabors.ImageSharp.ColorProfiles; using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats.Heif; using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; -using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.Metadata.Profiles.Exif; using SixLabors.ImageSharp.Metadata.Profiles.Icc; @@ -171,7 +170,7 @@ public class HeifEncoderTests } [Fact] - public void LegacyJpegRejectsLosslessEncoding() + public void LegacyJpegIgnoresLosslessOption() { using Image image = new(1, 1); using MemoryStream stream = new(); @@ -181,14 +180,19 @@ public class HeifEncoderTests Lossless = true }; - Assert.Throws(() => image.Save(stream, encoder)); - Assert.Equal(0, stream.Length); + image.Save(stream, encoder); + stream.Position = 0; + using Image decoded = Image.Load(stream); + Assert.Equal(image.Size, decoded.Size); + Assert.Equal( + HeifCompressionMethod.LegacyJpeg, + decoded.Metadata.GetHeifMetadata().CompressionMethod); } [Theory] [InlineData(HeifBitDepth.Bit10)] [InlineData(HeifBitDepth.Bit12)] - public void LegacyJpegRejectsHighBitDepth(HeifBitDepth bitDepth) + public void LegacyJpegNormalizesHighBitDepthToEightBit(HeifBitDepth bitDepth) { using Image image = new(1, 1); using MemoryStream stream = new(); @@ -198,8 +202,310 @@ public class HeifEncoderTests BitDepth = bitDepth }; - Assert.Throws(() => image.Save(stream, encoder)); - Assert.Equal(0, stream.Length); + image.Save(stream, encoder); + stream.Position = 0; + using Image decoded = Image.Load(stream); + Assert.Equal(HeifBitDepth.Bit8, decoded.Metadata.GetHeifMetadata().BitDepth); + } + + [Fact] + public void LegacyJpegEncodesRootFrameFromImageSequence() + { + using Image image = new(1, 1); + image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[0] = new Rgba32(255, 255, 255); + image.Frames.AddFrame(image.Frames.RootFrame); + image.Frames[1].PixelBuffer.DangerousGetRowSpan(0)[0] = new Rgba32(0, 0, 0); + using MemoryStream stream = new(); + HeifEncoder encoder = new() + { + CompressionMethod = HeifCompressionMethod.LegacyJpeg, + Quality = 100 + }; + + image.Save(stream, encoder); + stream.Position = 0; + using Image decoded = Image.Load(stream); + Assert.Single(decoded.Frames); + Assert.Equal(new Rgba32(255, 255, 255), decoded.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[0]); + } + + [Fact] + public void Av1ImageSequencePreservesSeparateRootFrame() + { + const int width = 8; + const int height = 8; + using Image image = new(width, height); + for (int row = 0; row < height; row++) + { + image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row).Fill(new Rgb24(255, 255, 255)); + } + + image.Frames.AddFrame(image.Frames.RootFrame); + for (int row = 0; row < height; row++) + { + image.Frames[1].PixelBuffer.DangerousGetRowSpan(row).Fill(new Rgb24(0, 0, 0)); + } + + image.Frames[1].Metadata.GetHeifMetadata().FrameDelay = new Rational(1, 20); + using MemoryStream stream = new(); + HeifEncoder encoder = new() + { + CompressionMethod = HeifCompressionMethod.Av1, + AnimateRootFrame = false, + Lossless = true, + Effort = 0 + }; + + image.Save(stream, encoder); + byte[] file = stream.ToArray(); + Span fileType = GetTopLevelBox(file, Heif4CharCode.Ftyp); + Assert.Equal(Heif4CharCode.Miaf, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[^sizeof(uint)..])); + Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(GetMetadataChild(file, Heif4CharCode.Pitm)[12..])); + + stream.Position = 0; + using Image decoded = Image.Load(stream); + Assert.Equal(2, decoded.Frames.Count); + Assert.False(decoded.Metadata.GetHeifMetadata().AnimateRootFrame); + Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded)); + Assert.Equal( + image.Frames[1].Metadata.GetHeifMetadata().FrameDelay, + decoded.Frames[1].Metadata.GetHeifMetadata().FrameDelay); + } + + [Fact] + public void GridDecoderAcceptsSmallerRightAndBottomColorAndAlphaCells() + { + const int tileWidth = 64; + const int tileHeight = 64; + const int outputWidth = 96; + const int outputHeight = 96; + Size[] tileSizes = + [ + new(tileWidth, tileHeight), + new(outputWidth - tileWidth, tileHeight), + new(tileWidth, outputHeight - tileHeight), + new(outputWidth - tileWidth, outputHeight - tileHeight) + ]; + + Rgba32[] tileColors = + [ + new(32, 32, 32), + new(64, 64, 64), + new(96, 96, 96), + new(128, 128, 128) + ]; + + ObuColorConfig colorConfig = new() + { + IsColorDescriptionPresent = true, + ColorPrimaries = ObuColorPrimaries.Bt709, + TransferCharacteristics = ObuTransferCharacteristics.Srgb, + MatrixCoefficients = ObuMatrixCoefficients.Bt709, + ColorRange = true, + IsMonochrome = true, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + }; + + List items = []; + Dictionary payloads = []; + HeifItem gridItem = new(Heif4CharCode.Grid, 1); + gridItem.SetExtent(new Size(outputWidth, outputHeight)); + items.Add(gridItem); + HeifItemLink gridLink = new(Heif4CharCode.Dimg, gridItem.Id); + for (int tileIndex = 0; tileIndex < tileSizes.Length; tileIndex++) + { + Size tileSize = tileSizes[tileIndex]; + using Image tile = new(tileSize.Width, tileSize.Height, tileColors[tileIndex]); + using MemoryStream payload = new(); + ObuSequenceHeader header = Av1FrameEncoder.Encode( + Configuration.Default, + tile.Frames.RootFrame, + payload, + colorConfig, + qIndex: 0, + effort: 0); + + uint itemId = (uint)tileIndex + 2; + HeifItem tileItem = new(Heif4CharCode.Av01, itemId) + { + Av1CodecConfiguration = new Av1CodecConfiguration(header) + }; + + tileItem.SetExtent(tileSize); + items.Add(tileItem); + gridLink.DestinationIds.Add(itemId); + payloads.Add(itemId, payload.ToArray()); + } + + List links = [gridLink]; + GridHeifItemDecoder decoder = new(items, links, ReadItem); + Span descriptor = [0, 0, 1, 1, 0, outputWidth, 0, outputHeight]; + using Image result = decoder.DecodeItemData( + new DecoderOptions { Configuration = Configuration.Default }, + gridItem, + descriptor, + null, + TestContext.Current.CancellationToken); + + for (int y = 0; y < outputHeight; y++) + { + for (int x = 0; x < outputWidth; x++) + { + int tileIndex = (y < tileHeight ? 0 : 2) + (x < tileWidth ? 0 : 1); + Assert.Equal(tileColors[tileIndex], result[x, y]); + } + } + + Rgba32 opaqueColor = new(7, 11, 13); + using Image alphaResult = new(outputWidth, outputHeight, opaqueColor); + decoder.DecodeAlphaItemData( + new DecoderOptions { Configuration = Configuration.Default }, + gridItem, + descriptor, + alphaResult.Frames.RootFrame, + alphaResult.Size, + new Rectangle(Point.Empty, alphaResult.Size), + false, + TestContext.Current.CancellationToken); + + for (int y = 0; y < outputHeight; y++) + { + for (int x = 0; x < outputWidth; x++) + { + int tileIndex = (y < tileHeight ? 0 : 2) + (x < tileWidth ? 0 : 1); + Rgba32 expected = opaqueColor; + expected.A = tileColors[tileIndex].R; + Assert.Equal(expected, alphaResult[x, y]); + } + } + + IMemoryOwner ReadItem(HeifItem item) + { + byte[] payload = payloads[item.Id]; + IMemoryOwner owner = Configuration.Default.MemoryAllocator.Allocate(payload.Length); + payload.CopyTo(owner.Memory.Span); + return owner; + } + } + + [Fact] + public void GridDecoderRejectsCellSmallerThanMiafMinimum() + { + ObuColorConfig colorConfig = new() + { + IsColorDescriptionPresent = true, + ColorPrimaries = ObuColorPrimaries.Bt709, + TransferCharacteristics = ObuTransferCharacteristics.Srgb, + MatrixCoefficients = ObuMatrixCoefficients.Bt709, + ColorRange = true, + IsMonochrome = true, + SubSamplingX = true, + SubSamplingY = true, + BitDepth = Av1BitDepth.EightBit + }; + + InvalidImageContentException exception = Assert.Throws( + () => + { + using Image decoded = DecodeSingleCellGrid(63, 64, colorConfig); + }); + + Assert.Contains("grid cells must be at least 64 samples", exception.Message, StringComparison.Ordinal); + } + + /// + /// Verifies grid dimensions preserve chroma alignment for AV1's 4:2:2 and 4:2:0 layouts. + /// + [Theory] + [InlineData(65, 64, true, false)] + [InlineData(65, 64, true, true)] + [InlineData(64, 65, true, true)] + public void GridDecoderRejectsOddSubsampledDimension( + int width, + int height, + bool subsamplingX, + bool subsamplingY) + { + ObuColorConfig colorConfig = new() + { + IsColorDescriptionPresent = true, + ColorPrimaries = ObuColorPrimaries.Bt709, + TransferCharacteristics = ObuTransferCharacteristics.Srgb, + MatrixCoefficients = ObuMatrixCoefficients.Bt709, + ColorRange = true, + BitDepth = Av1BitDepth.EightBit, + SubSamplingX = subsamplingX, + SubSamplingY = subsamplingY + }; + + InvalidImageContentException exception = Assert.Throws( + () => + { + using Image decoded = DecodeSingleCellGrid(width, height, colorConfig); + }); + + Assert.Contains("must be even", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Av1OversizedStillImageWritesAndDecodesGrid() + { + const int width = 65537; + using Image image = new(width, 1); + image[0, 0] = new Rgb24(1, 2, 3); + image[32768, 0] = new Rgb24(11, 13, 17); + image[32769, 0] = new Rgb24(19, 23, 29); + image[width - 1, 0] = new Rgb24(31, 37, 41); + + // Identity-matrix 4:4:4 makes the lossless AV1 cells preserve the packed RGB channels exactly. + image.Metadata.CicpProfile = new CicpProfile(1, 13, 0, true); + using MemoryStream stream = new(); + HeifEncoder encoder = new() + { + CompressionMethod = HeifCompressionMethod.Av1, + Lossless = true, + Effort = 0 + }; + + image.Save(stream, encoder); + byte[] file = stream.ToArray(); + Assert.Equal( + [0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1], + GetItemPayload(file, 1).ToArray()); + + using Av1Decoder firstCellDecoder = new(Configuration.Default); + using Image firstCell = firstCellDecoder.Decode(GetItemPayload(file, 2)); + using Av1Decoder secondCellDecoder = new(Configuration.Default); + using Image secondCell = secondCellDecoder.Decode(GetItemPayload(file, 3)); + + // AVIF requires the first grid cell to be at least 64 samples on both axes. The derived grid trims + // the replicated right and bottom edges back to the presentation encoded in its descriptor. + Assert.Equal(new Size(32769, 64), firstCell.Size); + Assert.Equal(new Size(32769, 64), secondCell.Size); + Assert.Equal(image[32768, 0], firstCell[32768, 0]); + Assert.Equal(image[32769, 0], secondCell[0, 0]); + Assert.Equal(image[width - 1, 0], secondCell[32767, 0]); + Assert.Equal(secondCell[32767, 0], secondCell[32768, 0]); + Assert.Equal(secondCell[0, 0], secondCell[0, 63]); + Assert.Equal(1U, GetItemInfoFlags(file, 2)); + Assert.Equal(1U, GetItemInfoFlags(file, 3)); + + ReadOnlySpan references = GetMetadataChild(file, Heif4CharCode.Iref); + const int FirstReferenceOffset = 12; + Assert.Equal( + Heif4CharCode.Dimg, + (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(references[(FirstReferenceOffset + 4)..])); + + Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 8)..])); + Assert.Equal(2, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 10)..])); + Assert.Equal(2, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 12)..])); + Assert.Equal(3, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 14)..])); + + stream.Position = 0; + using Image decoded = Image.Load(stream); + Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded)); } [Fact] @@ -335,9 +641,13 @@ public class HeifEncoderTests => (ushort)(((sample * (long)ushort.MaxValue) + (maximum / 2)) / maximum); [Theory] - [InlineData((ushort)0)] - [InlineData((ushort)3)] - public void Av1LosslessImageSequencePreservesFramesTimingAndAlpha(ushort repeatCount) + [InlineData((ushort)0, null, (ushort)0)] + [InlineData((ushort)3, null, (ushort)3)] + [InlineData((ushort)3, (ushort)7, (ushort)7)] + public void Av1LosslessImageSequencePreservesFramesTimingAndAlpha( + ushort metadataRepeatCount, + ushort? encoderRepeatCount, + ushort expectedRepeatCount) { const int width = 8; const int height = 8; @@ -376,24 +686,36 @@ public class HeifEncoderTests image.Metadata.ExifProfile = exifProfile; byte[] xmpData = Encoding.UTF8.GetBytes("ImageSharp AV1 sequence"); image.Metadata.XmpProfile = new XmpProfile(xmpData); - image.Metadata.GetHeifMetadata().RepeatCount = repeatCount; + image.Metadata.GetHeifMetadata().RepeatCount = metadataRepeatCount; using MemoryStream stream = new(); HeifEncoder encoder = new() { CompressionMethod = HeifCompressionMethod.Av1, Lossless = true, - Effort = 0 + Effort = 0, + RepeatCount = encoderRepeatCount }; image.Save(stream, encoder); byte[] file = stream.ToArray(); Assert.Equal((uint)Heif4CharCode.Avis, BinaryPrimitives.ReadUInt32BigEndian(file.AsSpan(8))); + Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(GetMetadataChild(file, Heif4CharCode.Pitm)[12..])); + + using (Av1Decoder sampleDecoder = new(Configuration.Default)) + using (Image decodedSample = sampleDecoder.Decode(GetItemPayload(file, 1))) + { + ObuSequenceHeader sampleHeader = sampleDecoder.SequenceHeader; + Assert.NotNull(sampleHeader); + Assert.False(sampleHeader.IsStillPicture); + Assert.False(sampleHeader.IsReducedStillPictureHeader); + } stream.Position = 0; DecoderOptions preserveOptions = new() { ColorProfileHandling = ColorProfileHandling.Preserve }; using Image decoded = Image.Load(preserveOptions, stream); Assert.Equal(frameCount, decoded.Frames.Count); - Assert.Equal(repeatCount, decoded.Metadata.GetHeifMetadata().RepeatCount); + Assert.Equal(expectedRepeatCount, decoded.Metadata.GetHeifMetadata().RepeatCount); + Assert.True(decoded.Metadata.GetHeifMetadata().AnimateRootFrame); Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded)); Assert.Equal( IccTestDataProfiles.ProfileRandomArray, @@ -483,7 +805,7 @@ public class HeifEncoderTests image.Save(stream, encoder); byte[] file = stream.ToArray(); - ReadOnlySpan fileType = file.AsSpan(0, 28); + ReadOnlySpan fileType = GetTopLevelBox(file, Heif4CharCode.Ftyp); Assert.Equal(28, BinaryPrimitives.ReadInt32BigEndian(fileType)); Assert.Equal(Heif4CharCode.Ftyp, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[4..])); Assert.Equal(Heif4CharCode.Avif, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[8..])); @@ -997,6 +1319,49 @@ public class HeifEncoderTests ipmaPayload[8..].ToArray()); } + [Fact] + public void ItemPropertiesReuseAnEarlierIdenticalCellPropertySet() + { + ObuSequenceHeader sequenceHeader = new() + { + SequenceProfile = ObuSequenceProfile.Main, + OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }], + ColorConfig = new ObuColorConfig + { + BitDepth = Av1BitDepth.EightBit, + SubSamplingX = false, + SubSamplingY = false + } + }; + + HeifItem firstCell = new(Heif4CharCode.Av01, 1) + { + ChannelBitDepths = [8, 8, 8], + Av1CodecConfiguration = new Av1CodecConfiguration(sequenceHeader) + }; + + firstCell.SetExtent(new Size(64, 64)); + HeifItem secondCell = new(Heif4CharCode.Av01, 2) + { + PropertySource = firstCell + }; + + secondCell.SetExtent(firstCell.Extent); + List items = [firstCell, secondCell]; + int expectedLength = HeifEncoderCore.GetItemPropertiesBoxLength(items); + using IMemoryOwner owner = Configuration.Default.MemoryAllocator.Allocate(expectedLength); + Span propertyBox = owner.Memory.Span[..expectedLength]; + int length = HeifEncoderCore.WriteItemPropertiesBox(propertyBox, 0, items); + + Assert.Equal(92, length); + const int IpcoOffset = 8; + int ipmaOffset = IpcoOffset + BinaryPrimitives.ReadInt32BigEndian(propertyBox[IpcoOffset..]); + ReadOnlySpan ipmaPayload = propertyBox[(ipmaOffset + 8)..]; + Assert.Equal( + [0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 3, 1, 2, 0x83, 0, 2, 3, 1, 2, 0x83], + ipmaPayload.ToArray()); + } + [Fact] public void Av1ItemPropertiesWriteIccBeforeCicpAndExcludeMetadataItemsFromAssociations() { @@ -1188,6 +1553,87 @@ public class HeifEncoderTests throw new InvalidImageContentException($"The encoded file has no payload for item {itemId}."); } + private static uint GetItemInfoFlags(Span file, ushort itemId) + { + ReadOnlySpan itemInformation = GetMetadataChild(file, Heif4CharCode.Iinf); + int entryOffset = 14; + while (entryOffset < itemInformation.Length) + { + int entrySize = BinaryPrimitives.ReadInt32BigEndian(itemInformation[entryOffset..]); + ReadOnlySpan entry = itemInformation.Slice(entryOffset, entrySize); + ushort currentItemId = BinaryPrimitives.ReadUInt16BigEndian(entry[12..]); + if (currentItemId == itemId) + { + return (uint)((entry[9] << 16) | (entry[10] << 8) | entry[11]); + } + + entryOffset += entrySize; + } + + throw new InvalidImageContentException($"The encoded file has no item-information entry for item {itemId}."); + } + + private static Image DecodeSingleCellGrid(int width, int height, ObuColorConfig colorConfig) + { + using Image tile = new(width, height, new Rgba32(127, 127, 127)); + using MemoryStream payloadStream = new(); + ObuSequenceHeader header = Av1FrameEncoder.Encode( + Configuration.Default, + tile.Frames.RootFrame, + payloadStream, + colorConfig, + qIndex: 0, + effort: 0); + + byte[] payload = payloadStream.ToArray(); + HeifItem gridItem = new(Heif4CharCode.Grid, 1); + gridItem.SetExtent(new Size(width, height)); + HeifItem tileItem = new(Heif4CharCode.Av01, 2) + { + Av1CodecConfiguration = new Av1CodecConfiguration(header) + }; + + tileItem.SetExtent(new Size(width, height)); + HeifItemLink gridLink = new(Heif4CharCode.Dimg, gridItem.Id); + gridLink.DestinationIds.Add(tileItem.Id); + GridHeifItemDecoder decoder = new([gridItem, tileItem], [gridLink], ReadItem); + byte[] descriptor = new byte[8]; + BinaryPrimitives.WriteUInt16BigEndian(descriptor.AsSpan(4), (ushort)width); + BinaryPrimitives.WriteUInt16BigEndian(descriptor.AsSpan(6), (ushort)height); + return decoder.DecodeItemData( + new DecoderOptions { Configuration = Configuration.Default }, + gridItem, + descriptor, + null, + TestContext.Current.CancellationToken); + + IMemoryOwner ReadItem(HeifItem item) + { + Assert.Equal(tileItem.Id, item.Id); + IMemoryOwner owner = Configuration.Default.MemoryAllocator.Allocate(payload.Length); + payload.CopyTo(owner.Memory.Span); + return owner; + } + } + + private static Span GetTopLevelBox(Span file, Heif4CharCode requestedType) + { + int offset = 0; + while (offset < file.Length) + { + int boxSize = BinaryPrimitives.ReadInt32BigEndian(file[offset..]); + Heif4CharCode boxType = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(file[(offset + sizeof(uint))..]); + if (boxType == requestedType) + { + return file.Slice(offset, boxSize); + } + + offset += boxSize; + } + + throw new InvalidImageContentException($"The encoded file has no {requestedType} top-level box."); + } + private static Span GetMetadataChild(Span file, Heif4CharCode childType) { int offset = 0;