diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 11d36c31c6..2d216272ac 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -830,7 +830,7 @@ Encoder verification contract: - [ ] Implement inter mode search for bounded sequences, including reference selection and the decoder-supported inter tools. - [~] Current-libaom `av1_quantize_fp_no_qmatrix` arithmetic is implemented as a closed generic forward-quantizer family with Vector512, Vector256, Vector128, and scalar paths, raster-order output, coded 64-point coefficient limits, and scan-order EOB selection. Transform search, coefficient optimization, and lossless behavior remain. - [~] Implement real rate-distortion selection and make quality and effort change work, size, and output quality. The complete luma and joint chroma candidate sets, including chroma-from-luma, filter-intra, palette, and intra-block copy, now perform live rate-distortion selection; quality mapping, effort tiers above the current uniform-transform search ceiling, and the remaining searches are not implemented. -- [~] Frame effort now progressively expands the available current search: zero is DC-only, one adds every zero-angle spatial mode, two adds every legal directional adjustment, three adds transform refinement, four adds filter-intra and chroma-from-luma, and five adds adaptive palette and intra-block-copy analysis. Lower tiers do not signal unavailable sequence or frame tools, and tiers below five skip the whole-frame screen-content scan. Effort six enables `TX_MODE_SELECT` and compares the winning ordinary spatial luma mode as one 8x8 transform against four raster-ordered 4x4 transforms. Every 4x4 transform searches all legal transform types with live coefficient contexts and reconstructed intra references. The search reuses the aligned block workspace, preserves only improving candidates, and performs no per-block or per-transform rent. Non-skipped intra-block copy writes and costs the current-libaom unsplit variable-transform root; skipped intra-block copy emits no transform-partition symbol. Values seven through ten currently share the effort-six ceiling. Transform-size integration for filter-intra and palette, broader joint mode/transform refinement, partition search, and pruning remain. Ten decoder-visible production cases verify emitted flags, mode restrictions, real 4x4 selection, intra-block-copy syntax, and successful decode. The exact net11 Release test-project build remains at 1,005 baseline warnings and zero errors; the clean complete non-HEVC HEIF/AV1 namespace passes 9,294 of 9,294 through direct foreground VSTest. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts both new effort-six payloads in addition to the previously verified set; their decoded-frame MD5 values are `2dd1cbe449fe2d0471dc2c15c50acb69` for spatial 4x4 transform selection and `677435e5af39c930af1178f91c34af6a` for intra-block copy. Roslynk reports zero compiler errors and no analyzer diagnostics in the touched files. +- [~] Frame effort now progressively expands the available current search: zero is DC-only, one adds every zero-angle spatial mode, two adds every legal directional adjustment, three adds transform refinement, four adds filter-intra and chroma-from-luma, and five adds adaptive palette and intra-block-copy analysis. Lower tiers do not signal unavailable sequence or frame tools, and tiers below five skip the whole-frame screen-content scan. Effort six enables `TX_MODE_SELECT` and compares the winning ordinary spatial or filter-intra luma mode as one 8x8 transform against four raster-ordered 4x4 transforms. Each luma palette candidate performs that same transform-size comparison before competing with other palette sizes. Every 4x4 transform searches all legal transform types with live coefficient contexts and reconstructed intra references. The search reuses the aligned block workspace, preserves only improving candidates, and performs no per-block or per-transform rent. Non-skipped intra-block copy writes and costs the current-libaom unsplit variable-transform root; skipped intra-block copy emits no transform-partition symbol. Values seven through ten currently share the effort-six ceiling. Broader joint mode/transform refinement, partition search, and pruning remain. Decoder-visible production cases verify emitted flags, mode restrictions, real 4x4 selection, filter-intra, palette, intra-block-copy syntax, and successful decode. The latest incremental net11 Release build reports three warnings and zero errors, and the complete non-HEVC HEIF/AV1 namespace passes 9,298 of 9,298 through direct foreground VSTest. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts the generated 8-bit and 12-bit palette transform-size payloads and the effort-six intra-block-copy payload. Roslynk reports zero compiler errors. - [~] Encoder rate accounting converts the entropy writer's live inverse cumulative distributions into current-libaom fixed-point symbol costs without allocating or duplicating probability state. Read-only luma-mode, directional-delta, filter-intra, chroma-mode, block-skip, transform-size, transform-block-skip, and complete transform-coefficient queries share the exact distributions mutated by the subsequent entropy write. Complete coefficient costing follows current libaom's optimized shape: it returns immediately for an empty transform, uses the EOB-specific base-range context, fuses magnitude, sign, base-range, and Golomb accounting into one reverse traversal, and combines repeated full base-range chunks instead of replaying each emitted symbol. Tile-lifetime level and context scratch is reused, the one-coefficient path neither clears nor initializes the forward-neighbor level map, and steady-state queries allocate nothing. Transform-size writing and costing share one subdivision-depth calculation, while shared closed symbol operations keep the writer and cost mappings for transform skip, transform type, and EOB syntax identical without forcing the estimator through the writer's slower two-pass coefficient traversal. The current-libaom fixed-point RD combiner preserves 64-bit distortion and rounds the weighted 1/512-bit rate at the required boundary. Its key-frame multiplier follows libaom's squared DC-quantizer formula and exact 10/12-bit normalization. Live final-block selection evaluates all 61 legal 8x8 luma candidates: the 13 zero-angle base modes in current-libaom order, followed by six nonzero adjustments for each directional mode. Joint chroma selection evaluates the equivalent 61 spatial candidates, combines U and V distortion plus coefficient rate, and charges one live chroma-mode and shared-angle symbol over the actual subsampled 4x4, 4x8, or 8x8 geometry. Chroma-from-luma subsamples the reconstructed luma block once into fixed-stride Q3 stack scratch, subtracts the rounded mean, evaluates all 33 signed alpha values independently for each plane with complete transform RD, and combines the cached plane results across all 1,088 valid joint pairs with one live sign cost and the conditional U/V magnitude costs. This is the allocation-free equivalent of current libaom's exhaustive 33-value path: it requires 66 evaluation transforms rather than transforming every joint pair, preserves DC-before-CfL-before-spatial tie order, and fixes the implicit chroma transform to DCT-DCT. Filter-intra follows ordinary luma candidates, searches all five predictors in syntax order, and evaluates every legal transform while reusing one prepared prediction and source residual per filter mode. Every candidate includes its live mode, angle, filter mode, alpha, and coefficient rate plus normalized pixel-domain distortion. Each prepared reference edge retains the common-corner prefix and twice the transform dimension required by directional prediction. A shared encoder/decoder availability calculation selects reconstructed top-right and bottom-left extensions according to tile, frame, superblock, and block reconstruction order; unavailable extensions repeat the nearest coded endpoint. Missing top or left edges retain current libaom's perpendicular-sample and bit-depth-midpoint rules. Directional prediction applies the AV1 three-degree adjustment step and reuses transform workspace for zone-three transposition before the transform overwrites it, keeping candidate evaluation allocation-free. The winning luma and chroma signed adjustments are retained in the packed final-block state consumed by the tile writer. The tile writer invokes these reusable workspace-backed selectors after mapping current neighbors and immediately before writing each block, so later decisions see reconstructed samples, coefficient contexts, and CDF updates from every preceding block. Block skip is read only after the callback has combined every coded plane. Luma and chroma candidate scratch is partitioned from the encoder's single aligned reusable block workspace; transform-size search uses that owner for four retained 4x4 transform states, local coefficient contexts, and the compact trial reconstruction needed to preserve the best result. No candidate path rents a buffer per block or per transform. Only a newly winning candidate is copied into retained frame storage. Production fixtures force every luma base predictor, both extreme adjustments in all three directional zones, available top-right and bottom-left extensions, high-bit-depth adjustment propagation, exact signed luma and chroma angle-rate terms, joint U/V decisions, packed chroma state, and 4:2:0, 4:2:2, and 4:4:4 transform geometry. The CfL fixtures derive target chroma from a pilot production encode's actual reconstructed luma through an independent scalar Q3 oracle and prove exact positive/negative alpha syntax plus zero-residual DCT-DCT reconstruction for all three subsampling geometries at 8, 10, and 12 bits. The stable fixed-DC traversal comparison uses neutral samples for which both the baseline and live search are contractually DC and skipped, instead of relying on textured content to happen to select the baseline mode. Luma palette selection now evaluates dominant-color and one-dimensional K-means candidates for every legal size, snaps near-cache colors with the reference threshold and tie order, removes duplicate snapped colors, extends boundary maps from active samples, and performs complete transform rate-distortion search. Ordinary DC and filter-intra candidates pay the palette-disabled symbol whenever screen-content syntax is enabled. The exact net11 Release rebuild reports 1,992 test-project warnings and zero errors, all 58 intra-superblock cases pass, all 8,935 AVIF cases pass, and all 230 HEIF cases pass. Remaining mode decision work includes transform-size coverage for filter-intra and palette, broader joint mode/transform refinement, partition search, and effort-dependent pruning. Non-empty intra blocks deliberately remain non-skipped, matching current libaom; later inter mode selection owns its distinct skip-transform RD decision. - [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Partition, transform, and coefficient neighbor state retains only the above and left context regions used by current libaom; the unused third top-left region, its granularity state, and its unused sentinel are removed. One picture owner now packs segmentation plus every tile's partition, luma, chroma, and transform edges into one clean byte allocation with typed non-owning views; together with the separately typed packed mode-information owner, the complete picture state uses two allocator rents rather than seven. Exact aligned lengths, clean initialization, and balanced exactly-once returns are covered in Release. Multi-tile payload ownership and verified CDF update behavior remain. - [~] Encoder mode information now uses a frame-owned integer alias grid over a packed 8-byte value allocation, matching current libaom's `mi_grid_base` and `mi_alloc` relationship without a managed object or reference per 4x4 entry. The visible dimensions are aligned to eight luma samples, the grid stride and allocated row count are aligned to 32 mode-information units, and optional 8x8 allocation granularity reduces the value store in both dimensions exactly as current libaom does. One clean ImageSharp byte owner contains both independently typed regions, reducing libaom's two allocation lifetimes to one without a copy. At 4K, the 4x4 layout occupies about 6.0 MiB in total; the 8x8 layout occupies about 3.0 MiB. Exact geometry, clean allocation, typed lengths, aligned mapping, untouched row padding, and exactly-once return pass 4 of 4 direct net11 VSTest cases in Release. Every coded 4x4 cell covered by square, rectangular, or clipped edge blocks maps to its owning allocation entry before context-dependent symbols are written. Packed syntax, relative neighbor lookup, full block mapping, writer traversal, entropy, and OBU coverage pass 1,947 of 1,947 direct net11 VSTest cases in Release; complete mode decision still remains. @@ -868,7 +868,7 @@ Encoder verification contract: - [x] Operation-wide allocation tracking now exercises a real 64x64 12-bit 4:4:4 frame through packed-pixel conversion, both native frame owners, picture and coefficient state, reusable block workspaces, entropy coding, OBU framing, and a non-seekable destination. It proves exactly one 60 KiB tile-output reservation from current libaom's all-intra 2.5x rule and balanced exactly-once returns for every tracked allocation before the operation completes. The focused ownership case passes 1 of 1 and the complete HEIF/AV1 namespace passes 8,863 of 8,863 direct net11 VSTest cases with zero failures or skips. - [x] HEIF box offsets are now counted from the start of the encoded file instead of reading `Stream.Position`. This preserves ISO BMFF file-relative `iloc` offsets when the destination begins at a nonzero position and permits non-seekable output. Decoder item extents and image-sequence chunk offsets now resolve from that same file origin rather than the backing stream origin. Real legacy-JPEG HEIF round trips cover non-seekable output and a prefixed destination, while current-position AV1 decode covers both a still item and a five-frame sequence. All 96 encoder/decoder cases and all 38 sequence-parser cases pass direct net11 Release VSTest; the Release build remains at the established 1,005-warning baseline with zero errors. -- [~] Effort-six uniform luma transform selection now compares the winning ordinary spatial or filter-intra mode as one 8x8 transform against four raster-ordered 4x4 transforms. Each 4x4 transform searches every legal type with coefficient contexts derived from retained transform edges and preceding trial blocks, while reconstructed top-right and bottom-left references follow production coding order. Filter-intra trials rebuild each recursive prediction from those reconstructed edges and charge the retained filter mode in both header and coefficient syntax. The strided transform operator writes each candidate directly into its 8x8 reconstruction mosaic, and the existing aligned block-workspace owner retains prediction, residual, coefficients, contexts, compact reconstruction, and four final states; no allocator rent, managed array, best-candidate re-transform, or full-block intermediate copy was added. Non-skipped intra-block copy under `TX_MODE_SELECT` emits and costs the unsplit variable-transform root required by current libaom, while skipped intra-block copy emits no transform-partition symbol. Uniform transform-size contexts use coding-block extents for intra-block-copy neighbors and residual-transform extents for intra neighbors. The 18-case filter-intra group passes at 8, 10, and 12 bits, including full decoder syntax checks for a selected four-transform filter block, and the complete non-HEVC HEIF/AV1 namespace passes 9,297 of 9,297 cases with zero failures or skips. The exact Release build remains at 1,005 baseline warnings and zero errors. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts the three new filter-intra streams and both earlier effort-six streams; the earlier decoded-frame MD5 values remain `2dd1cbe449fe2d0471dc2c15c50acb69` and `677435e5af39c930af1178f91c34af6a`. Transform-size integration for palette, broader joint mode/transform refinement, partition search, and effort-dependent pruning remain. +- [~] Effort-six uniform luma transform selection now compares ordinary spatial and filter-intra winners, plus every individual luma palette candidate, as one 8x8 transform against four raster-ordered 4x4 transforms. Searching transform size inside each palette candidate matches current libaom's per-candidate uniform-transform decision and prevents an 8x8-only preliminary result from discarding the palette that is optimal with 4x4 residuals. Each 4x4 transform searches every legal type with coefficient contexts derived from retained transform edges and preceding trial blocks, while reconstructed top-right and bottom-left references follow production coding order. Palette prediction uses non-owning subregions of the retained color map, and filter-intra rebuilds each recursive prediction from reconstructed edges. The strided transform operator writes directly into the 8x8 reconstruction mosaic, and the existing aligned block-workspace owner retains prediction, residual, coefficients, contexts, compact reconstruction, and four final states; no allocator rent, managed array, best-candidate re-transform, or full-block intermediate copy was added. The source comparison also exposed that encoder transform edges began at zero while libaom and the ImageSharp decoder initialize unavailable edges to the largest transform. The packed encoder edge storage now initializes to 64, so variable-transform partition contexts agree before a coded neighbor publishes its size. Variable transform syntax is also gated to blocks larger than 4x4, matching libaom's `block_signals_txsize`. The focused Release verification passes 5 of 5 cases, including 8-bit and 12-bit palette split selection, clipped palette blocks, intra-block copy, and transform-size edge contexts. The complete non-HEVC HEIF/AV1 namespace passes 9,298 of 9,298 cases with zero failures or skips. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts both generated palette transform-size streams and the effort-six intra-block-copy stream. Broader joint mode/transform refinement, partition search, and effort-dependent pruning remain. ### 7. Write complete AVIF output diff --git a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs index 5b8781dbf6..8062fb7793 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs @@ -785,7 +785,7 @@ internal class Av1SymbolEncoder : IDisposable ushort endOfBlock, bool useReducedTransformSet, Av1FilterIntraMode filterIntraMode, - bool usesInterTransformSet = false) + bool usesInterTransformSet) { Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize); @@ -938,7 +938,7 @@ internal class Av1SymbolEncoder : IDisposable ushort endOfBlock, bool useReducedTransformSet, Av1FilterIntraMode filterIntraMode, - bool usesInterTransformSet = false) + bool usesInterTransformSet) { Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize); @@ -1384,7 +1384,7 @@ internal class Av1SymbolEncoder : IDisposable int baseQIndex, Av1FilterIntraMode filterIntraMode, Av1PredictionMode intraDirection, - bool usesInterTransformSet = false) + bool usesInterTransformSet) => this.ProcessTransformType( transformType, transformSize, @@ -1411,7 +1411,7 @@ internal class Av1SymbolEncoder : IDisposable int baseQIndex, Av1FilterIntraMode filterIntraMode, Av1PredictionMode intraDirection, - bool usesInterTransformSet = false) + bool usesInterTransformSet) { _ = this.ProcessTransformType( transformType, diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs index 4d8ede5d56..df02652215 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs @@ -527,7 +527,8 @@ internal static partial class Av1IntraSuperblockEncoder context, state.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); return distortion; } @@ -631,7 +632,8 @@ internal static partial class Av1IntraSuperblockEncoder blueContext, candidateBlueState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); rate += writer.GetCoefficientCost( transformSize, @@ -642,7 +644,8 @@ internal static partial class Av1IntraSuperblockEncoder redContext, candidateRedState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs index 0d853ea876..f67f950310 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs @@ -311,7 +311,8 @@ internal static partial class Av1IntraSuperblockEncoder blueContext, candidateBlueState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); rate += writer.GetCoefficientCost( transformSize, @@ -322,7 +323,8 @@ internal static partial class Av1IntraSuperblockEncoder redContext, candidateRedState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); if (candidateCost < bestCost) diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs index d311d3722d..6ba0014779 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs @@ -188,6 +188,7 @@ internal static partial class Av1IntraSuperblockEncoder 0, 0, LumaTransformSize, + Av1TransformType.AllTransformTypes, lumaContext, workspace.LumaPrediction, workspace.Residual, @@ -216,6 +217,19 @@ internal static partial class Av1IntraSuperblockEncoder Av1EncoderTransformBlockState emptyRedState = default; if (!this.source.IsMonochrome) { + Av1TransformType chromaTransformType = lumaCandidateState.TransformType; + Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType( + chromaTransformSize, + isInter: true, + this.picture.Parent.FrameHeader.UseReducedTransformSet); + + // Inter prediction does not signal an independent chroma transform type. Chroma reuses the + // selected luma type when that type belongs to its transform set and otherwise falls back to DCT. + if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet)) + { + chromaTransformType = Av1TransformType.DctDct; + } + this.EvaluateIntraBlockCopyPlane( writer, candidate, @@ -225,6 +239,7 @@ internal static partial class Av1IntraSuperblockEncoder subsamplingX, subsamplingY, chromaTransformSize, + chromaTransformType, blueContext, workspace.BluePrediction, workspace.Residual, @@ -248,6 +263,7 @@ internal static partial class Av1IntraSuperblockEncoder subsamplingX, subsamplingY, chromaTransformSize, + chromaTransformType, redContext, workspace.RedPrediction, workspace.Residual, @@ -424,6 +440,7 @@ internal static partial class Av1IntraSuperblockEncoder int subsamplingX, int subsamplingY, Av1TransformSize transformSize, + Av1TransformType transformTypeSelection, Av1TransformBlockContext blockContext, Span prediction, Span residual, @@ -461,6 +478,13 @@ internal static partial class Av1IntraSuperblockEncoder isInter: true, this.picture.Parent.FrameHeader.UseReducedTransformSet); + Av1TransformType firstTransformType = transformTypeSelection == Av1TransformType.AllTransformTypes + ? Av1TransformType.DctDct + : transformTypeSelection; + Av1TransformType transformTypeLimit = transformTypeSelection == Av1TransformType.AllTransformTypes + ? Av1TransformType.AllTransformTypes + : (Av1TransformType)((int)transformTypeSelection + 1); + long bestCost = long.MaxValue; selectedState = default; selectedRate = 0; @@ -468,8 +492,8 @@ internal static partial class Av1IntraSuperblockEncoder hasEmptyTransform = false; emptyState = default; emptyDistortion = 0; - for (Av1TransformType transformType = Av1TransformType.DctDct; - transformType < Av1TransformType.AllTransformTypes; + for (Av1TransformType transformType = firstTransformType; + transformType < transformTypeLimit; transformType++) { if (!transformType.IsExtendedSetUsed(transformSetType)) diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs index fcfc36c29e..ca9d5b0629 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs @@ -389,7 +389,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, 0, this.picture.Parent.FrameHeader.UseReducedTransformSet, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); } Span contexts = this.blockWorkspace @@ -423,7 +424,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, 0, this.picture.Parent.FrameHeader.UseReducedTransformSet, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); topContexts.Slice(blockColumn, transformWidth).Clear(); leftContexts.Slice(blockRow, transformHeight).Clear(); @@ -788,6 +790,7 @@ internal static partial class Av1IntraSuperblockEncoder } } + selectedTransformSize = TransformSize; if (this.effort >= 5 && this.picture.Parent.FrameHeader.AllowScreenContentTools && this.SelectLumaPalette( @@ -800,19 +803,20 @@ internal static partial class Av1IntraSuperblockEncoder transformSetType, blockContext, largestTransformRate, + transformSizeContext, candidateReconstruction, candidateCoefficients, retainedCoefficients, - ref retainedStates[0], + retainedStates, ref bestTransformCost, - ref paletteInfo)) + ref paletteInfo, + ref selectedTransformSize)) { bestMode = Av1PredictionMode.DC; selectedAngleDelta = 0; selectedFilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; } - selectedTransformSize = TransformSize; if (this.effort >= 6 && this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select && paletteInfo.PaletteSizes[0] == 0) @@ -827,6 +831,9 @@ internal static partial class Av1IntraSuperblockEncoder bestMode, selectedAngleDelta, selectedFilterIntraMode, + 0, + ReadOnlySpan.Empty, + 0, paletteDisabledCost, transformSizeContext, bestTransformCost, @@ -864,6 +871,9 @@ internal static partial class Av1IntraSuperblockEncoder Av1PredictionMode mode, int angleDelta, Av1FilterIntraMode filterIntraMode, + int paletteSize, + scoped ReadOnlySpan paletteColors, + int paletteHeaderRate, int paletteDisabledCost, int transformSizeContext, long costLimit, @@ -902,19 +912,34 @@ internal static partial class Av1IntraSuperblockEncoder TransformSize, useReducedTransformSet); - int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta); - rate += writer.GetTransformSizeCost(BlockSize, TransformSize, transformSizeContext); - if (mode == Av1PredictionMode.DC) + int rate = writer.GetTransformSizeCost(BlockSize, TransformSize, transformSizeContext); + if (paletteSize > 0) { - rate += paletteDisabledCost; - if (this.picture.Sequence.SequenceHeader.EnableFilterIntra) + rate += paletteHeaderRate; + } + else + { + rate += Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta); + if (mode == Av1PredictionMode.DC) { - rate += writer.GetFilterIntraModeCost( - filterIntraMode, - BlockSize); + rate += paletteDisabledCost; + if (this.picture.Sequence.SequenceHeader.EnableFilterIntra) + { + rate += writer.GetFilterIntraModeCost( + filterIntraMode, + BlockSize); + } } } + Buffer2DRegion colorIndexMap = default; + if (paletteSize > 0) + { + colorIndexMap = this.superblock.Workspace + .GetPaletteMaps() + .GetMap(Av1PlaneType.Y, BlockWidth, BlockWidth); + } + long distortion = 0; for (int transformRow = 0; transformRow < 2; transformRow++) { @@ -928,52 +953,73 @@ internal static partial class Av1IntraSuperblockEncoder transformColumn * TransformWidth, transformRow * TransformWidth); - Span aboveStorage = workspace.GetReferenceSamples(0); - Span leftStorage = workspace.GetReferenceSamples(1); - this.PrepareSplitLumaReferenceSamples( - reconstructionPlane, - blockOrigin, - macroBlock, - transformRow, - transformColumn, - candidateReconstruction, - aboveStorage, - leftStorage, - out bool hasLeft, - out bool hasAbove); - - if (filterIntraMode == Av1FilterIntraMode.AllFilterIntraModes) + if (paletteSize > 0) { - TOperator.PrepareIntra( - this.blockWorkspace, + // Palette prediction is block-local. A view over the retained map avoids copying indices or + // preparing reconstructed neighbor edges that this prediction mode cannot consume. + TOperator.PreparePalette( sourcePlane, transformOrigin, + paletteColors, + colorIndexMap.GetSubRegion( + new Rectangle( + transformColumn * TransformWidth, + transformRow * TransformWidth, + TransformWidth, + TransformWidth)), prediction, - aboveStorage.Slice(1, TransformWidth * 2), - leftStorage.Slice(1, TransformWidth * 2), - hasLeft, - hasAbove, - mode, - angleDelta, residual, - TransformSize, - this.bitDepth); + TransformSize); } else { - // Filter-intra prediction is recursive within each transform unit, so rebuild it from - // the reconstructed edges established by the preceding 4x4 candidate. - TOperator.PrepareFilterIntra( - this.blockWorkspace, - sourcePlane, - transformOrigin, - prediction, - aboveStorage.Slice(1, TransformWidth * 2), - leftStorage.Slice(1, TransformWidth * 2), - residual, - filterIntraMode, - TransformSize, - this.bitDepth); + Span aboveStorage = workspace.GetReferenceSamples(0); + Span leftStorage = workspace.GetReferenceSamples(1); + this.PrepareSplitLumaReferenceSamples( + reconstructionPlane, + blockOrigin, + macroBlock, + transformRow, + transformColumn, + candidateReconstruction, + aboveStorage, + leftStorage, + out bool hasLeft, + out bool hasAbove); + + if (filterIntraMode == Av1FilterIntraMode.AllFilterIntraModes) + { + TOperator.PrepareIntra( + this.blockWorkspace, + sourcePlane, + transformOrigin, + prediction, + aboveStorage.Slice(1, TransformWidth * 2), + leftStorage.Slice(1, TransformWidth * 2), + hasLeft, + hasAbove, + mode, + angleDelta, + residual, + TransformSize, + this.bitDepth); + } + else + { + // Filter-intra prediction is recursive within each transform unit, so rebuild it from + // the reconstructed edges established by the preceding 4x4 candidate. + TOperator.PrepareFilterIntra( + this.blockWorkspace, + sourcePlane, + transformOrigin, + prediction, + aboveStorage.Slice(1, TransformWidth * 2), + leftStorage.Slice(1, TransformWidth * 2), + residual, + filterIntraMode, + TransformSize, + this.bitDepth); + } } Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts( @@ -1029,7 +1075,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, candidateState.EndOfBlock, useReducedTransformSet, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); long candidateCost = Av1RateDistortion.GetCost( this.rateMultiplier, @@ -1302,7 +1349,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, candidateState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); } @@ -1362,7 +1410,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, candidateState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs index 937847f5c5..f959245e7a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs @@ -29,12 +29,14 @@ internal static partial class Av1IntraSuperblockEncoder Av1TransformSetType transformSetType, Av1TransformBlockContext blockContext, int transformSizeRate, + int transformSizeContext, Span candidateReconstruction, Span candidateCoefficients, Span retainedCoefficients, - ref Av1EncoderTransformBlockState retainedState, + Span retainedStates, ref long bestCost, - ref Av1EncoderPaletteInfo paletteInfo) + ref Av1EncoderPaletteInfo paletteInfo, + ref Av1TransformSize selectedTransformSize) { const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; const int BlockLength = 8; @@ -137,9 +139,11 @@ internal static partial class Av1IntraSuperblockEncoder writer, macroBlock, blockOrigin, + tileIndex, transformSetType, blockContext, transformSizeRate, + transformSizeContext, samples, rows, columns, @@ -151,11 +155,12 @@ internal static partial class Av1IntraSuperblockEncoder candidateReconstruction, candidateCoefficients, retainedCoefficients, + retainedStates, retainedColorIndexMap, reconstructionPlane, - ref retainedState, ref bestCost, ref paletteInfo, + ref selectedTransformSize, ref paletteSelected); } @@ -167,9 +172,11 @@ internal static partial class Av1IntraSuperblockEncoder writer, macroBlock, blockOrigin, + tileIndex, transformSetType, blockContext, transformSizeRate, + transformSizeContext, samples, rows, columns, @@ -181,11 +188,12 @@ internal static partial class Av1IntraSuperblockEncoder candidateReconstruction, candidateCoefficients, retainedCoefficients, + retainedStates, retainedColorIndexMap, reconstructionPlane, - ref retainedState, ref bestCost, ref paletteInfo, + ref selectedTransformSize, ref paletteSelected); } else @@ -206,9 +214,11 @@ internal static partial class Av1IntraSuperblockEncoder writer, macroBlock, blockOrigin, + tileIndex, transformSetType, blockContext, transformSizeRate, + transformSizeContext, samples, rows, columns, @@ -220,11 +230,12 @@ internal static partial class Av1IntraSuperblockEncoder candidateReconstruction, candidateCoefficients, retainedCoefficients, + retainedStates, retainedColorIndexMap, reconstructionPlane, - ref retainedState, ref bestCost, ref paletteInfo, + ref selectedTransformSize, ref paletteSelected); } } @@ -245,9 +256,11 @@ internal static partial class Av1IntraSuperblockEncoder Av1SymbolEncoder writer, Av1MacroBlockD macroBlock, Point blockOrigin, + ushort tileIndex, Av1TransformSetType transformSetType, Av1TransformBlockContext blockContext, int transformSizeRate, + int transformSizeContext, ReadOnlySpan samples, int rows, int columns, @@ -259,18 +272,21 @@ internal static partial class Av1IntraSuperblockEncoder Span candidateReconstruction, Span candidateCoefficients, Span retainedCoefficients, + Span retainedStates, Span retainedColorIndexMap, Buffer2DRegion reconstructionPlane, - ref Av1EncoderTransformBlockState retainedState, ref long bestCost, ref Av1EncoderPaletteInfo paletteInfo, + ref Av1TransformSize selectedTransformSize, ref bool paletteSelected) { const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; const Av1TransformSize TransformSize = Av1TransformSize.Size8x8; const int BlockLength = 8; - Av1EncoderPaletteWorkspace workspace = - this.blockWorkspace.GetModeDecisionWorkspace().Palette; + Av1EncoderModeDecisionWorkspace modeDecisionWorkspace = + this.blockWorkspace.GetModeDecisionWorkspace(); + + Av1EncoderPaletteWorkspace workspace = modeDecisionWorkspace.Palette; int bitDepth = this.bitDepth.GetBitCount(); int cacheThreshold = 4 << (bitDepth - 8); @@ -350,7 +366,6 @@ internal static partial class Av1IntraSuperblockEncoder Av1PredictionMode.DC, 0); - rate += transformSizeRate; rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext); rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y); rate += Av1SymbolEncoder.GetPaletteYColorCost(colorCache, paletteColors, bitDepth); @@ -389,7 +404,8 @@ internal static partial class Av1IntraSuperblockEncoder this.bitDepth, ref candidateState); - int candidateRate = rate + writer.GetCoefficientCost( + int candidateRate = rate + transformSizeRate; + candidateRate += writer.GetCoefficientCost( TransformSize, transformType, Av1PredictionMode.DC, @@ -398,7 +414,8 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, candidateState.EndOfBlock, this.picture.Parent.FrameHeader.UseReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, distortion); if (candidateCost < bestCost) @@ -411,7 +428,7 @@ internal static partial class Av1IntraSuperblockEncoder retainedCoefficients, TransformSize, candidateState, - ref retainedState); + ref retainedStates[0]); for (int row = 0; row < BlockLength; row++) { @@ -421,10 +438,61 @@ internal static partial class Av1IntraSuperblockEncoder paletteInfo.PaletteSizes[0] = (byte)paletteSize; paletteInfo.SetColors(Av1Plane.Y, paletteColors); + selectedTransformSize = TransformSize; bestCost = candidateCost; paletteSelected = true; } } + + if (this.effort >= 6 && + this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select) + { + // Transform size is part of each palette candidate's RD result. Searching it here preserves + // candidates whose 4x4 residual partition wins even when their 8x8 result does not. + long splitCost = this.GetSplitLumaCandidateCost( + writer, + macroBlock, + this.source.GetPlane(Av1Plane.Y), + reconstructionPlane, + blockOrigin, + tileIndex, + Av1PredictionMode.DC, + 0, + Av1FilterIntraMode.AllFilterIntraModes, + paletteSize, + paletteColors, + rate, + 0, + transformSizeContext, + bestCost, + candidateReconstruction, + candidateCoefficients, + modeDecisionWorkspace.CandidateTransformBlocks); + + if (splitCost < bestCost) + { + CopySplitCandidate( + candidateReconstruction, + candidateCoefficients, + modeDecisionWorkspace.CandidateTransformBlocks, + reconstructionPlane, + blockOrigin, + retainedCoefficients, + retainedStates); + + for (int row = 0; row < BlockLength; row++) + { + colorIndexMap.DangerousGetRowSpan(row)[..BlockLength] + .CopyTo(retainedColorIndexMap[(row * BlockLength)..]); + } + + paletteInfo.PaletteSizes[0] = (byte)paletteSize; + paletteInfo.SetColors(Av1Plane.Y, paletteColors); + selectedTransformSize = Av1TransformSize.Size4x4; + bestCost = splitCost; + paletteSelected = true; + } + } } } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs index 4a649b1d0f..c968f85bea 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs @@ -207,6 +207,11 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable GranularityNormalLog2 = Av1Constants.ModeInfoSizeLog2 }; + // Variable-transform contexts consult both edges without separate availability flags. The largest + // transform makes an unavailable edge compare as unsplit until a coded neighbor publishes its size. + this.transformContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize); + this.transformContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize); + if (frameHeader.AllowScreenContentTools) { this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit( diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs index 58a5c5d4b5..48b4531882 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs @@ -983,7 +983,8 @@ internal partial class Av1TileWriter bool writesVariableTransformSize = !isLossless && frameHeader.TransformMode == Av1TransformMode.Select && isInter && - !macroBlockModeInfo.Block.Skip; + !macroBlockModeInfo.Block.Skip && + blockSize > Av1BlockSize.Block4x4; Av1TransformSize transformSize = isLossless ? Av1TransformSize.Size4x4 @@ -1650,6 +1651,7 @@ internal partial class Av1TileWriter Av1NeighborArrayUnit luma_dc_sign_level_coeff_na) { ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; + bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy; Span lumaCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.Y); Span lumaTransformBlocks = coefficientBuffer.GetTransformBlockSpan(superblockIndex, Av1Plane.Y); @@ -1720,7 +1722,8 @@ internal partial class Av1TileWriter blockContext, endOfBlock, frameHeader.UseReducedTransformSet, - blk_ptr.FilterIntraMode); + blk_ptr.FilterIntraMode, + usesInterTransformSet); int transformWidth = transformSize.GetWidth(); int transformHeight = transformSize.GetHeight(); @@ -1771,6 +1774,7 @@ internal partial class Av1TileWriter } ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; + bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy; Span blueCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.U); Span redCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.V); Span blueTransformBlocks = @@ -1847,7 +1851,8 @@ internal partial class Av1TileWriter blockContext, blueTransformBlock.EndOfBlock, frameHeader.UseReducedTransformSet, - blk_ptr.FilterIntraMode); + blk_ptr.FilterIntraMode, + usesInterTransformSet); coefficients = redCoefficients[entropyCodingContext.CodedAreaSuperblockUv..]; blockContext = GetTransformBlockContexts( @@ -1866,7 +1871,8 @@ internal partial class Av1TileWriter blockContext, redTransformBlock.EndOfBlock, frameHeader.UseReducedTransformSet, - blk_ptr.FilterIntraMode); + blk_ptr.FilterIntraMode, + usesInterTransformSet); cb_dc_sign_level_coeff_na.UnitModeWrite( (byte)culLevelCb, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs index 047dfb39c9..1d3e815017 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs @@ -1128,7 +1128,7 @@ public class Av1CoefficientsEntropyTests Span actuals = new int[16]; // Act - encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode); + encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode, usesInterTransformSet: false); using IMemoryOwner encoded = encoder.Exit(); @@ -1203,7 +1203,7 @@ public class Av1CoefficientsEntropyTests Span actuals = new int[16 + 1]; // Act - encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode); + encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode, usesInterTransformSet: false); using IMemoryOwner encoded = encoder.Exit(); @@ -1331,7 +1331,8 @@ public class Av1CoefficientsEntropyTests transformBlockContext, endOfBlock, useReducedTransformSet, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); using IMemoryOwner encoded = encoder.Exit(); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs index 0bb7484115..72a74e68d7 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs @@ -330,7 +330,8 @@ public class Av1EntropyTests transformBlockContext, 0, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); Assert.Equal(Av1ProbabilityCost.GetSymbolCost(transformSkip, 1), emptyCost); @@ -346,7 +347,8 @@ public class Av1EntropyTests transformBlockContext, 16, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); coefficients.Clear(); coefficients[0] = 1; @@ -371,7 +373,8 @@ public class Av1EntropyTests transformBlockContext, 1, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); Assert.Equal(expectedDcCost, dcCost); } @@ -417,7 +420,8 @@ public class Av1EntropyTests transformBlockContext, 1, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); Assert.Equal(expected, actual); } @@ -540,7 +544,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - filterIntraMode); + filterIntraMode, + usesInterTransformSet: false); Assert.Equal(expected, actual); } @@ -570,7 +575,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); Assert.Equal( initialCost, @@ -583,7 +589,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes)); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false)); int actualContext = actualEncoder.WriteCoefficients( transformSize, @@ -594,7 +601,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); int adaptedCost = actualEncoder.GetCoefficientCost( transformSize, @@ -605,7 +613,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); int expectedContext = expectedEncoder.WriteCoefficients( transformSize, @@ -616,7 +625,8 @@ public class Av1EntropyTests transformBlockContext, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); using IMemoryOwner actual = actualEncoder.Exit(); using IMemoryOwner expected = expectedEncoder.Exit(); @@ -648,7 +658,8 @@ public class Av1EntropyTests default, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); long before = GC.GetAllocatedBytesForCurrentThread(); for (int i = 0; i < 1000; i++) @@ -662,7 +673,8 @@ public class Av1EntropyTests default, endOfBlock, false, - Av1FilterIntraMode.AllFilterIntraModes); + Av1FilterIntraMode.AllFilterIntraModes, + usesInterTransformSet: false); } long after = GC.GetAllocatedBytesForCurrentThread(); @@ -825,7 +837,8 @@ public class Av1EntropyTests default, 0, false, - Av1FilterIntraMode.DC); + Av1FilterIntraMode.DC, + usesInterTransformSet: false); Assert.Equal(0, emptyContext); Assert.Single(allocator.AllocationLog); @@ -840,7 +853,8 @@ public class Av1EntropyTests default, 1, false, - Av1FilterIntraMode.DC); + Av1FilterIntraMode.DC, + usesInterTransformSet: false); Assert.Equal(3, allocator.AllocationLog.Count); @@ -853,7 +867,8 @@ public class Av1EntropyTests default, 1, false, - Av1FilterIntraMode.DC); + Av1FilterIntraMode.DC, + usesInterTransformSet: false); Assert.Equal(3, allocator.AllocationLog.Count); TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1]; @@ -1634,7 +1649,7 @@ public class Av1EntropyTests // Act foreach (Av1TransformType value in values) { - encoder.WriteTransformType(value, transformSizeContext, true, BaseQIndex, filterIntraMode, intraDirection); + encoder.WriteTransformType(value, transformSizeContext, true, BaseQIndex, filterIntraMode, intraDirection, usesInterTransformSet: false); } using IMemoryOwner encoded = encoder.Exit(); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs index 15beae25d0..9dd099f223 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs @@ -728,6 +728,7 @@ public class Av1IntraSuperblockEncoderTests 8, 8, 8, + false, (byte)32, (byte)224, 32, @@ -749,6 +750,7 @@ public class Av1IntraSuperblockEncoderTests 12, 8, 8, + false, (ushort)512, (ushort)3584, 512, @@ -770,6 +772,7 @@ public class Av1IntraSuperblockEncoderTests 8, 5, 3, + false, (byte)48, (byte)208, 48, @@ -787,6 +790,54 @@ public class Av1IntraSuperblockEncoderTests effort: 5)); } + [Fact] + public void ProductionTileSelectsLumaPaletteWithFourByFourTransforms() + { + AssertProductionTileSelectsExactLumaPalette( + Av1BitDepth.EightBit, + 8, + 8, + 8, + true, + (byte)64, + (byte)192, + 64, + 192, + static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new Av1IntraTileWriter( + Configuration.Default, + source, + reconstruction, + picture, + coefficients, + superblockWorkspace, + blockWorkspace, + initialSize: 256, + effort: 6)); + + AssertProductionTileSelectsExactLumaPalette( + Av1BitDepth.TwelveBit, + 12, + 8, + 8, + true, + (ushort)1024, + (ushort)3072, + 1024, + 3072, + static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new Av1IntraTileWriter( + Configuration.Default, + source, + reconstruction, + picture, + coefficients, + superblockWorkspace, + blockWorkspace, + initialSize: 256, + effort: 6)); + } + [Fact] public void ProductionTileSelectsExactPairedChromaPalette() { @@ -1926,49 +1977,7 @@ public class Av1IntraSuperblockEncoderTests long predictionOnlyError = 64L * 40 * 40 * sampleScale * sampleScale; Assert.InRange(reconstructionError, 1, predictionOnlyError - 1); - // Tile fixtures initialize only entropy state. Complete the same still-picture headers as the frame - // encoder before serializing so the independent decoder validates the real OBU syntax. - ObuSequenceHeader sequenceHeader = pictureTemplate.Sequence.SequenceHeader; - sequenceHeader.IsStillPicture = true; - sequenceHeader.IsReducedStillPictureHeader = true; - sequenceHeader.SequenceProfile = bitDepth == 12 - ? ObuSequenceProfile.Professional - : ObuSequenceProfile.Main; - - sequenceHeader.OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }]; - sequenceHeader.FrameWidthBits = 4; - sequenceHeader.FrameHeightBits = 4; - sequenceHeader.MaxFrameWidth = Width; - sequenceHeader.MaxFrameHeight = Height; - sequenceHeader.ForceScreenContentTools = 2; - sequenceHeader.ForceIntegerMotionVector = 2; - - ObuFrameHeader frameHeader = pictureTemplate.Parent.FrameHeader; - frameHeader.FrameType = ObuFrameType.KeyFrame; - frameHeader.ShowFrame = true; - frameHeader.ErrorResilientMode = true; - frameHeader.RefreshFrameFlags = byte.MaxValue; - frameHeader.DisableFrameEndUpdateCdf = true; - frameHeader.FrameSize = new ObuFrameSize - { - FrameWidth = Width, - FrameHeight = Height, - SuperResolutionDenominator = Av1Constants.ScaleNumerator, - SuperResolutionUpscaledWidth = Width, - RenderWidth = Width, - RenderHeight = Height - }; - - frameHeader.TilesInfo.HasUniformTileSpacing = true; - using MemoryStream stream = new(); - new ObuWriter().WriteAll( - Configuration.Default, - stream, - sequenceHeader, - frameHeader, - tileWriter); - - byte[] payload = stream.ToArray(); + byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, Width, Height); using Av1Decoder decoder = new(Configuration.Default); using Image decoded = decoder.Decode(payload); Assert.NotNull(decoder.FrameInfo); @@ -2517,6 +2526,62 @@ public class Av1IntraSuperblockEncoderTests Assert.NotEqual(0, tileWriter.GetTileData(0).Length); } + private static byte[] WriteCompleteTileObu( + Av1PictureControlSet pictureTemplate, + IAv1TileWriter tileWriter, + int width, + int height) + { + // Tile fixtures initialize only entropy state. Complete the same still-picture headers as the frame + // encoder before serializing so independent decoders validate the real OBU syntax. + ObuSequenceHeader sequenceHeader = pictureTemplate.Sequence.SequenceHeader; + ObuColorConfig colorConfig = sequenceHeader.ColorConfig; + Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); + sequenceHeader.IsStillPicture = true; + sequenceHeader.IsReducedStillPictureHeader = true; + sequenceHeader.SequenceProfile = colorConfig.BitDepth == Av1BitDepth.TwelveBit || + colorFormat == Av1ColorFormat.Yuv422 + ? ObuSequenceProfile.Professional + : colorFormat == Av1ColorFormat.Yuv444 + ? ObuSequenceProfile.High + : ObuSequenceProfile.Main; + + sequenceHeader.OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }]; + sequenceHeader.FrameWidthBits = width > 1 ? Av1Math.MostSignificantBit((uint)(width - 1)) + 1 : 1; + sequenceHeader.FrameHeightBits = height > 1 ? Av1Math.MostSignificantBit((uint)(height - 1)) + 1 : 1; + sequenceHeader.MaxFrameWidth = width; + sequenceHeader.MaxFrameHeight = height; + sequenceHeader.ForceScreenContentTools = 2; + sequenceHeader.ForceIntegerMotionVector = 2; + + ObuFrameHeader frameHeader = pictureTemplate.Parent.FrameHeader; + frameHeader.FrameType = ObuFrameType.KeyFrame; + frameHeader.ShowFrame = true; + frameHeader.ErrorResilientMode = true; + frameHeader.RefreshFrameFlags = byte.MaxValue; + frameHeader.DisableFrameEndUpdateCdf = true; + frameHeader.FrameSize = new ObuFrameSize + { + FrameWidth = width, + FrameHeight = height, + SuperResolutionDenominator = Av1Constants.ScaleNumerator, + SuperResolutionUpscaledWidth = width, + RenderWidth = width, + RenderHeight = height + }; + + frameHeader.TilesInfo.HasUniformTileSpacing = true; + using MemoryStream stream = new(); + new ObuWriter().WriteAll( + Configuration.Default, + stream, + sequenceHeader, + frameHeader, + tileWriter); + + return stream.ToArray(); + } + private static Av1PictureControlSet CreatePicture( Av1EncoderModeInfoBuffer modeInfo, ObuColorConfig colorConfig, @@ -2581,12 +2646,13 @@ public class Av1IntraSuperblockEncoderTests int bitDepthValue, int width, int height, + bool useSplitTransform, TSample lowerColor, TSample upperColor, ushort expectedLowerColor, ushort expectedUpperColor, TileWriterFactory createTileWriter) - where TSample : unmanaged + where TSample : unmanaged, IBinaryInteger { const int QIndex = 37; ObuColorConfig colorConfig = new() @@ -2619,7 +2685,32 @@ public class Av1IntraSuperblockEncoderTests for (int row = 0; row < sourcePlane.Height; row++) { int visibleRow = Math.Min(row, height - 1); - sourcePlane.DangerousGetRowSpan(row).Fill(visibleRow < height / 2 ? lowerColor : upperColor); + Span sourceRow = sourcePlane.DangerousGetRowSpan(row); + if (!useSplitTransform) + { + sourceRow.Fill(visibleRow < height / 2 ? lowerColor : upperColor); + continue; + } + + int transformRow = visibleRow >> 2; + int localRow = visibleRow & 3; + for (int column = 0; column < sourceRow.Length; column++) + { + int transformColumn = column >> 2; + int localColumn = column & 3; + int transformIndex = (transformRow * 2) + transformColumn; + int residual = transformIndex switch + { + 0 => (localRow * 2) - 3, + 1 => (localColumn * 2) - 3, + 2 => (localRow + localColumn) - 3, + _ => localRow - localColumn + }; + + int baseColor = int.CreateChecked(visibleRow < height / 2 ? lowerColor : upperColor); + sourceRow[column] = TSample.CreateChecked( + baseColor + (residual * 4 * (1 << (bitDepthValue - 8)))); + } } ClearPlane(reconstruction.Luma); @@ -2636,6 +2727,10 @@ public class Av1IntraSuperblockEncoderTests QIndex); pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true; + pictureTemplate.Parent.FrameHeader.TransformMode = useSplitTransform + ? Av1TransformMode.Select + : Av1TransformMode.Largest; + pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = width; pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = height; using Av1EncoderPictureBuffer picture = new( @@ -2664,27 +2759,96 @@ public class Av1IntraSuperblockEncoderTests ref Av1MacroBlockModeInfo mode = ref picture.Picture.GetMacroBlockModeInfo(default); Assert.Equal(Av1PredictionMode.DC, mode.Block.Mode); Assert.Equal(Av1FilterIntraMode.AllFilterIntraModes, superblockWorkspace.FinalBlocks[0].FilterIntraMode); - Assert.Equal((ushort)0, coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[0].EndOfBlock); - Assert.Equal(2, superblockWorkspace.PaletteInfo.PaletteSizes[0]); Assert.Equal( - [expectedLowerColor, expectedUpperColor], - superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y).ToArray()); + useSplitTransform ? Av1TransformSize.Size4x4 : Av1TransformSize.Size8x8, + mode.Block.TransformSize); + + Span transformStates = coefficients + .GetTransformBlockSpan(0, Av1Plane.Y)[..(useSplitTransform ? 4 : 1)]; + + if (useSplitTransform) + { + int coefficientBearingTransformCount = 0; + foreach (Av1EncoderTransformBlockState transformState in transformStates) + { + if (transformState.EndOfBlock > 0) + { + coefficientBearingTransformCount++; + } + } + + Assert.InRange(coefficientBearingTransformCount, 1, transformStates.Length); + Assert.InRange(superblockWorkspace.PaletteInfo.PaletteSizes[0], 2, Av1Constants.PaletteMaxSize); + } + else + { + Assert.Equal((ushort)0, transformStates[0].EndOfBlock); + Assert.Equal(2, superblockWorkspace.PaletteInfo.PaletteSizes[0]); + Assert.Equal( + [expectedLowerColor, expectedUpperColor], + superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y).ToArray()); + } Buffer2DRegion colorIndexMap = superblockWorkspace .GetPaletteMaps() .GetMap(Av1PlaneType.Y, 8, 8); Buffer2DRegion reconstructionPlane = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y); + ReadOnlySpan selectedPaletteColors = superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y); + long predictionOnlyError = 0; + long reconstructionError = 0; for (int row = 0; row < reconstructionPlane.Height; row++) { - int visibleRow = Math.Min(row, height - 1); - byte expectedIndex = (byte)(visibleRow < height / 2 ? 0 : 1); - foreach (byte index in colorIndexMap.DangerousGetRowSpan(row)) + if (useSplitTransform) { - Assert.Equal(expectedIndex, index); + ReadOnlySpan sourceRow = sourcePlane.DangerousGetRowSpan(row); + ReadOnlySpan reconstructionRow = reconstructionPlane.DangerousGetRowSpan(row); + ReadOnlySpan mapRow = colorIndexMap.DangerousGetRowSpan(row); + for (int column = 0; column < reconstructionRow.Length; column++) + { + long sourceSample = long.CreateChecked(sourceRow[column]); + long predictionDifference = sourceSample - selectedPaletteColors[mapRow[column]]; + long reconstructionDifference = sourceSample - long.CreateChecked(reconstructionRow[column]); + predictionOnlyError += predictionDifference * predictionDifference; + reconstructionError += reconstructionDifference * reconstructionDifference; + } } + else + { + int visibleRow = Math.Min(row, height - 1); + byte expectedIndex = (byte)(visibleRow < height / 2 ? 0 : 1); + foreach (byte index in colorIndexMap.DangerousGetRowSpan(row)) + { + Assert.Equal(expectedIndex, index); + } - Assert.True(sourcePlane.DangerousGetRowSpan(row).SequenceEqual(reconstructionPlane.DangerousGetRowSpan(row))); + Assert.True(sourcePlane.DangerousGetRowSpan(row).SequenceEqual(reconstructionPlane.DangerousGetRowSpan(row))); + } + } + + if (useSplitTransform) + { + Assert.True(predictionOnlyError > 0); + Assert.True(reconstructionError < predictionOnlyError); + byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, width, height); + using Av1Decoder decoder = new(Configuration.Default); + using Image decoded = decoder.Decode(payload); + Assert.NotNull(decoder.FrameInfo); + Av1BlockModeInfo decodedBlock = decoder.FrameInfo.GetModeInfoAt(default); + Assert.True(decodedBlock.GetPaletteSize(Av1Plane.Y) > 0); + Assert.Equal(4, decodedBlock.GetTransformUnitCount(Av1Plane.Y)); + Assert.Equal(new Size(width, height), decoded.Size); + + string outputDirectory = Path.Combine( + TestEnvironment.ActualOutputDirectoryFullPath, + "Formats", + "Heif", + "Av1"); + + Directory.CreateDirectory(outputDirectory); + File.WriteAllBytes( + Path.Combine(outputDirectory, $"encoder-palette-transform-size-select-{bitDepthValue}b.obu"), + payload); } Assert.NotEqual(0, tileWriter.GetTileData(0).Length);