From f3641ede18582d29c7b125fa41993b0e1eded738 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 24 Aug 2026 20:40:16 +1000 Subject: [PATCH] Document HEIF mode decision geometry --- .../Heif/Av1/ModeDecision/Av1BlockGeometry.cs | 68 +++++++++-- .../ModeDecision/Av1BlockGeometryFactory.cs | 114 +++++++++++++++--- .../Heif/Av1/ModeDecision/Av1GeometryIndex.cs | 34 ++++++ 3 files changed, 194 insertions(+), 22 deletions(-) diff --git a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs index af5acbd86..d0e8ca69f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs +++ b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs @@ -5,11 +5,24 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.ModeDecision; +/// +/// Describes the spatial, chroma, and transform layout of one AV1 block considered by mode-decision scanning. +/// internal class Av1BlockGeometry { + /// + /// The luma block size from which the cached luma dimensions are derived. + /// private Av1BlockSize blockSize; + + /// + /// The chroma block size from which the cached chroma dimensions are derived. + /// private Av1BlockSize blockSizeUv; + /// + /// Initializes a new instance of the class with storage for every supported transform depth. + /// public Av1BlockGeometry() { this.RedunancyList = []; @@ -20,6 +33,9 @@ internal class Av1BlockGeometry } } + /// + /// Gets or sets the luma block size and updates and to match. + /// public Av1BlockSize BlockSize { get => this.blockSize; @@ -31,6 +47,9 @@ internal class Av1BlockGeometry } } + /// + /// Gets or sets the chroma block size and updates and to match. + /// public Av1BlockSize BlockSizeUv { get => this.blockSizeUv; @@ -43,42 +62,57 @@ internal class Av1BlockGeometry } /// - /// Gets or sets the Origin point from lop left of the superblock. + /// Gets or sets the block origin in pixels relative to the top-left corner of its superblock. /// public Point Origin { get; internal set; } + /// + /// Gets or sets a value indicating whether this luma block owns chroma samples in the mode-decision layout. + /// public bool HasUv { get; internal set; } /// - /// Gets the blocks width. + /// Gets the luma block width in pixels. /// public int BlockWidth { get; private set; } /// - /// Gets the blocks height. + /// Gets the luma block height in pixels. /// public int BlockHeight { get; private set; } + /// + /// Gets the number of luma transform blocks at each transform depth. + /// public int[] TransformBlockCount { get; } = new int[Av1Constants.MaxVarTransform + 1]; + /// + /// Gets the luma transform size selected at each transform depth. + /// public Av1TransformSize[] TransformSize { get; } = new Av1TransformSize[Av1Constants.MaxVarTransform + 1]; + /// + /// Gets the chroma transform size selected at each transform depth. + /// public Av1TransformSize[] TransformSizeUv { get; } = new Av1TransformSize[Av1Constants.MaxVarTransform + 1]; + /// + /// Gets the pixel origins of the transform blocks at each transform depth. + /// public Point[][] TransformOrigin { get; private set; } /// - /// Gets or sets the blocks index in the Mode Decision scan. + /// Gets or sets the block index in mode-decision scan order. /// public int ModeDecisionIndex { get; set; } /// - /// Gets or sets the offset to the next nsq block (skip remaining d2 blocks). + /// Gets or sets the scan offset from this square block to the next block at the same depth. /// public int NextDepthOffset { get; set; } /// - /// Gets or sets the offset to the next d1 sq block + /// Gets or sets the scan offset from this square block to its first child at the next depth. /// public int Depth1Offset { get; set; } @@ -88,24 +122,42 @@ internal class Av1BlockGeometry public bool IsRedundant => this.RedunancyList.Count > 0; /// - /// Gets or sets the list where the block is redundant. + /// Gets or sets the mode-decision indices of blocks with the same size and origin as this block. /// public List RedunancyList { get; internal set; } /// - /// Gets or sets the non square index within a partition 0..totns-1 + /// Gets or sets the zero-based component index of this block within a non-square partition. /// public int NonSquareIndex { get; internal set; } + /// + /// Gets or sets the number of component blocks produced by this partition shape. + /// public int TotalNonSuareCount { get; internal set; } + /// + /// Gets the chroma block width in pixels. + /// public int BlockWidthUv { get; private set; } + /// + /// Gets the chroma block height in pixels. + /// public int BlockHeightUv { get; private set; } + /// + /// Gets or sets the quadtree depth of this block within its superblock. + /// public int Depth { get; internal set; } + /// + /// Gets or sets the width and height, in pixels, of the square sequence region that produced this block. + /// public int SequenceSize { get; internal set; } + /// + /// Gets or sets a value indicating whether this block belongs to the last quadrant of its parent. + /// public bool IsLastQuadrant { get; internal set; } } diff --git a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs index 6563895bb..c7e77c92b 100644 --- a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs +++ b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs @@ -5,10 +5,24 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.ModeDecision; +/// +/// Builds the AV1 block and transform geometries traversed by the mode-decision scan. +/// internal class Av1BlockGeometryFactory { + /// + /// The number of scan entries required by the largest supported 128-pixel superblock geometry. + /// private const int MaxBlocksAllocated = 4421; + + /// + /// Marks a geometry-depth combination that has no valid scan offset. + /// private const int NotUsedValue = 0; + + /// + /// Maps each partition shape, axis, and component to its origin offset measured in quarter-block units. + /// private static readonly int[][][] NonSkipQuarterOffMult = [ @@ -27,6 +41,9 @@ internal class Av1BlockGeometryFactory /*P=6*/ [[0, 2, 2, 9], [0, 0, 2, 9]] ]; + /// + /// Maps each partition shape, axis, and component to its dimension measured in quarter-block units. + /// private static readonly uint[][][] NonSkipSizeMult = [ @@ -45,7 +62,9 @@ internal class Av1BlockGeometryFactory /*P=6*/ [[2, 2, 2, 9], [4, 2, 2, 9]] ]; - // gives the index of next quadrant child within a depth + /// + /// Maps geometry and quadtree depth to the scan offset of the next quadrant at that depth. + /// private static readonly int[][] NonSkipDepthOffset = [ [85, 21, 5, 1, NotUsedValue, NotUsedValue], @@ -59,7 +78,9 @@ internal class Av1BlockGeometryFactory [2377, 593, 145, 33, 5, NotUsedValue] ]; - // gives the next depth block(first qudrant child) from a given parent square + /// + /// Maps geometry and quadtree depth to the scan offset of the square block's first child. + /// private static readonly int[][] Depth1DepthOffset = [ [1, 1, 1, 1, 1, NotUsedValue], @@ -73,24 +94,47 @@ internal class Av1BlockGeometryFactory [5, 13, 13, 13, 5, NotUsedValue] ]; + /// + /// The geometry whose lookup-table row is active while a scan is constructed. + /// private static Av1GeometryIndex geometryIndex; + + /// + /// The active geometry's superblock width and height in pixels. + /// private static int maxSuperblock; + + /// + /// The number of quadtree depths generated for the active geometry. + /// private static int maxDepth; + + /// + /// The number of partition shapes considered by the active geometry before size-specific restrictions. + /// private static int maxPart; // private static int maxActiveBlockCount; + + /// + /// Stores block geometries by mode-decision scan index. + /// private readonly Av1BlockGeometry[] blockGeometryModeDecisionScan; /// /// Initializes a new instance of the class. /// - /// SVT: md_scan_all_blks + /// The predefined geometry used to size and populate the mode-decision scan. + /// SVT-AV1: md_scan_all_blks. public Av1BlockGeometryFactory(Av1GeometryIndex geom) { this.blockGeometryModeDecisionScan = new Av1BlockGeometry[MaxBlocksAllocated]; int max_block_count; geometryIndex = geom; byte min_nsq_bsize; + + // These preset limits and the enum order form the row index contract for the offset tables above. + // Changing one without the other would make parent and sibling scan offsets refer to a different geometry. if (geom == Av1GeometryIndex.Geometry0) { maxSuperblock = 64; @@ -176,8 +220,11 @@ internal class Av1BlockGeometryFactory } /// - /// SVT: count_total_num_of_active_blks + /// Counts the block entries produced by every enabled partition at every depth of the active geometry. /// + /// The smallest square size, in pixels, at which non-square partitions remain enabled. + /// The number of active mode-decision scan entries. + /// SVT-AV1: count_total_num_of_active_blks. private static int CountTotalNumberOfActiveBlocks(int min_nsq_bsize) { int depth_scan_idx = 0; @@ -185,12 +232,17 @@ internal class Av1BlockGeometryFactory for (int depthIterator = 0; depthIterator < maxDepth; depthIterator++) { int totalSquareCount = 1 << depthIterator; + + // Each quadtree depth halves the square sequence dimension. The final branch covers the deepest + // 128-pixel-superblock geometry, whose sixth level contains 4-pixel squares. int sequenceSize = depthIterator == 0 ? maxSuperblock : depthIterator == 1 ? maxSuperblock / 2 : depthIterator == 2 ? maxSuperblock / 4 : depthIterator == 3 ? maxSuperblock / 8 : depthIterator == 4 ? maxSuperblock / 16 : maxSuperblock / 32; + // AV1 restricts the partition shapes allowed at the largest and smallest block sizes. Apply those + // caps before walking the shape table so a row is never interpreted for an illegal block size. int max_part_updated = sequenceSize == 128 ? Math.Min(maxPart, maxPart < 9 && maxPart > 3 ? 3 : 7) : sequenceSize == 8 ? Math.Min(maxPart, 3) : sequenceSize == 4 ? 1 : maxPart; @@ -216,8 +268,12 @@ internal class Av1BlockGeometryFactory } /// - /// SVT: get_num_ns_per_part + /// Gets the number of component blocks emitted by one partition shape. /// + /// The zero-based partition-shape index in scan order. + /// The width and height, in pixels, of the square being partitioned. + /// The number of component blocks in the partition. + /// SVT-AV1: get_num_ns_per_part. private static int GetNonSquareCountPerPart(int partitionIterator, int sequenceSize) { int tot_num_ns_per_part = partitionIterator < 1 ? 1 : partitionIterator < 3 ? 2 : partitionIterator < 5 && sequenceSize < 128 ? 4 : 3; @@ -225,8 +281,10 @@ internal class Av1BlockGeometryFactory } /// - /// SVT: log_redundancy_similarity + /// Records scan entries that represent the same block size at the same pixel origin. /// + /// The number of populated scan entries to compare. + /// SVT-AV1: log_redundancy_similarity. private static void LogRedundancySimilarity(int max_block_count) { for (int blockIterator = 0; blockIterator < max_block_count; blockIterator++) @@ -252,10 +310,24 @@ internal class Av1BlockGeometryFactory } /// - /// SVT: get_blk_geom_mds + /// Gets the block geometry at a mode-decision scan index. /// + /// The zero-based mode-decision scan index. + /// The geometry stored at . + /// Always thrown because the geometry lookup has not been implemented. + /// SVT-AV1: get_blk_geom_mds. public static Av1BlockGeometry GetBlockGeometryByModeDecisionScanIndex(int modeDecisionScanIndex) => throw new NotImplementedException(); + /// + /// Appends every enabled partition and transform layout for a square region to scan order. + /// + /// The next scan index; advanced once for every emitted block geometry. + /// The width and height, in pixels, of the square region being partitioned. + /// The region's horizontal origin in pixels relative to the superblock. + /// The region's vertical origin in pixels relative to the superblock. + /// Whether the region is the final quadrant of its parent. + /// The zero-based quadrant index within the parent. + /// The smallest square size, in pixels, at which non-square partitions remain enabled. private void ScanAllBlocks(ref int index, int sequenceSize, int x, int y, bool isLastQuadrant, byte quadIterator, byte minNonSquareBlockSize) { // The input block is the parent square block of size sq_size located at pos (x,y) @@ -264,6 +336,8 @@ internal class Av1BlockGeometryFactory int halfsize = sequenceSize / 2; int quartsize = sequenceSize / 4; + // AV1 removes partition shapes that cannot be represented at 128-, 8-, and 4-pixel square sizes. + // The scan tables are ordered by the remaining shape set, so the cap must be applied before indexing them. int max_part_updated = sequenceSize == 128 ? Math.Min(maxPart, maxPart is < 9 and > 3 ? 3 : 7) : sequenceSize == 8 ? Math.Min(maxPart, 3) : sequenceSize == 4 ? 1 : maxPart; @@ -280,6 +354,8 @@ internal class Av1BlockGeometryFactory for (int nonSquareIterator = 0; nonSquareIterator < tot_num_ns_per_part; nonSquareIterator++) { + // Geometry presets use power-of-two superblocks, so the current square dimension uniquely identifies + // its quadtree depth without carrying recursion state in every scan entry. this.blockGeometryModeDecisionScan[index].Depth = sequenceSize == maxSuperblock / 1 ? 0 : sequenceSize == maxSuperblock / 2 ? 1 : sequenceSize == maxSuperblock / 4 ? 2 @@ -315,6 +391,8 @@ internal class Av1BlockGeometryFactory this.blockGeometryModeDecisionScan[index].NonSquareIndex = nonSquareIterator; uint blockWidth = (uint)quartsize * NonSkipSizeMult[part_it_idx][0][nonSquareIterator]; uint blockHeight = (uint)quartsize * NonSkipSizeMult[part_it_idx][1][nonSquareIterator]; + + // Av1BlockSize indexes dimensions by log2(size) - 2 because 4x4 is the smallest coded block. this.blockGeometryModeDecisionScan[index].BlockSize = Av1BlockSizeExtensions.FromWidthAndHeight(Av1Math.Log2_32(blockWidth) - 2u, Av1Math.Log2_32(blockHeight) - 2u); this.blockGeometryModeDecisionScan[index].BlockSizeUv = this.blockGeometryModeDecisionScan[index].BlockSize.GetSubsampled(true, true); @@ -323,6 +401,8 @@ internal class Av1BlockGeometryFactory // this.blockGeometryModeDecisionScan[index].BlockHeightUv = Math.Max(4, this.blockGeometryModeDecisionScan[index].BlockHeight >> 1); this.blockGeometryModeDecisionScan[index].HasUv = true; + // Chroma cannot be subdivided below its minimum block dimensions. When several luma blocks map to + // the same chroma block, only the final contributing luma component owns that shared U/V geometry. if (this.blockGeometryModeDecisionScan[index].BlockWidth == 4 && this.blockGeometryModeDecisionScan[index].BlockHeight == 4) { this.blockGeometryModeDecisionScan[index].HasUv = isLastQuadrant; @@ -350,7 +430,8 @@ internal class Av1BlockGeometryFactory } } - // tx_depth 1 geom settings + // Transform depth zero keeps the largest legal transform. Blocks larger than AV1's 64x64 transform + // limit are represented by two or four transform blocks whose origins cover the coded block. int tx_depth = 0; this.blockGeometryModeDecisionScan[index].TransformBlockCount[tx_depth] = this.blockGeometryModeDecisionScan[index].BlockSize == Av1BlockSize.Block128x128 ? 4 @@ -405,7 +486,8 @@ internal class Av1BlockGeometryFactory tx_size_high[this.blockGeometryModeDecisionScan[index].TransformSizeUv[tx_depth]];*/ } - // tx_depth 1 geom settings + // Transform depth one subdivides eligible luma blocks once while chroma retains its depth-zero size. + // The block-count cases below mirror the legal rectangular AV1 transform partitions. tx_depth = 1; this.blockGeometryModeDecisionScan[index].TransformBlockCount[tx_depth] = this.blockGeometryModeDecisionScan[index].BlockSize == Av1BlockSize.Block128x128 ? 4 @@ -695,7 +777,8 @@ internal class Av1BlockGeometryFactory this.blockGeometryModeDecisionScan[index].tx_height_uv[tx_depth] = this.blockGeometryModeDecisionScan[index].tx_height_uv[0];*/ } - // tx_depth 2 geom settings + // Transform depth two performs a second subdivision. The origin tables enumerate the child + // transforms in raster order so coefficient reconstruction visits the same spatial layout. tx_depth = 2; this.blockGeometryModeDecisionScan[index].TransformBlockCount[tx_depth] = this.blockGeometryModeDecisionScan[index].BlockSize == Av1BlockSize.Block128x128 @@ -970,18 +1053,21 @@ internal class Av1BlockGeometryFactory } /// - /// SVT: av1_get_tx_size + /// Gets the largest legal transform size for a luma or subsampled chroma block. /// + /// The coded block size whose transform limit is requested. + /// The plane index, where zero selects luma and a positive value selects chroma. + /// The maximum transform size for the selected plane. + /// SVT-AV1: av1_get_tx_size. private static Av1TransformSize GetTransformSize(Av1BlockSize blockSize, int plane) { - // const MbModeInfo* mbmi = xd->mi[0]; - // if (xd->lossless[mbmi->segment_id]) return TX_4X4; + // Luma uses the coded block's normative transform ceiling directly. if (plane == 0) { return blockSize.GetMaximumTransformSize(); } - // const MacroblockdPlane *pd = &xd->plane[plane]; + // This geometry models 4:2:0 chroma, so both chroma axes are subsampled before selecting their limit. bool subsampling_x = plane > 0; bool subsampling_y = plane > 0; return blockSize.GetMaxUvTransformSize(subsampling_x, subsampling_y); diff --git a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs index 5392c3f20..cc7831017 100644 --- a/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs +++ b/src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs @@ -3,14 +3,48 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.ModeDecision; +/// +/// Identifies a predefined AV1 mode-decision geometry with a fixed superblock size, search depth, and partition set. +/// internal enum Av1GeometryIndex { + /// + /// The 64-pixel, four-depth geometry limited to square partitions. + /// Geometry0, + + /// + /// The 64-pixel, four-depth geometry with horizontal and vertical binary partitions down to 16 pixels. + /// Geometry1, + + /// + /// The 64-pixel, four-depth geometry with horizontal and vertical binary partitions down to 8 pixels. + /// Geometry2, + + /// + /// The 64-pixel, four-depth geometry with binary partitions at every supported size. + /// Geometry3, + + /// + /// The 64-pixel, five-depth geometry with binary partitions at every supported size. + /// Geometry4, + + /// + /// The 64-pixel, five-depth geometry that also enables four-way horizontal and vertical partitions. + /// Geometry5, + + /// + /// The 64-pixel, five-depth geometry that enables all supported partition shapes. + /// Geometry6, + + /// + /// The 128-pixel, six-depth geometry that enables all supported partition shapes. + /// Geometry7, }