Browse Source

Fix AV1 encoder partition and transform traversal

pull/2633/head
James Jackson-South 1 month ago
parent
commit
8a4ec3e07e
  1. 5
      HEIF_IMPLEMENTATION_PLAN.md
  2. 10
      src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs
  3. 163
      src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs
  4. 1050
      src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs
  5. 50
      src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs
  6. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs
  7. 24
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs
  8. 61
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderCoefficientBuffer.cs
  9. 42
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderTransformBlockState.cs
  10. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EntropyCodingContext.cs
  11. 10
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockD.cs
  12. 4
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs
  13. 15
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs
  14. 2
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1Superblock.cs
  15. 15
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockGeometry.cs
  16. 1153
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs
  17. 54
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TransformUnit.cs
  18. 428
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

5
HEIF_IMPLEMENTATION_PLAN.md

@ -822,8 +822,9 @@ Encoder verification contract:
- [ ] Implement real rate-distortion selection and make quality and effort change work, size, and output quality.
- [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Complete tile traversal, initialized picture state, and verified CDF update behavior remain.
- [~] Encoder mode information now uses a frame-owned reference grid over its contiguous allocation, matching current libaom's `mi_grid_base` and `mi_alloc` ownership without per-block tail copies. Signed relative neighbor lookup, 4x4-unit addressing, and mutable selected skip syntax have focused contracts; complete mode decision still remains.
- [~] Per-block transform, palette-size, and prediction syntax now uses fixed inline storage matching libaom's embedded block state. All 16 transform entries are immediately usable, directional deltas retain their signed range, and traversal performs no managed allocation; picture-level block storage and complete decision state remain.
- [~] Finalized transform coefficients now use raster-ordered, per-superblock plane segments matching current libaom's frame coefficient-pool geometry. ImageSharp may segment the frame owner between superblock rows, avoiding decoder borders and giant contiguous rentals while preserving disjoint luma/chroma ranges; the forward transform and mode-decision stages still need to populate this owner.
- [~] Final block decisions now use contiguous value storage with palette and prediction syntax inline. Macroblock edge and neighbor state is reused by the entropy-coding operation instead of allocating one managed object per final block; directional deltas retain their signed range, while complete mode decision still remains.
- [~] Finalized transform coefficients and packed EOB/type state now use raster-ordered, per-superblock plane segments matching current libaom's coefficient-pool geometry. One ImageSharp allocator owner replaces libaom's separate coefficient, EOB, and entropy-context allocations while preserving the full 1024 luma and 256-per-chroma 4x4 state capacity of a 128x128 4:2:0 superblock; the forward transform and mode-decision stages still need to populate this owner.
- [~] Tile partition writing now follows current libaom's recursive `write_modes_sb` preorder traversal and `update_ext_partition_context` edge updates directly. The obsolete SVT-derived global geometry catalog and its unimplemented lookup are removed; transform geometry is derived in libaom's bounded 64x64 residual order from the selected block transform size, frame-edge and segmentation syntax use mode-information units, and 128x128 CDEF units use libaom's 0-to-3 indexing and first-block strength ownership. Partition analysis and selected-transform syntax still need to populate and encode these retained decisions.
- [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions.
- [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform. Every remaining encoder fragment must be audited before it becomes active.
- [~] The planar conversion, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Apply the same rule to every later hot-path family.

10
src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs

@ -286,21 +286,11 @@ internal static class Av1Constants
/// </summary>
public const int MaxVarTransform = 2;
/// <summary>
/// The maximum number of transform blocks at one depth.
/// </summary>
public const int MaxTransformBlockCount = 16;
/// <summary>
/// Number of items in the <see cref="Av1PlaneType"/> enumeration.
/// </summary>
public const int PlaneTypeCount = 2;
/// <summary>
/// The maximum number of transform units stored for one encoded block.
/// </summary>
public const int MaxTransformUnitCount = 16;
/// <summary>
/// Gets the number of payload bits used by each segmentation feature.
/// </summary>

163
src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometry.cs

@ -1,163 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.ModeDecision;
/// <summary>
/// Describes the spatial, chroma, and transform layout of one AV1 block considered by mode-decision scanning.
/// </summary>
internal class Av1BlockGeometry
{
/// <summary>
/// The luma block size from which the cached luma dimensions are derived.
/// </summary>
private Av1BlockSize blockSize;
/// <summary>
/// The chroma block size from which the cached chroma dimensions are derived.
/// </summary>
private Av1BlockSize blockSizeUv;
/// <summary>
/// Initializes a new instance of the <see cref="Av1BlockGeometry"/> class with storage for every supported transform depth.
/// </summary>
public Av1BlockGeometry()
{
this.RedunancyList = [];
this.TransformOrigin = new Point[Av1Constants.MaxVarTransform + 1][];
for (int i = 0; i < this.TransformOrigin.Length; i++)
{
this.TransformOrigin[i] = new Point[Av1Constants.MaxTransformBlockCount];
}
}
/// <summary>
/// Gets or sets the luma block size and updates <see cref="BlockWidth"/> and <see cref="BlockHeight"/> to match.
/// </summary>
public Av1BlockSize BlockSize
{
get => this.blockSize;
set
{
this.blockSize = value;
this.BlockWidth = value.GetWidth();
this.BlockHeight = value.GetHeight();
}
}
/// <summary>
/// Gets or sets the chroma block size and updates <see cref="BlockWidthUv"/> and <see cref="BlockHeightUv"/> to match.
/// </summary>
public Av1BlockSize BlockSizeUv
{
get => this.blockSizeUv;
set
{
this.blockSizeUv = value;
this.BlockWidthUv = value.GetWidth();
this.BlockHeightUv = value.GetHeight();
}
}
/// <summary>
/// Gets or sets the block origin in pixels relative to the top-left corner of its superblock.
/// </summary>
public Point Origin { get; set; }
/// <summary>
/// Gets or sets a value indicating whether this luma block owns chroma samples in the mode-decision layout.
/// </summary>
public bool HasUv { get; set; }
/// <summary>
/// Gets the luma block width in pixels.
/// </summary>
public int BlockWidth { get; private set; }
/// <summary>
/// Gets the luma block height in pixels.
/// </summary>
public int BlockHeight { get; private set; }
/// <summary>
/// Gets the number of luma transform blocks at each transform depth.
/// </summary>
public int[] TransformBlockCount { get; } = new int[Av1Constants.MaxVarTransform + 1];
/// <summary>
/// Gets the luma transform size selected at each transform depth.
/// </summary>
public Av1TransformSize[] TransformSize { get; } = new Av1TransformSize[Av1Constants.MaxVarTransform + 1];
/// <summary>
/// Gets the chroma transform size selected at each transform depth.
/// </summary>
public Av1TransformSize[] TransformSizeUv { get; } = new Av1TransformSize[Av1Constants.MaxVarTransform + 1];
/// <summary>
/// Gets the pixel origins of the transform blocks at each transform depth.
/// </summary>
public Point[][] TransformOrigin { get; private set; }
/// <summary>
/// Gets or sets the block index in mode-decision scan order.
/// </summary>
public int ModeDecisionIndex { get; set; }
/// <summary>
/// Gets or sets the scan offset from this square block to the next block at the same depth.
/// </summary>
public int NextDepthOffset { get; set; }
/// <summary>
/// Gets or sets the scan offset from this square block to its first child at the next depth.
/// </summary>
public int Depth1Offset { get; set; }
/// <summary>
/// Gets a value indicating whether this block is redundant to another.
/// </summary>
public bool IsRedundant => this.RedunancyList.Count > 0;
/// <summary>
/// Gets or sets the mode-decision indices of blocks with the same size and origin as this block.
/// </summary>
public List<int> RedunancyList { get; set; }
/// <summary>
/// Gets or sets the zero-based component index of this block within a non-square partition.
/// </summary>
public int NonSquareIndex { get; set; }
/// <summary>
/// Gets or sets the number of component blocks produced by this partition shape.
/// </summary>
public int TotalNonSuareCount { get; set; }
/// <summary>
/// Gets the chroma block width in pixels.
/// </summary>
public int BlockWidthUv { get; private set; }
/// <summary>
/// Gets the chroma block height in pixels.
/// </summary>
public int BlockHeightUv { get; private set; }
/// <summary>
/// Gets or sets the quadtree depth of this block within its superblock.
/// </summary>
public int Depth { get; set; }
/// <summary>
/// Gets or sets the width and height, in pixels, of the square sequence region that produced this block.
/// </summary>
public int SequenceSize { get; set; }
/// <summary>
/// Gets or sets a value indicating whether this block belongs to the last quadrant of its parent.
/// </summary>
public bool IsLastQuadrant { get; set; }
}

1050
src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1BlockGeometryFactory.cs

File diff suppressed because it is too large

50
src/ImageSharp/Formats/Heif/Av1/ModeDecision/Av1GeometryIndex.cs

@ -1,50 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif.Av1.ModeDecision;
/// <summary>
/// Identifies a predefined AV1 mode-decision geometry with a fixed superblock size, search depth, and partition set.
/// </summary>
internal enum Av1GeometryIndex
{
/// <summary>
/// The 64-pixel, four-depth geometry limited to square partitions.
/// </summary>
Geometry0,
/// <summary>
/// The 64-pixel, four-depth geometry with horizontal and vertical binary partitions down to 16 pixels.
/// </summary>
Geometry1,
/// <summary>
/// The 64-pixel, four-depth geometry with horizontal and vertical binary partitions down to 8 pixels.
/// </summary>
Geometry2,
/// <summary>
/// The 64-pixel, four-depth geometry with binary partitions at every supported size.
/// </summary>
Geometry3,
/// <summary>
/// The 64-pixel, five-depth geometry with binary partitions at every supported size.
/// </summary>
Geometry4,
/// <summary>
/// The 64-pixel, five-depth geometry that also enables four-way horizontal and vertical partitions.
/// </summary>
Geometry5,
/// <summary>
/// The 64-pixel, five-depth geometry that enables all supported partition shapes.
/// </summary>
Geometry6,
/// <summary>
/// The 128-pixel, six-depth geometry that enables all supported partition shapes.
/// </summary>
Geometry7,
}

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs

@ -2,6 +2,7 @@
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -41,9 +42,9 @@ internal class Av1EncoderBlockModeInfo
public int SegmentId { get; set; }
/// <summary>
/// Gets or sets the transform-tree depth selected for the block.
/// Gets or sets the luma transform size selected for the block.
/// </summary>
public int TransformDepth { get; set; }
public Av1TransformSize TransformSize { get; set; }
/// <summary>
/// Gets or sets the luma prediction mode written for the block.

24
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -8,13 +9,8 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
/// Stores encoder block geometry and its selected coding-mode information.
/// </summary>
internal class Av1EncoderBlockStruct
internal struct Av1EncoderBlockStruct
{
/// <summary>
/// Stores transform-unit state inline with the block.
/// </summary>
private InlineArray16<Av1TransformUnit> transformBlocks;
/// <summary>
/// Stores the luma and shared chroma palette sizes inline with the block.
/// </summary>
@ -26,19 +22,9 @@ internal class Av1EncoderBlockStruct
private Av1EncoderPredictionUnit predictionUnit;
/// <summary>
/// Gets the transform-unit state in transform traversal order.
/// </summary>
public Span<Av1TransformUnit> TransformBlocks => this.transformBlocks;
/// <summary>
/// Gets or sets the macroblock edge and neighbor state used while writing the block.
/// </summary>
public required Av1MacroBlockD MacroBlock { get; set; }
/// <summary>
/// Gets or sets the index used to resolve the block geometry from mode-decision scan order.
/// Gets or sets a value indicating whether this luma block owns the corresponding chroma syntax.
/// </summary>
public int ModeDecisionScanIndex { get; set; }
public bool HasChroma { get; set; }
/// <summary>
/// Gets or sets the quantizer index used for the block.
@ -58,11 +44,13 @@ internal class Av1EncoderBlockStruct
/// <summary>
/// Gets the writable palette sizes for luma and for the shared chroma mode.
/// </summary>
[UnscopedRef]
public Span<byte> PaletteSize => this.paletteSize;
/// <summary>
/// Gets the encoder prediction-unit state for the block.
/// </summary>
[UnscopedRef]
public ref Av1EncoderPredictionUnit PredictionUnit => ref this.predictionUnit;
/// <summary>

61
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderCoefficientBuffer.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Memory;
@ -12,9 +13,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
internal sealed class Av1EncoderCoefficientBuffer : IDisposable
{
/// <summary>
/// Stores one complete superblock's luma and chroma coefficients in each row.
/// The number of coefficients represented by one entry in libaom's EOB arrays.
/// </summary>
private readonly Buffer2D<int> coefficients;
public const int TransformBlockUnitCoefficientCount = 1 << (Av1Constants.ModeInfoSizeLog2 * 2);
/// <summary>
/// Stores one complete superblock's coefficients and packed transform-block state in each row.
/// </summary>
private readonly Buffer2D<int> storage;
/// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderCoefficientBuffer"/> class.
@ -40,11 +46,15 @@ internal sealed class Av1EncoderCoefficientBuffer : IDisposable
int chromaSubsampling = (colorConfig.SubSamplingX ? 1 : 0) + (colorConfig.SubSamplingY ? 1 : 0);
this.ChromaCoefficientCount = colorConfig.IsMonochrome ? 0 : this.LumaCoefficientCount >> chromaSubsampling;
this.CoefficientsPerSuperblock = this.LumaCoefficientCount + (2 * this.ChromaCoefficientCount);
this.LumaTransformBlockCount = this.LumaCoefficientCount / TransformBlockUnitCoefficientCount;
this.ChromaTransformBlockCount = this.ChromaCoefficientCount / TransformBlockUnitCoefficientCount;
this.TransformBlocksPerSuperblock = this.LumaTransformBlockCount + (2 * this.ChromaTransformBlockCount);
int storageElementsPerSuperblock = this.CoefficientsPerSuperblock + this.TransformBlocksPerSuperblock;
// libaom stores finalized coefficients by raster-ordered superblock. A two-dimensional owner preserves that
// layout while allowing ImageSharp's allocator to segment the frame instead of demanding one giant rental.
this.coefficients = configuration.MemoryAllocator.Allocate2D<int>(
this.CoefficientsPerSuperblock,
// layout, and packing the EOB/type state into the same row removes its two additional frame-sized allocations.
this.storage = configuration.MemoryAllocator.Allocate2D<int>(
storageElementsPerSuperblock,
this.SuperblockCount);
}
@ -78,6 +88,21 @@ internal sealed class Av1EncoderCoefficientBuffer : IDisposable
/// </summary>
public int CoefficientsPerSuperblock { get; }
/// <summary>
/// Gets the number of luma transform-block positions reserved for each superblock.
/// </summary>
public int LumaTransformBlockCount { get; }
/// <summary>
/// Gets the number of transform-block positions reserved for each chroma plane in each superblock.
/// </summary>
public int ChromaTransformBlockCount { get; }
/// <summary>
/// Gets the number of transform-block positions reserved for each complete superblock.
/// </summary>
public int TransformBlocksPerSuperblock { get; }
/// <summary>
/// Gets the total number of coefficient positions retained for the frame.
/// </summary>
@ -91,7 +116,7 @@ internal sealed class Av1EncoderCoefficientBuffer : IDisposable
/// <returns>The complete coefficient span reserved for that plane and superblock.</returns>
public Span<int> GetPlaneSpan(int superblockIndex, Av1Plane plane)
{
Span<int> superblock = this.coefficients.DangerousGetRowSpan(superblockIndex);
Span<int> superblock = this.storage.DangerousGetRowSpan(superblockIndex);
return plane switch
{
Av1Plane.Y => superblock[..this.LumaCoefficientCount],
@ -102,8 +127,30 @@ internal sealed class Av1EncoderCoefficientBuffer : IDisposable
};
}
/// <summary>
/// Gets one component plane's transform-block state for a raster-ordered superblock.
/// </summary>
/// <param name="superblockIndex">The raster-ordered superblock index.</param>
/// <param name="plane">The requested component plane.</param>
/// <returns>One state entry for every 4x4 coefficient unit in the plane.</returns>
public Span<Av1EncoderTransformBlockState> GetTransformBlockSpan(int superblockIndex, Av1Plane plane)
{
Span<int> superblock = this.storage.DangerousGetRowSpan(superblockIndex);
Span<Av1EncoderTransformBlockState> transformBlocks =
MemoryMarshal.Cast<int, Av1EncoderTransformBlockState>(superblock[this.CoefficientsPerSuperblock..]);
return plane switch
{
Av1Plane.Y => transformBlocks[..this.LumaTransformBlockCount],
Av1Plane.U => transformBlocks.Slice(this.LumaTransformBlockCount, this.ChromaTransformBlockCount),
_ => transformBlocks.Slice(
this.LumaTransformBlockCount + this.ChromaTransformBlockCount,
this.ChromaTransformBlockCount)
};
}
/// <summary>
/// Releases the frame coefficient storage.
/// </summary>
public void Dispose() => this.coefficients.Dispose();
public void Dispose() => this.storage.Dispose();
}

42
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderTransformBlockState.cs

@ -0,0 +1,42 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
/// Stores the entropy syntax retained for one AV1 transform block.
/// </summary>
[StructLayout(LayoutKind.Sequential, Size = sizeof(int))]
internal struct Av1EncoderTransformBlockState
{
/// <summary>
/// Stores the position after the final nonzero coefficient.
/// </summary>
private ushort endOfBlock;
/// <summary>
/// Stores the selected transform type.
/// </summary>
private Av1TransformType transformType;
/// <summary>
/// Gets or sets the position after the final nonzero coefficient.
/// </summary>
public ushort EndOfBlock
{
readonly get => this.endOfBlock;
set => this.endOfBlock = value;
}
/// <summary>
/// Gets or sets the selected transform type.
/// </summary>
public Av1TransformType TransformType
{
readonly get => this.transformType;
set => this.transformType = value;
}
}

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EntropyCodingContext.cs

@ -13,6 +13,11 @@ internal partial class Av1TileWriter
/// </summary>
internal class Av1EntropyCodingContext
{
/// <summary>
/// Gets the reusable macroblock edge and neighbor state for the current entropy-coding operation.
/// </summary>
public required Av1MacroBlockD MacroBlock { get; init; }
/// <summary>
/// Gets or sets the macroblock mode information currently being encoded.
/// </summary>

10
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockD.cs

@ -68,16 +68,6 @@ internal class Av1MacroBlockD
/// </summary>
public int ToRightEdge { get; set; }
/// <summary>
/// Gets or sets the block dimensions in samples for rectangular-partition context selection.
/// </summary>
public Size N8Size { get; set; }
/// <summary>
/// Gets or sets a value indicating whether this block is the second half of a rectangular partition.
/// </summary>
public bool IsSecondRectangle { get; set; }
/// <summary>
/// Selects the current entry in the frame-owned mode-information reference grid.
/// </summary>

4
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs

@ -113,8 +113,8 @@ internal class Av1PictureControlSet
int mi_col = origin.X >> Av1Constants.ModeInfoSizeLog2;
int mi_row = origin.Y >> Av1Constants.ModeInfoSizeLog2;
int mi_offset = (mi_row * cm.ModeInfoColumnCount) + mi_col;
int bw = blockSize.GetWidth();
int bh = blockSize.GetHeight();
int bw = blockSize.Get4x4WideCount();
int bh = blockSize.Get4x4HighCount();
int xmis = Math.Min(cm.ModeInfoColumnCount - mi_col, bw);
int ymis = Math.Min(cm.ModeInfoRowCount - mi_row, bh);

15
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs

@ -29,19 +29,4 @@ internal class Av1PictureParentControlSet
/// Gets or sets the encoder palette-search level.
/// </summary>
public int PaletteLevel { get; set; }
/// <summary>
/// Gets or sets the frame width aligned for superblock traversal.
/// </summary>
public int AlignedWidth { get; set; }
/// <summary>
/// Gets or sets the frame height aligned for superblock traversal.
/// </summary>
public int AlignedHeight { get; set; }
/// <summary>
/// Gets or sets the geometry state for each superblock in the picture.
/// </summary>
public required Av1SuperblockGeometry[] SuperblockGeometry { get; set; }
}

2
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1Superblock.cs

@ -1,8 +1,6 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using static SixLabors.ImageSharp.Formats.Heif.Av1.Tiling.Av1TileWriter;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>

15
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockGeometry.cs

@ -1,15 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
/// Describes a superblock node's location, dimensions, and children in the encoder partition tree.
/// </summary>
internal class Av1SuperblockGeometry
{
/// <summary>
/// Gets or sets a value indicating whether the superblock lies completely within the coded frame.
/// </summary>
public bool IsComplete { get; set; }
}

1153
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs

File diff suppressed because it is too large

54
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TransformUnit.cs

@ -1,54 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
/// Stores the transform syntax and coefficient range for one AV1 transform unit.
/// </summary>
internal struct Av1TransformUnit
{
/// <summary>
/// Stores the luma, blue-difference, and red-difference end-of-block positions.
/// </summary>
private InlineArray3<ushort> nzCoefficientCount;
/// <summary>
/// Stores the luma and shared chroma transform types.
/// </summary>
private InlineArray2<Av1TransformType> transformType;
/// <summary>
/// Gets the nonzero-coefficient count for each color plane.
/// </summary>
[UnscopedRef]
public Span<ushort> NzCoefficientCount => this.nzCoefficientCount;
/// <summary>
/// Gets the transform type selected for each color plane.
/// </summary>
[UnscopedRef]
public Span<Av1TransformType> TransformType => this.transformType;
/// <summary>
/// Stores the three per-plane coefficient counts inline.
/// </summary>
[InlineArray(3)]
private struct InlineArray3<T>
{
private T element;
}
/// <summary>
/// Stores the luma and shared chroma transform types inline.
/// </summary>
[InlineArray(Av1Constants.PlaneTypeCount)]
private struct InlineArray2<T>
{
private T element;
}
}

428
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

@ -102,8 +102,7 @@ public class Av1CoefficientsEntropyTests
TilesInfo = new ObuTileGroupHeader()
},
FrameHeader = new ObuFrameHeader(),
PreviousQIndex = [],
SuperblockGeometry = []
PreviousQIndex = []
},
SegmentationNeighborMap = [],
ModeInfoGrid = new Av1ModeInfo[16],
@ -166,39 +165,37 @@ public class Av1CoefficientsEntropyTests
}
[Fact]
public void EncoderBlockInlineStateSupportsEveryTransformWithoutTraversalAllocations()
public void EncoderBlocksKeepInlineModeStateWithoutPerBlockAllocations()
{
Av1EncoderBlockStruct block = new() { MacroBlock = CreateMacroBlock() };
Av1EncoderBlockStruct[] blocks = new Av1EncoderBlockStruct[2];
long before = GC.GetAllocatedBytesForCurrentThread();
Span<Av1TransformUnit> transforms = block.TransformBlocks;
transforms[^1].NzCoefficientCount[2] = 17;
transforms[^1].TransformType[(int)Av1PlaneType.Uv] = Av1TransformType.VerticalAdst;
ref Av1EncoderBlockStruct block = ref blocks[1];
block.PaletteSize[0] = 3;
block.PaletteSize[1] = 5;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = -2;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 3;
long allocated = GC.GetAllocatedBytesForCurrentThread() - before;
Assert.Equal(Av1Constants.MaxTransformUnitCount, transforms.Length);
Assert.Equal(17, block.TransformBlocks[^1].NzCoefficientCount[2]);
Assert.Equal(Av1TransformType.VerticalAdst, block.TransformBlocks[^1].TransformType[(int)Av1PlaneType.Uv]);
Assert.Equal(3, block.PaletteSize[0]);
Assert.Equal(5, block.PaletteSize[1]);
Assert.Equal(-2, block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y]);
Assert.Equal(3, block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv]);
Assert.Equal(3, blocks[1].PaletteSize[0]);
Assert.Equal(5, blocks[1].PaletteSize[1]);
Assert.Equal(-2, blocks[1].PredictionUnit.AngleDelta[(int)Av1PlaneType.Y]);
Assert.Equal(3, blocks[1].PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv]);
Assert.Equal(0, allocated);
}
[Theory]
[InlineData(false, 6, 4096, 1024, 6144, 36864L)]
[InlineData(true, 2, 16384, 4096, 24576, 49152L)]
[InlineData(false, 6, 4096, 1024, 6144, 256, 64, 384, 36864L)]
[InlineData(true, 2, 16384, 4096, 24576, 1024, 256, 1536, 49152L)]
public void EncoderCoefficientBufferMatchesLibaom420SuperblockLayout(
bool use128x128Superblock,
int expectedSuperblockCount,
int expectedLumaCount,
int expectedChromaCount,
int expectedCoefficientsPerSuperblock,
int expectedLumaTransformBlockCount,
int expectedChromaTransformBlockCount,
int expectedTransformBlocksPerSuperblock,
long expectedTotalCoefficientCount)
{
ObuSequenceHeader sequenceHeader = new() { Use128x128Superblock = use128x128Superblock };
@ -216,10 +213,16 @@ public class Av1CoefficientsEntropyTests
Assert.Equal(expectedLumaCount, coefficients.LumaCoefficientCount);
Assert.Equal(expectedChromaCount, coefficients.ChromaCoefficientCount);
Assert.Equal(expectedCoefficientsPerSuperblock, coefficients.CoefficientsPerSuperblock);
Assert.Equal(expectedLumaTransformBlockCount, coefficients.LumaTransformBlockCount);
Assert.Equal(expectedChromaTransformBlockCount, coefficients.ChromaTransformBlockCount);
Assert.Equal(expectedTransformBlocksPerSuperblock, coefficients.TransformBlocksPerSuperblock);
Assert.Equal(expectedTotalCoefficientCount, coefficients.TotalCoefficientCount);
Assert.Equal(expectedLumaCount, coefficients.GetPlaneSpan(0, Av1Plane.Y).Length);
Assert.Equal(expectedChromaCount, coefficients.GetPlaneSpan(0, Av1Plane.U).Length);
Assert.Equal(expectedChromaCount, coefficients.GetPlaneSpan(0, Av1Plane.V).Length);
Assert.Equal(expectedLumaTransformBlockCount, coefficients.GetTransformBlockSpan(0, Av1Plane.Y).Length);
Assert.Equal(expectedChromaTransformBlockCount, coefficients.GetTransformBlockSpan(0, Av1Plane.U).Length);
Assert.Equal(expectedChromaTransformBlockCount, coefficients.GetTransformBlockSpan(0, Av1Plane.V).Length);
}
[Fact]
@ -240,11 +243,342 @@ public class Av1CoefficientsEntropyTests
coefficients.GetPlaneSpan(0, Av1Plane.U)[0] = 22;
coefficients.GetPlaneSpan(0, Av1Plane.V)[0] = 33;
coefficients.GetPlaneSpan(1, Av1Plane.Y)[0] = 44;
coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[0].EndOfBlock = 55;
coefficients.GetTransformBlockSpan(0, Av1Plane.U)[0].EndOfBlock = 66;
coefficients.GetTransformBlockSpan(0, Av1Plane.V)[0].EndOfBlock = 77;
coefficients.GetTransformBlockSpan(1, Av1Plane.Y)[0].EndOfBlock = 88;
Assert.Equal(11, coefficients.GetPlaneSpan(0, Av1Plane.Y)[0]);
Assert.Equal(22, coefficients.GetPlaneSpan(0, Av1Plane.U)[0]);
Assert.Equal(33, coefficients.GetPlaneSpan(0, Av1Plane.V)[0]);
Assert.Equal(44, coefficients.GetPlaneSpan(1, Av1Plane.Y)[0]);
Assert.Equal(55, coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[0].EndOfBlock);
Assert.Equal(66, coefficients.GetTransformBlockSpan(0, Av1Plane.U)[0].EndOfBlock);
Assert.Equal(77, coefficients.GetTransformBlockSpan(0, Av1Plane.V)[0].EndOfBlock);
Assert.Equal(88, coefficients.GetTransformBlockSpan(1, Av1Plane.Y)[0].EndOfBlock);
}
[Fact]
public void EncoderLumaTraversalRepresentsAllTransformsIn128x128Block()
{
Av1PictureControlSet picture = CreateEncoderPicture(32, 32, use128x128Superblock: true);
Av1MacroBlockModeInfo modeInfo = picture.ModeInfoAllocation[0].MacroBlockModeInfo;
modeInfo.Block.BlockSize = Av1BlockSize.Block128x128;
modeInfo.Block.TransformSize = Av1TransformSize.Size16x16;
modeInfo.Block.SegmentId = 0;
Av1TileInfo tile = new(0, 0, picture.Parent.FrameHeader);
Av1TileWriter.Av1EntropyCodingContext context = new()
{
MacroBlock = new Av1MacroBlockD { Tile = tile },
MacroBlockModeInfo = modeInfo,
SuperblockOrigin = Point.Empty
};
using Av1NeighborArrayUnit<byte> luma = new(
Configuration.Default,
leftSize: 128,
topSize: 128,
topLeftSize: 256)
{
GranularityNormalLog2 = Av1Constants.ModeInfoSizeLog2,
GranularityTopLeftLog2 = Av1Constants.ModeInfoSizeLog2
};
using Av1EncoderCoefficientBuffer coefficients = new(
Configuration.Default,
picture.Sequence.SequenceHeader,
width: 128,
height: 128);
Span<Av1EncoderTransformBlockState> transformBlocks =
coefficients.GetTransformBlockSpan(0, Av1Plane.Y);
transformBlocks.Fill(new Av1EncoderTransformBlockState { TransformType = Av1TransformType.Identity });
Av1EncoderBlockStruct block = default;
Av1SymbolEncoder writer = new(Configuration.Default, 4096, BaseQIndex);
Av1TileWriter.EncodeTransformCoefficientsY(
picture,
context,
ref writer,
ref block,
Point.Empty,
Av1PredictionMode.DC,
Av1BlockSize.Block128x128,
coefficients,
superblockIndex: 0,
luma);
writer.Dispose();
int visitedTransformCount = 0;
for (int index = 0; index < transformBlocks.Length; index++)
{
if ((index % 16) == 0)
{
Assert.Equal(Av1TransformType.DctDct, transformBlocks[index].TransformType);
visitedTransformCount++;
}
else
{
Assert.Equal(Av1TransformType.Identity, transformBlocks[index].TransformType);
}
}
Assert.Equal(64, visitedTransformCount);
Assert.Equal(16384, context.CodedAreaSuperblock);
}
[Fact]
public void SegmentationUpdateUsesModeInfoUnits()
{
Av1PictureControlSet picture = CreateEncoderPicture(8, 8);
picture.SegmentationNeighborMap = new byte[64];
picture.UpdateSegmentation(Av1BlockSize.Block16x8, new Point(8, 12), segmentId: 5);
for (int row = 0; row < 8; row++)
{
for (int column = 0; column < 8; column++)
{
byte expected = row is 3 or 4 && column >= 2 && column < 6 ? (byte)5 : (byte)0;
Assert.Equal(expected, picture.SegmentationNeighborMap[(row * 8) + column]);
}
}
}
[Theory]
[InlineData((int)Av1PartitionType.None, 24, 24)]
[InlineData((int)Av1PartitionType.Horizontal, 24, 28)]
[InlineData((int)Av1PartitionType.Vertical, 28, 24)]
[InlineData((int)Av1PartitionType.Split, 0, 0)]
[InlineData((int)Av1PartitionType.HorizontalA, 24, 28)]
[InlineData((int)Av1PartitionType.HorizontalB, 28, 28)]
[InlineData((int)Av1PartitionType.VerticalA, 28, 24)]
[InlineData((int)Av1PartitionType.VerticalB, 28, 28)]
[InlineData((int)Av1PartitionType.Horizontal4, 24, 30)]
[InlineData((int)Av1PartitionType.Vertical4, 30, 24)]
public void PartitionContextUpdatesMatchLibaomExtendedPartitionRules(
int partitionValue,
byte expectedAbove,
byte expectedLeft)
{
using Av1NeighborArrayUnit<Av1PartitionContext> neighbors = new(
Configuration.Default,
leftSize: 16,
topSize: 16,
topLeftSize: 32)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
Av1PartitionType partition = (Av1PartitionType)partitionValue;
Av1BlockSize blockSize = Av1BlockSize.Block32x32;
Av1BlockSize subSize = partition.GetBlockSubSize(blockSize);
Av1TileWriter.UpdatePartitionContexts(
neighbors,
new Point(8, 12),
subSize,
blockSize,
partition);
for (int index = 0; index < 16; index++)
{
byte above = index is >= 2 and < 10 ? expectedAbove : (byte)0;
byte left = index is >= 3 and < 11 ? expectedLeft : (byte)0;
Assert.Equal(above, neighbors.Top[index].Above);
Assert.Equal(left, neighbors.Left[index].Left);
}
}
[Fact]
public void EightByEightSplitPublishesFourByFourPartitionContexts()
{
using Av1NeighborArrayUnit<Av1PartitionContext> neighbors = new(
Configuration.Default,
leftSize: 4,
topSize: 4,
topLeftSize: 8)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
Av1TileWriter.UpdatePartitionContexts(
neighbors,
new Point(4, 4),
Av1BlockSize.Block4x4,
Av1BlockSize.Block8x8,
Av1PartitionType.Split);
Assert.Equal(31, neighbors.Top[1].Above);
Assert.Equal(31, neighbors.Top[2].Above);
Assert.Equal(31, neighbors.Left[1].Left);
Assert.Equal(31, neighbors.Left[2].Left);
}
[Fact]
public void EncoderModeInfoEdgesUseFourByFourUnits()
{
Av1PictureControlSet picture = CreateEncoderPicture(6, 5);
Av1TileInfo tile = new(0, 0, picture.Parent.FrameHeader);
Av1MacroBlockD macroBlock = new() { Tile = tile };
Point position = new(2, 3);
Av1TileWriter.SetModeInfoRowAndColumn(
picture,
macroBlock,
tile,
position,
Av1BlockSize.Block16x8,
picture.ModeInfoStride,
picture.Parent.Common.ModeInfoRowCount,
picture.Parent.Common.ModeInfoColumnCount);
Assert.Equal(-96, macroBlock.ToTopEdge);
Assert.Equal(0, macroBlock.ToBottomEdge);
Assert.Equal(-64, macroBlock.ToLeftEdge);
Assert.Equal(0, macroBlock.ToRightEdge);
Assert.Same(picture.ModeInfoGrid[14].MacroBlockModeInfo, macroBlock.AboveMacroBlock);
Assert.Same(picture.ModeInfoGrid[19].MacroBlockModeInfo, macroBlock.LeftMacroBlock);
}
[Fact]
public void CdefUsesLibaomUnitIndexAndFirstBlockStrength()
{
Av1PictureControlSet picture = CreateEncoderPicture(32, 32, use128x128Superblock: true);
picture.Parent.FrameHeader.CdefParameters.BitCount = 2;
picture.ModeInfoGrid[16].MacroBlockModeInfo.CdefStrength = 3;
picture.ModeInfoGrid[20].MacroBlockModeInfo.CdefStrength = 1;
using Av1SymbolEncoder writer = new(Configuration.Default, 16, BaseQIndex);
Av1TileWriter.WriteCdef(
picture.Sequence,
picture,
writer,
tileIndex: 0,
skip: false,
modeInfoPosition: new Point(20, 4));
Assert.Equal(new[] { -1, 3, -1, -1 }, picture.CdefPreset[0]);
}
[Fact]
public void SuperblockWriterTraversesSplitTreeFromAbsoluteOrigin()
{
Av1PictureControlSet picture = CreateEncoderPicture(32, 16);
picture.Sequence.SequenceHeader.ColorConfig.IsMonochrome = true;
picture.Parent.FrameHeader.CodedLossless = true;
using Av1NeighborArrayUnit<Av1PartitionContext> partitions = new(
Configuration.Default,
leftSize: 16,
topSize: 32,
topLeftSize: 48)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
using Av1NeighborArrayUnit<byte> luma = new(
Configuration.Default,
leftSize: 16,
topSize: 32,
topLeftSize: 48)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
using Av1NeighborArrayUnit<byte> red = new(
Configuration.Default,
leftSize: 16,
topSize: 32,
topLeftSize: 48)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
using Av1NeighborArrayUnit<byte> blue = new(
Configuration.Default,
leftSize: 16,
topSize: 32,
topLeftSize: 48)
{
GranularityNormalLog2 = 2,
GranularityTopLeftLog2 = 2
};
picture.PartitionContexts = [partitions];
picture.LuminanceDcSignLevelCoefficientNeighbors = [luma];
picture.CrDcSignLevelCoefficientNeighbors = [red];
picture.CbDcSignLevelCoefficientNeighbors = [blue];
Av1TileInfo tile = new(0, 0, picture.Parent.FrameHeader);
Point[] blockPositions = [new(16, 0), new(24, 0), new(16, 8), new(24, 8)];
Av1EncoderBlockStruct[] blocks = new Av1EncoderBlockStruct[blockPositions.Length];
for (int index = 0; index < blockPositions.Length; index++)
{
Point position = blockPositions[index];
Av1EncoderBlockModeInfo blockMode = picture.ModeInfoAllocation[
(position.Y * picture.ModeInfoStride) + position.X].MacroBlockModeInfo.Block;
blockMode.BlockSize = Av1BlockSize.Block32x32;
blockMode.Skip = true;
blockMode.Mode = Av1PredictionMode.DC;
blockMode.UvMode = Av1ChromaPredictionMode.DC;
blocks[index] = new Av1EncoderBlockStruct { HasChroma = false };
}
Av1Superblock superblock = new()
{
FinalBlocks = blocks,
TileInfo = tile,
CodingUnitPartitionTypes =
[
Av1PartitionType.Split,
Av1PartitionType.None,
Av1PartitionType.None,
Av1PartitionType.None,
Av1PartitionType.None
],
Index = 1
};
Av1TileWriter.Av1EntropyCodingContext context = new()
{
MacroBlock = new Av1MacroBlockD { Tile = tile },
MacroBlockModeInfo = picture.ModeInfoAllocation[16].MacroBlockModeInfo,
SuperblockOrigin = new Point(64, 0)
};
using Av1EncoderCoefficientBuffer coefficients = new(
Configuration.Default,
picture.Sequence.SequenceHeader,
width: 128,
height: 64);
Av1SymbolEncoder writer = new(Configuration.Default, 512, BaseQIndex);
Av1TileWriter.WriteSuperblock(
picture,
context,
ref writer,
superblock,
coefficients,
tileIndex: 0);
writer.Dispose();
Assert.Equal(4096, context.CodedAreaSuperblock);
Assert.Equal(0, context.CodedAreaSuperblockUv);
for (int index = 0; index < partitions.Top.Length; index++)
{
Assert.Equal(index < 16 ? 0 : 24, partitions.Top[index].Above);
}
for (int index = 0; index < partitions.Left.Length; index++)
{
Assert.Equal(24, partitions.Left[index].Left);
}
}
[Fact]
@ -262,7 +596,7 @@ public class Av1CoefficientsEntropyTests
Av1TransformInfo transformInfo = new(transformSize, 0, 0);
int[] aboveContexts = new int[1];
int[] leftContexts = new int[1];
Av1TransformBlockContext transformBlockContext = new();
Av1TransformBlockContext transformBlockContext = default;
Configuration configuration = Configuration.Default;
Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex);
Span<int> coefficientsBuffer = [1, 2, 3, 4, 5];
@ -332,7 +666,7 @@ public class Av1CoefficientsEntropyTests
Av1TransformInfo transformInfo = new(transformSize, 0, 0);
int[] aboveContexts = new int[1];
int[] leftContexts = new int[1];
Av1TransformBlockContext transformBlockContext = new();
Av1TransformBlockContext transformBlockContext = default;
Configuration configuration = Configuration.Default;
Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex);
Span<int> coefficientsBuffer = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
@ -407,7 +741,7 @@ public class Av1CoefficientsEntropyTests
Av1TransformInfo transformInfo = new(transformSize, 0, 0);
int[] aboveContexts = new int[transformSize.Get4x4WideCount()];
int[] leftContexts = new int[transformSize.Get4x4HighCount()];
Av1TransformBlockContext transformBlockContext = new();
Av1TransformBlockContext transformBlockContext = default;
Configuration configuration = Configuration.Default;
Av1SymbolEncoder encoder = new(configuration, 100 / 8, BaseQIndex);
Span<int> coefficientsBuffer = Enumerable.Range(0, blockSize.GetHeight() * blockSize.GetWidth()).ToArray();
@ -461,6 +795,62 @@ public class Av1CoefficientsEntropyTests
}
};
private static Av1PictureControlSet CreateEncoderPicture(
int modeInfoColumnCount,
int modeInfoRowCount,
bool use128x128Superblock = false)
{
ObuTileGroupHeader tiles = new()
{
TileColumnCount = 1,
TileRowCount = 1
};
tiles.TileColumnStartModeInfo[1] = modeInfoColumnCount;
tiles.TileRowStartModeInfo[1] = modeInfoRowCount;
ObuSequenceHeader sequenceHeader = new() { Use128x128Superblock = use128x128Superblock };
ObuFrameHeader frameHeader = new()
{
ModeInfoColumnCount = modeInfoColumnCount,
ModeInfoRowCount = modeInfoRowCount,
TilesInfo = tiles
};
Av1ModeInfo[] modeInfoGrid = new Av1ModeInfo[modeInfoColumnCount * modeInfoRowCount];
for (int index = 0; index < modeInfoGrid.Length; index++)
{
modeInfoGrid[index] = CreateModeInfo(Av1PredictionMode.DC);
}
return new Av1PictureControlSet
{
PartitionContexts = [],
LuminanceDcSignLevelCoefficientNeighbors = [],
CrDcSignLevelCoefficientNeighbors = [],
CbDcSignLevelCoefficientNeighbors = [],
TransformFunctionContexts = [],
Sequence = new Av1SequenceControlSet { SequenceHeader = sequenceHeader },
Parent = new Av1PictureParentControlSet
{
Common = new Av1EncoderCommon
{
ModeInfoColumnCount = modeInfoColumnCount,
ModeInfoRowCount = modeInfoRowCount,
ModeInfoStride = modeInfoColumnCount,
FrameSize = new ObuFrameSize(),
TilesInfo = tiles
},
FrameHeader = frameHeader,
PreviousQIndex = []
},
SegmentationNeighborMap = [],
ModeInfoGrid = modeInfoGrid,
ModeInfoAllocation = modeInfoGrid,
ModeInfoStride = modeInfoColumnCount,
CdefPreset = [[-1, -1, -1, -1]]
};
}
private static Av1MacroBlockD CreateMacroBlock()
{
ObuTileGroupHeader tiles = new()

Loading…
Cancel
Save