Browse Source

Fix AV1 encoder mode info mapping

pull/2633/head
James Jackson-South 1 month ago
parent
commit
2e9cf86670
  1. 1
      HEIF_IMPLEMENTATION_PLAN.md
  2. 34
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs
  3. 35
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockD.cs
  4. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockModeInfo.cs
  5. 11
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PaletteLumaModeInfo.cs
  6. 52
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs
  7. 34
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs
  8. 117
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

1
HEIF_IMPLEMENTATION_PLAN.md

@ -821,6 +821,7 @@ Encoder verification contract:
- [~] Current-libaom `av1_quantize_fp_no_qmatrix` arithmetic is implemented as a closed generic forward-quantizer family with Vector512, Vector256, Vector128, and scalar paths, raster-order output, coded 64-point coefficient limits, and scan-order EOB selection. Transform search, coefficient optimization, and lossless behavior remain.
- [ ] Implement real rate-distortion selection and make quality and effort change work, size, and output quality.
- [~] The tile writer now publishes one packed coefficient context per covered 4x4 edge unit and derives luma/chroma skip plus DC-sign contexts from the complete transform edges using current-libaom units. Complete tile traversal, initialized picture state, and verified CDF update behavior remain.
- [~] Encoder mode information now uses a frame-owned reference grid over its contiguous allocation, matching current libaom's `mi_grid_base` and `mi_alloc` ownership without per-block tail copies. Signed relative neighbor lookup, 4x4-unit addressing, and mutable selected skip syntax have focused contracts; complete mode decision still remains.
- [ ] Implement legal deblocking, CDEF, restoration, super-resolution, and film-grain signaling decisions.
- [~] The coefficient symbol encoder now reuses tile-lifetime level and context workspaces instead of allocating per transform. Every remaining encoder fragment must be audited before it becomes active.
- [~] The planar conversion, forward transform, and forward quantizer use descending SIMD dispatch: Vector512, Vector256, Vector128, then scalar. Apply the same rule to every later hot-path family.

34
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs

@ -11,44 +11,34 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
internal class Av1EncoderBlockModeInfo
{
/// <summary>
/// Gets the encoded block size.
/// Gets or sets the encoded block size.
/// </summary>
public Av1BlockSize BlockSize { get; }
public Av1BlockSize BlockSize { get; set; }
/// <summary>
/// Gets the selected luma prediction mode.
/// Gets or sets the partition type that produced the block.
/// </summary>
public Av1PredictionMode PredictionMode { get; }
public Av1PartitionType PartitionType { get; set; }
/// <summary>
/// Gets the partition type that produced the block.
/// Gets or sets a value indicating whether residual coefficients are omitted for the block.
/// </summary>
public Av1PartitionType PartitionType { get; }
public bool Skip { get; set; }
/// <summary>
/// Gets the selected chroma prediction mode.
/// Gets or sets a value indicating whether compound skip mode is selected.
/// </summary>
public Av1ChromaPredictionMode UvPredictionMode { get; }
public bool SkipMode { get; set; }
/// <summary>
/// Gets a value indicating whether residual coefficients are omitted for the block.
/// Gets or sets a value indicating whether intra block copy is selected.
/// </summary>
public bool Skip { get; } = true;
public bool UseIntraBlockCopy { get; set; }
/// <summary>
/// Gets a value indicating whether compound skip mode is selected.
/// Gets or sets the segmentation identifier assigned to the block.
/// </summary>
public bool SkipMode { get; } = true;
/// <summary>
/// Gets a value indicating whether intra block copy is selected.
/// </summary>
public bool UseIntraBlockCopy { get; } = true;
/// <summary>
/// Gets the segmentation identifier assigned to the block.
/// </summary>
public int SegmentId { get; }
public int SegmentId { get; set; }
/// <summary>
/// Gets or sets the transform-tree depth selected for the block.

35
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockD.cs

@ -9,23 +9,14 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
internal class Av1MacroBlockD
{
/// <summary>
/// Stores the mode-information entries exposed through <see cref="ModeInfo"/>.
/// Stores the frame's mode-information reference grid.
/// </summary>
private Av1ModeInfo[] modeInfo = [];
private Av1ModeInfo[] modeInfoGrid = [];
/// <summary>
/// Gets or sets the mode-information entries for the current block and its mapped neighbors.
/// Stores the current block's linear position in <see cref="modeInfoGrid"/>.
/// </summary>
public required ReadOnlySpan<Av1ModeInfo> ModeInfo
{
get => this.modeInfo;
set
{
// A span cannot be retained by the class, so preserve the selected map entries in owned storage.
this.modeInfo = new Av1ModeInfo[value.Length];
value.CopyTo(this.modeInfo);
}
}
private int modeInfoIndex;
/// <summary>
/// Gets or sets the tile containing the current block.
@ -86,4 +77,22 @@ internal class Av1MacroBlockD
/// Gets or sets a value indicating whether this block is the second half of a rectangular partition.
/// </summary>
public bool IsSecondRectangle { get; set; }
/// <summary>
/// Selects the current entry in the frame-owned mode-information reference grid.
/// </summary>
/// <param name="grid">The frame-owned mode-information reference grid.</param>
/// <param name="index">The current block's linear grid index.</param>
public void SetModeInfoGrid(Av1ModeInfo[] grid, int index)
{
this.modeInfoGrid = grid;
this.modeInfoIndex = index;
}
/// <summary>
/// Gets a mode-information entry relative to the current block.
/// </summary>
/// <param name="offset">The signed linear offset from the current block.</param>
/// <returns>The mapped neighboring or current entry.</returns>
public Av1ModeInfo GetRelativeModeInfo(int offset) => this.modeInfoGrid[this.modeInfoIndex + offset];
}

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1MacroBlockModeInfo.cs

@ -13,11 +13,6 @@ internal class Av1MacroBlockModeInfo
/// </summary>
public required Av1EncoderBlockModeInfo Block { get; set; }
/// <summary>
/// Gets or sets the luma palette decisions for the block.
/// </summary>
public required Av1PaletteLumaModeInfo Palette { get; set; }
/// <summary>
/// Gets or sets the constrained directional enhancement filter strength for the block.
/// </summary>

11
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PaletteLumaModeInfo.cs

@ -1,11 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
/// Reserves encoder-side state for AV1 luma palette mode decisions.
/// </summary>
internal class Av1PaletteLumaModeInfo
{
}

52
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs

@ -1,8 +1,6 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary>
@ -51,14 +49,14 @@ internal class Av1PictureControlSet
public required byte[] SegmentationNeighborMap { get; set; }
/// <summary>
/// Gets the frame grid that maps each 4x4 position to its mode-information span.
/// Gets or sets the frame grid that maps each 4x4 position to its mode-information entry.
/// </summary>
public Av1ModeInfo[][] ModeInfoGrid { get; } = [];
public required Av1ModeInfo[] ModeInfoGrid { get; set; }
/// <summary>
/// Gets or sets the contiguous mode-information storage addressed by <see cref="ModeInfoGrid"/>.
/// </summary>
public required Av1ModeInfo[] Mip { get; set; }
public required Av1ModeInfo[] ModeInfoAllocation { get; set; }
/// <summary>
/// Gets or sets the row stride of <see cref="ModeInfoGrid"/> in 4x4 mode-information units.
@ -76,51 +74,29 @@ internal class Av1PictureControlSet
public required int[][] CdefPreset { get; set; }
/// <summary>
/// Gets the mode-information span mapped to a frame position.
/// Gets the mode-information entry mapped to a frame position.
/// </summary>
/// <param name="position">The frame position in 4x4 mode-information units.</param>
/// <returns>The mode-information span beginning at the position.</returns>
public Span<Av1ModeInfo> GetFromModeInfoGrid(Point position)
/// <returns>The mapped mode-information entry.</returns>
public Av1ModeInfo GetFromModeInfoGrid(Point position)
=> this.ModeInfoGrid[(position.Y * this.ModeInfoStride) + position.X];
/// <summary>
/// Maps a frame position to the supplied mode-information span.
/// </summary>
/// <param name="position">The frame position in 4x4 mode-information units.</param>
/// <param name="span">The mode-information entries to map.</param>
public void SetModeInfoGridRow(Point position, ReadOnlySpan<Av1ModeInfo> span)
=> this.SetModeInfoGridRow((position.Y * this.ModeInfoStride) + position.X, span);
/// <summary>
/// Maps a linear grid offset to the supplied mode-information span.
/// </summary>
/// <param name="offset">The linear grid offset.</param>
/// <param name="span">The mode-information entries to map.</param>
public void SetModeInfoGridRow(int offset, ReadOnlySpan<Av1ModeInfo> span)
{
// Grid entries own their arrays because the source span can refer to temporary traversal state.
this.ModeInfoGrid[offset] = new Av1ModeInfo[span.Length];
span.CopyTo(this.ModeInfoGrid[offset]);
}
/// <summary>
/// Gets the macroblock mode information at a block origin and refreshes its grid mapping.
/// </summary>
/// <param name="blockOrigin">The block origin in 4x4 mode-information units.</param>
/// <param name="modeInfoPosition">The block position in 4x4 mode-information units.</param>
/// <returns>The macroblock mode information at the origin.</returns>
public Av1MacroBlockModeInfo GetMacroBlockModeInfo(Point blockOrigin)
public Av1MacroBlockModeInfo GetMacroBlockModeInfo(Point modeInfoPosition)
{
int modeInfoStride = this.ModeInfoStride;
int offset = (blockOrigin.Y * modeInfoStride) + blockOrigin.X;
int offset = (modeInfoPosition.Y * modeInfoStride) + modeInfoPosition.X;
// Rectangular mode-decision blocks can replace grid entries. Restore the entry from the
// contiguous backing store, whose index is halved when 4x4 blocks are globally disabled.
// The grid stores references into the contiguous mode-info allocation, matching libaom's
// mi_grid_base/mi_alloc ownership without copying a tail for every coded block.
int disallow4x4 = this.Disallow4x4AllFrames ? 1 : 0;
int mipOffset = ((blockOrigin.Y >> disallow4x4) * (modeInfoStride >> disallow4x4)) + (blockOrigin.X >> disallow4x4);
this.SetModeInfoGridRow(offset, ((Span<Av1ModeInfo>)this.Mip)[mipOffset..]);
// The first mapped entry owns the macroblock state for the entire block.
Av1ModeInfo modeInfo = this.ModeInfoGrid[offset][0];
int allocationOffset = ((modeInfoPosition.Y >> disallow4x4) * (modeInfoStride >> disallow4x4)) + (modeInfoPosition.X >> disallow4x4);
Av1ModeInfo modeInfo = this.ModeInfoAllocation[allocationOffset];
this.ModeInfoGrid[offset] = modeInfo;
return modeInfo.MacroBlockModeInfo;
}

34
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs

@ -364,26 +364,26 @@ internal partial class Av1TileWriter
Av1BlockGeometry blockGeometry = Av1BlockGeometryFactory.GetBlockGeometryByModeDecisionScanIndex(blk_ptr.ModeDecisionScanIndex);
Point blockOrigin = Point.Add(entropyCodingContext.SuperblockOrigin, (Size)blockGeometry.Origin);
Av1BlockSize blockSize = blockGeometry.BlockSize;
Av1MacroBlockModeInfo macroBlockModeInfo = pcs.GetMacroBlockModeInfo(blockOrigin);
int mi_row = blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2;
int mi_col = blockOrigin.X >> Av1Constants.ModeInfoSizeLog2;
int mi_stride = pcs.Parent.Common.ModeInfoStride;
int offset = (mi_row * mi_stride) + mi_col;
Point modeInfoPosition = new(mi_col, mi_row);
Av1MacroBlockModeInfo macroBlockModeInfo = pcs.GetMacroBlockModeInfo(modeInfoPosition);
bool skipWritingCoefficients = macroBlockModeInfo.Block.Skip;
entropyCodingContext.MacroBlockModeInfo = macroBlockModeInfo;
bool skip_mode = macroBlockModeInfo.Block.SkipMode;
Guard.MustBeLessThan((int)blockSize, (int)Av1BlockSize.AllSizes, nameof(blockSize));
int mi_row = blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2;
int mi_col = blockOrigin.X >> Av1Constants.ModeInfoSizeLog2;
int mi_stride = pcs.Parent.Common.ModeInfoStride;
int offset = (mi_row * mi_stride) + mi_col;
Point modeInfoPosition = new(mi_col, mi_row);
blk_ptr.MacroBlock.ModeInfo = pcs.ModeInfoGrid[offset];
blk_ptr.MacroBlock.SetModeInfoGrid(pcs.ModeInfoGrid, offset);
blk_ptr.MacroBlock.Tile = new Av1TileInfo(tb_ptr.TileInfo);
blk_ptr.MacroBlock.IsUpAvailable = modeInfoPosition.Y > tb_ptr.TileInfo.ModeInfoRowStart;
blk_ptr.MacroBlock.IsLeftAvailable = modeInfoPosition.X > tb_ptr.TileInfo.ModeInfoColumnStart;
if (blk_ptr.MacroBlock.IsUpAvailable)
{
blk_ptr.MacroBlock.AboveMacroBlock = blk_ptr.MacroBlock.ModeInfo[-mi_stride].MacroBlockModeInfo;
blk_ptr.MacroBlock.AboveMacroBlock = blk_ptr.MacroBlock.GetRelativeModeInfo(-mi_stride).MacroBlockModeInfo;
}
else
{
@ -392,7 +392,7 @@ internal partial class Av1TileWriter
if (blk_ptr.MacroBlock.IsLeftAvailable)
{
blk_ptr.MacroBlock.LeftMacroBlock = blk_ptr.MacroBlock.ModeInfo[-1].MacroBlockModeInfo;
blk_ptr.MacroBlock.LeftMacroBlock = blk_ptr.MacroBlock.GetRelativeModeInfo(-1).MacroBlockModeInfo;
}
else
{
@ -600,12 +600,12 @@ internal partial class Av1TileWriter
if (xd.IsLeftAvailable)
{
// Key-frame neighbors are intra blocks, so their luma modes directly select the context class.
intraLumaLeftMode = xd.ModeInfo[-1].MacroBlockModeInfo.Block.Mode;
intraLumaLeftMode = xd.GetRelativeModeInfo(-1).MacroBlockModeInfo.Block.Mode;
}
if (xd.IsUpAvailable)
{
intraLumaTopMode = xd.ModeInfo[-xd.ModeInfoStride].MacroBlockModeInfo.Block.Mode;
intraLumaTopMode = xd.GetRelativeModeInfo(-xd.ModeInfoStride).MacroBlockModeInfo.Block.Mode;
}
above_ctx = IntraModeContextLookup[(int)intraLumaTopMode];
@ -738,7 +738,8 @@ internal partial class Av1TileWriter
Av1NeighborArrayUnit<byte> cr_dc_sign_level_coeff_na = pcs.CrDcSignLevelCoefficientNeighbors[tile_idx];
Av1NeighborArrayUnit<byte> cb_dc_sign_level_coeff_na = pcs.CbDcSignLevelCoefficientNeighbors[tile_idx];
Av1BlockGeometry blk_geom = Av1BlockGeometryFactory.GetBlockGeometryByModeDecisionScanIndex(blk_ptr.ModeDecisionScanIndex);
Av1MacroBlockModeInfo mbmi = pcs.GetMacroBlockModeInfo(blockOrigin);
Point modeInfoPosition = blockOrigin >> Av1Constants.ModeInfoSizeLog2;
Av1MacroBlockModeInfo mbmi = pcs.GetMacroBlockModeInfo(modeInfoPosition);
bool skip_coeff = mbmi.Block.Skip;
// Store the block-size split mask across the edges that future partition symbols can observe.
@ -843,7 +844,7 @@ internal partial class Av1TileWriter
// int m = ~((1 << (6 - Av1Constants.ModeInfoSizeLog2)) - 1);
// cm->mi_grid_visible[(mi_row & m) * cm->mi_stride + (mi_col & m)];
Av1ModeInfo mi = pcs.GetFromModeInfoGrid(modeInfoPosition)[0];
Av1ModeInfo mi = pcs.GetFromModeInfoGrid(modeInfoPosition);
// Each superblock begins with all contained 64x64 filter units unassigned.
if ((modeInfoPosition.Y & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0 &&
@ -897,11 +898,12 @@ internal partial class Av1TileWriter
// Prediction cannot cross tile boundaries even when frame mode information exists there.
macroBlock.IsUpAvailable = modeInfoPosition.Y > tile.ModeInfoRowStart;
macroBlock.IsLeftAvailable = modeInfoPosition.X > tile.ModeInfoColumnStart;
macroBlock.ModeInfo = pcs.GetFromModeInfoGrid(modeInfoPosition);
int modeInfoIndex = (modeInfoPosition.Y * modeInfoStride) + modeInfoPosition.X;
macroBlock.SetModeInfoGrid(pcs.ModeInfoGrid, modeInfoIndex);
if (macroBlock.IsUpAvailable)
{
macroBlock.AboveMacroBlock = macroBlock.ModeInfo[-modeInfoStride].MacroBlockModeInfo;
macroBlock.AboveMacroBlock = macroBlock.GetRelativeModeInfo(-modeInfoStride).MacroBlockModeInfo;
}
else
{
@ -910,7 +912,7 @@ internal partial class Av1TileWriter
if (macroBlock.IsLeftAvailable)
{
macroBlock.LeftMacroBlock = macroBlock.ModeInfo[-1].MacroBlockModeInfo;
macroBlock.LeftMacroBlock = macroBlock.GetRelativeModeInfo(-1).MacroBlockModeInfo;
}
else
{

117
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

@ -4,6 +4,7 @@
using System.Buffers;
using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
@ -72,6 +73,113 @@ public class Av1CoefficientsEntropyTests
Assert.Equal(expectedSkipContext, context.SkipContext);
}
[Theory]
[InlineData(false, 2, 1, 6, 6)]
[InlineData(true, 2, 2, 10, 3)]
public void PictureControlSetMapsModeInfoAllocationByReference(
bool disallow4x4,
int column,
int row,
int gridOffset,
int allocationOffset)
{
Av1ModeInfo expected = CreateModeInfo(Av1PredictionMode.Paeth);
Av1ModeInfo[] allocation = new Av1ModeInfo[8];
allocation[allocationOffset] = expected;
Av1PictureControlSet picture = new()
{
PartitionContexts = [],
LuminanceDcSignLevelCoefficientNeighbors = [],
CrDcSignLevelCoefficientNeighbors = [],
CbDcSignLevelCoefficientNeighbors = [],
TransformFunctionContexts = [],
Sequence = new Av1SequenceControlSet { SequenceHeader = new ObuSequenceHeader() },
Parent = new Av1PictureParentControlSet
{
Common = new Av1EncoderCommon
{
FrameSize = new ObuFrameSize(),
TilesInfo = new ObuTileGroupHeader()
},
FrameHeader = new ObuFrameHeader(),
PreviousQIndex = [],
SuperblockGeometry = []
},
SegmentationNeighborMap = [],
ModeInfoGrid = new Av1ModeInfo[16],
ModeInfoAllocation = allocation,
ModeInfoStride = 4,
Disallow4x4AllFrames = disallow4x4,
CdefPreset = []
};
Av1MacroBlockModeInfo result = picture.GetMacroBlockModeInfo(new Point(column, row));
Assert.Same(expected.MacroBlockModeInfo, result);
Assert.Same(expected, picture.ModeInfoGrid[gridOffset]);
}
[Fact]
public void MacroBlockReadsNeighborsRelativeToCurrentGridEntry()
{
Av1ModeInfo above = CreateModeInfo(Av1PredictionMode.Vertical);
Av1ModeInfo left = CreateModeInfo(Av1PredictionMode.Horizontal);
Av1ModeInfo current = CreateModeInfo(Av1PredictionMode.DC);
Av1ModeInfo[] grid = new Av1ModeInfo[9];
grid[1] = above;
grid[3] = left;
grid[4] = current;
ObuTileGroupHeader tiles = new()
{
TileColumnCount = 1,
TileRowCount = 1
};
tiles.TileColumnStartModeInfo[1] = 3;
tiles.TileRowStartModeInfo[1] = 3;
ObuFrameHeader frameHeader = new()
{
ModeInfoColumnCount = 3,
ModeInfoRowCount = 3,
TilesInfo = tiles
};
Av1MacroBlockD macroBlock = new() { Tile = new Av1TileInfo(0, 0, frameHeader) };
macroBlock.SetModeInfoGrid(grid, 4);
Assert.Same(left, macroBlock.GetRelativeModeInfo(-1));
Assert.Same(above, macroBlock.GetRelativeModeInfo(-3));
Assert.Same(current, macroBlock.GetRelativeModeInfo(0));
}
[Fact]
public void EncoderBlockModeInfoStoresSelectedSyntax()
{
Av1EncoderBlockModeInfo modeInfo = new();
Assert.False(modeInfo.Skip);
Assert.False(modeInfo.SkipMode);
Assert.False(modeInfo.UseIntraBlockCopy);
modeInfo.Skip = true;
modeInfo.SkipMode = true;
modeInfo.UseIntraBlockCopy = true;
modeInfo.BlockSize = Av1BlockSize.Block16x16;
modeInfo.PartitionType = Av1PartitionType.Split;
modeInfo.SegmentId = 3;
modeInfo.Mode = Av1PredictionMode.Smooth;
modeInfo.UvMode = Av1ChromaPredictionMode.Smooth;
Assert.True(modeInfo.Skip);
Assert.True(modeInfo.SkipMode);
Assert.True(modeInfo.UseIntraBlockCopy);
Assert.Equal(Av1BlockSize.Block16x16, modeInfo.BlockSize);
Assert.Equal(Av1PartitionType.Split, modeInfo.PartitionType);
Assert.Equal(3, modeInfo.SegmentId);
Assert.Equal(Av1PredictionMode.Smooth, modeInfo.Mode);
Assert.Equal(Av1ChromaPredictionMode.Smooth, modeInfo.UvMode);
}
[Fact]
public void RoundTripZeroEndOfBlock()
{
@ -277,6 +385,15 @@ public class Av1CoefficientsEntropyTests
}
}
private static Av1ModeInfo CreateModeInfo(Av1PredictionMode mode)
=> new()
{
MacroBlockModeInfo = new Av1MacroBlockModeInfo
{
Block = new Av1EncoderBlockModeInfo { Mode = mode }
}
};
public static TheoryData<int> GetTransformTypes()
{
TheoryData<int> result = [];

Loading…
Cancel
Save