diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 1af028959..980b3898b 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -58,7 +58,7 @@ This snapshot pins or classifies the available references and failures; it does | --- | --- | --- | --- | | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. | -| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | +| `Av1PredictionDecoder`, `Av1HighBitDepthPredictor`, `Av1ChromaFromLumaContext`, `Av1PartitionInfo`, and the scalar DC, directional, Paeth, smooth, filter-intra, and chroma-from-luma predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c`, `av1/common/reconintra.c`, `av1/common/av1_common_int.h`, `av1/common/cfl.c`, and `av1/common/cfl.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for luma/chroma mode-neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, high-bit-depth clipping, chroma-from-luma storage/subsampling, and chroma-from-luma row strides. Existing managed scalar tables and predictors remain the implementation base. The WIP rectangular byte-pipeline smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | | `Av1Inverse2dTransformer` and `Av1InverseTransformerFactory` | AV1 section 7.11.2 inverse transform and reconstruction | libaom `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, and `av1/common/idct.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar transform oracle for coefficient-row traversal, intermediate layout, stage ranges, clipping, and high-bit-depth sample addition. The managed 16-bit overload is also used as a parity oracle for the byte overload. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs index a004797c1..4ea46f7fd 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs @@ -48,7 +48,7 @@ internal partial class Av1FrameInfo // Allocate the arrays. this.superblockInfos = new Av1SuperblockInfo[superblockCount]; this.modeInfos = new Av1BlockModeInfo[superblockCount * this.modeInfoCountPerSuperblock]; - this.modeInfoMap = new Av1FrameModeInfoMap(new Size(this.modeInfoSizePerSuperblock * this.superblockColumnCount, this.modeInfoSizePerSuperblock * this.superblockRowCount), superblockSizeLog2); + this.modeInfoMap = new Av1FrameModeInfoMap(new Size(this.modeInfoSizePerSuperblock * this.superblockColumnCount, this.modeInfoSizePerSuperblock * this.superblockRowCount)); this.transformInfosY = new Av1TransformInfo[superblockCount * this.modeInfoCountPerSuperblock]; this.transformInfosUv = new Av1TransformInfo[2 * superblockCount * this.modeInfoCountPerSuperblock]; @@ -105,12 +105,7 @@ internal partial class Av1FrameInfo return span[i]; } - public Av1BlockModeInfo GetModeInfo(Point superblockIndex) - { - Span span = this.modeInfos; - int superblock = (superblockIndex.Y * this.superblockColumnCount) + superblockIndex.X; - return span[superblock * this.modeInfoCountPerSuperblock]; - } + public Av1BlockModeInfo GetModeInfo(Point superblockIndex) => this.GetModeInfo(superblockIndex, Point.Empty); public Av1BlockModeInfo GetModeInfo(Point superblockIndex, Point modeInfoIndex) { @@ -119,6 +114,11 @@ internal partial class Av1FrameInfo return this.modeInfos[index]; } + /// + /// Gets the mode information record covering the specified frame-relative mode information position. + /// + public Av1BlockModeInfo GetModeInfoAt(Point modeInfoPosition) => this.modeInfos[this.modeInfoMap[modeInfoPosition]]; + /// /// Gets the mode information records parsed for the specified superblock in bitstream order. /// @@ -223,8 +223,8 @@ internal partial class Av1FrameInfo private Point GetModeInfoPosition(Point superblockPosition, Point positionInSuperblock) { - int x = (superblockPosition.X * this.modeInfoCountPerSuperblock) + positionInSuperblock.X; - int y = (superblockPosition.Y * this.modeInfoCountPerSuperblock) + positionInSuperblock.Y; + int x = (superblockPosition.X * this.modeInfoSizePerSuperblock) + positionInSuperblock.X; + int y = (superblockPosition.Y * this.modeInfoSizePerSuperblock) + positionInSuperblock.Y; return new Point(x, y); } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameModeInfoMap.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameModeInfoMap.cs index d13b1592a..c2448e5a0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameModeInfoMap.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameModeInfoMap.cs @@ -14,13 +14,11 @@ internal partial class Av1FrameInfo public class Av1FrameModeInfoMap { private readonly ushort[] offsets; - private Size alignedModeInfoCount; + private readonly Size alignedModeInfoCount; - public Av1FrameModeInfoMap(Size modeInfoCount, int superblockSizeLog2) + public Av1FrameModeInfoMap(Size modeInfoCount) { - this.alignedModeInfoCount = new Size( - modeInfoCount.Width * (1 << (superblockSizeLog2 - Av1Constants.ModeInfoSizeLog2)), - modeInfoCount.Height * (1 << (superblockSizeLog2 - Av1Constants.ModeInfoSizeLog2))); + this.alignedModeInfoCount = modeInfoCount; this.NextIndex = 0; this.offsets = new ushort[this.alignedModeInfoCount.Width * this.alignedModeInfoCount.Height]; } diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs index 159a06e41..ac11fc78d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PartitionInfo.cs @@ -120,6 +120,40 @@ internal class Av1PartitionInfo this.HeightInPixels[2] = Math.Max(1, bh4 >> subY) * modeInfoSize; } + public void PopulateModeInfoNeighbors(Av1FrameInfo frameInfo, ObuColorConfig colorConfig) + { + if (this.AvailableAbove) + { + this.AboveModeInfo = frameInfo.GetModeInfoAt(new Point(this.ColumnIndex, this.RowIndex - 1)); + } + + if (this.AvailableLeft) + { + this.LeftModeInfo = frameInfo.GetModeInfoAt(new Point(this.ColumnIndex - 1, this.RowIndex)); + } + + if (!this.IsChroma) + { + return; + } + + int subX = colorConfig.SubSamplingX ? 1 : 0; + int subY = colorConfig.SubSamplingY ? 1 : 0; + int chromaBaseColumn = this.ColumnIndex - (this.ColumnIndex & subX); + int chromaBaseRow = this.RowIndex - (this.RowIndex & subY); + + // Chroma neighbors refer to the bottom-right luma mode covered by each adjacent chroma block. + if (this.AvailableAboveForChroma) + { + this.AboveModeInfoForChroma = frameInfo.GetModeInfoAt(new Point(chromaBaseColumn + subX, chromaBaseRow - 1)); + } + + if (this.AvailableLeftForChroma) + { + this.LeftModeInfoForChroma = frameInfo.GetModeInfoAt(new Point(chromaBaseColumn - 1, chromaBaseRow + subY)); + } + } + public int GetMaxBlockWide(Av1BlockSize blockSize, bool subX) { int maxBlockWide = blockSize.GetWidth(); diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index f8a735ffd..6e6c4008d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -290,8 +290,6 @@ internal class Av1TileReader : IAv1TileReader int block4x4Width = blockSize.Get4x4WideCount(); int block4x4Height = blockSize.Get4x4HighCount(); int planesCount = this.SequenceHeader.ColorConfig.PlaneCount; - int subX = this.SequenceHeader.ColorConfig.SubSamplingX ? 1 : 0; - int subY = this.SequenceHeader.ColorConfig.SubSamplingY ? 1 : 0; Point superblockLocation = superblockInfo.Position * this.SequenceHeader.SuperblockModeInfoSize; Point locationInSuperblock = new Point(modeInfoLocation.X - superblockLocation.X, modeInfoLocation.Y - superblockLocation.Y); Av1BlockModeInfo blockModeInfo = new(planesCount, blockSize, locationInSuperblock); @@ -317,25 +315,7 @@ internal class Av1TileReader : IAv1TileReader } } - if (partitionInfo.AvailableAbove) - { - partitionInfo.AboveModeInfo = superblockInfo.GetModeInfo(new Point(rowIndex - 1, columnIndex)); - } - - if (partitionInfo.AvailableLeft) - { - partitionInfo.LeftModeInfo = superblockInfo.GetModeInfo(new Point(rowIndex, columnIndex - 1)); - } - - if (partitionInfo.AvailableAboveForChroma) - { - partitionInfo.AboveModeInfoForChroma = superblockInfo.GetModeInfo(new Point(rowIndex & ~subY, columnIndex | subX)); - } - - if (partitionInfo.AvailableLeftForChroma) - { - partitionInfo.LeftModeInfoForChroma = superblockInfo.GetModeInfo(new Point(rowIndex | subY, columnIndex & ~subX)); - } + partitionInfo.PopulateModeInfoNeighbors(this.FrameInfo, this.SequenceHeader.ColorConfig); this.ReadModeInfo(ref reader, partitionInfo); ReadPaletteTokens(ref reader, partitionInfo); diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index 96028ab96..a66aa2d1d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -83,6 +83,8 @@ internal class Av1BlockDecoder } } + partitionInfo.PopulateModeInfoNeighbors(this.frameInfo, colorConfig); + int maxBlocksWide = partitionInfo.GetMaxBlockWide(blockSize, false); int maxBlocksHigh = partitionInfo.GetMaxBlockHigh(blockSize, false); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs index efd583760..228d99b8c 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs @@ -142,6 +142,23 @@ public class Av1TilingTests Assert.Equal(superblockInfo.BlockCount, modeInfos.Length); Assert.DoesNotContain(modeInfos.ToArray(), modeInfo => modeInfo is null); + Assert.Same(modeInfos[0], tileReader.FrameInfo.GetModeInfo(superblockPosition)); + + foreach (Av1BlockModeInfo modeInfo in modeInfos) + { + Point modeInfoPosition = new( + superblockInfo.ModeInfoPosition.X + modeInfo.PositionInSuperblock.X, + superblockInfo.ModeInfoPosition.Y + modeInfo.PositionInSuperblock.Y); + + for (int y = 0; y < modeInfo.BlockSize.Get4x4HighCount(); y++) + { + for (int x = 0; x < modeInfo.BlockSize.Get4x4WideCount(); x++) + { + Assert.Same(modeInfo, tileReader.FrameInfo.GetModeInfoAt(new Point(modeInfoPosition.X + x, modeInfoPosition.Y + y))); + } + } + } + parsedModeInfoCount += modeInfos.Length; } }