From 37541f7f3c94479cacb7428736e9b60b1f16c575 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sat, 5 Sep 2026 18:03:12 +1000 Subject: [PATCH] Encode AV1 palette maps through coded frame boundaries --- HEIF_IMPLEMENTATION_PLAN.md | 50 +++++ ...rblockEncoder.ChromaPaletteModeDecision.cs | 8 +- ...raSuperblockEncoder.PaletteModeDecision.cs | 7 +- .../Formats/Heif/Av1/Tiling/Av1TileWriter.cs | 7 +- .../Av1/Av1IntraSuperblockEncoderTests.cs | 204 ++++++++++++++---- 5 files changed, 221 insertions(+), 55 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 52e7065cf8..dea9a97980 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -228,6 +228,56 @@ Further quantization, distortion, and final-packing source comparison on 2026-09 delayed packing. Simply flipping CDEF/restoration flags or saving only probability state is insufficient. - This comparison ran no benchmark or runtime test and establishes no new numerical or performance result. +Further controller-state findings after `b2aee3036` on 2026-09-05: + +- Reference `av1/encoder/block.h:239-259`, `av1/encoder/rdopt.h:315-328`, and + `av1/encoder/partition_search.c:1555-1556` retain the winning reference-MV stack, weights, count, mode context, + and global vectors for final packing. `av1/encoder/bitstream.c:1062-1089,1133-1158,1251-1264` consumes that + retained state for inter-mode, DRL, MV, and IBC symbols. `Av1TileWriter.cs:928-975` currently rebuilds the stack + at the immediate-write boundary. Delaying the write requires preserving its decision-time state, not assuming + a rebuild against a completed frame grid is equivalent. +- Reference `av1/common/av1_common_int.h:1775-1856` derives final partition structure from the retained mode grid. + A second frame-sized partition-tree copy is therefore not required by the reference architecture. +- `Av1SymbolEncoder.cs:424-480,1183-1334` calculates candidate costs from live adaptive distributions. + Reference `av1/encoder/rd.c:82-130,602-668` fills distinct mode/coefficient cost tables, including marginal + coefficient costs needed by optimization. `av1/encoder/encodeframe_utils.c:1556-1692` updates mode, + coefficient, MV, and displacement-vector costs at separately configured superblock/row/tile frequencies; + `av1/encoder/speed_features.c:2385-2387` starts the inter cost policies at superblock frequency. + Updating CDFs after each selected block is not the same as refreshing every RD cost after that block. + This missing cost-state lifecycle is an architectural deviation. Its contribution to time or quality has + not been measured; no isolated cache or invented effort-dependent refresh policy has been introduced. + +Palette coded-boundary correction, verified after `b2aee3036` on 2026-09-05: + +- The encoder clipped luma/chroma palette search to visible frame dimensions + (`Av1IntraSuperblockEncoder.PaletteModeDecision.cs:46-49`, + `Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs:54-57` before correction), and + `Av1TileWriter.cs:1085-1086` omitted palette symbols outside those visible dimensions. This is a numerical + and syntax defect, separate from the conditional distortion-model border policy described above. +- Reference palette search (`av1/encoder/palette.c:555-597,787-804`), tokenization + (`av1/encoder/tokenize.c:229-241`), and decoding (`av1/decoder/detokenize.c:65-77`) all use + `av1/common/blockd.h:1517-1557`. Its distances come from coded mode-info dimensions + (`av1/common/av1_common_int.h:1358-1364`), not visible-pixel dimensions or the optional RD border policy. + The managed decoder already follows those coded distances (`Av1PartitionInfo.cs:190-194`, + `Av1TileReader.cs:2720-2744`). Search, rate evaluation, and writing now agree on that same extent. +- The existing 5x3 luma regression checked retained reconstruction without decoding its payload. Extending it + to decode every case exposed a truncated tile entropy stream (`palette-bounds-red.trx`). An earlier assertion + incorrectly read `FrameBuffer` after `Decode` had released the native planes; that test mistake was corrected + using `DecodeFrameBuffer` and is not codec-failure evidence (`palette-bounds-before.trx`). +- Existing palette/color/EOB/reconstruction assertions remain. Added cases cover clipped/transposed luma at + 10/12 bits and 4:4:4, 4:2:2, and 4:2:0 chroma, including one-pixel source axes. The same boundary correction + prevents those subsampled axes from creating zero-length palette input. No defensive rejection, new owner, + extra production buffer, or change to search effort thresholds was introduced. +- Final Release .NET 11 build: zero errors, 1,009 existing warnings. A preceding build stopped on a missing + blank line before a comment; it was corrected before verification. Roslynk reports zero compiler errors. + Serialized Visual Studio VSTest passes 190/190 cases in 18.4734 seconds: intra-superblock encoder, + HEIF encoder, and AV1 palette cases (`palette-bounds-final.trx`). +- Optimized official libaom decodes all 23 regenerated palette payloads with exact agreement against retained + encoder reconstruction: 985 visible Y/U/V samples, maximum error 0, zero samples exceeding one unit. + Per-plane results and output sizes are in `D:\GitHub\ynse01\av1-takeover-20260905\palette-comparison.json`. + All generated streams, raw planes, scripts, and native output remain temporary and excluded from commits. + This is same-bitstream reconstruction verification, not separate-encoder parity or performance acceptance. + Explicit grid-sampling conversion correction, verified after `5f7bad3a6` on 2026-09-05: - `HeifEncoderCore.Sequence.cs:97-109` promoted incompatible odd-grid sampling only when the public option diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs index 1afa5f6140..e50dcef246 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs @@ -51,9 +51,11 @@ internal static partial class Av1IntraSuperblockEncoder int subsamplingY = colorConfig.SubSamplingY ? 1 : 0; int width = transformSize.GetWidth(); int height = transformSize.GetHeight(); - ObuFrameSize frameSize = this.picture.Parent.FrameHeader.FrameSize; - int rows = Math.Min(LumaBlockLength, frameSize.FrameHeight - lumaOrigin.Y) >> subsamplingY; - int columns = Math.Min(LumaBlockLength, frameSize.FrameWidth - lumaOrigin.X) >> subsamplingX; + + // Clip against the coded mode-info boundary before subsampling, as the decoder does. Visible odd + // dimensions still have complete coded chroma samples; truncating them here can leave an empty palette input. + int rows = (LumaBlockLength + (Math.Min(0, macroBlock.ToBottomEdge) >> 3)) >> subsamplingY; + int columns = (LumaBlockLength + (Math.Min(0, macroBlock.ToRightEdge) >> 3)) >> subsamplingX; int activeSampleCount = rows * columns; Span blueSamples = workspace.GetSamples(0)[..activeSampleCount]; Span redSamples = workspace.GetSamples(1)[..activeSampleCount]; diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs index fa8cdb39c4..b0ccb8ee9c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs @@ -43,9 +43,10 @@ internal static partial class Av1IntraSuperblockEncoder Av1EncoderPaletteWorkspace workspace = this.blockWorkspace.GetModeDecisionWorkspace().Palette; - ObuFrameSize frameSize = this.picture.Parent.FrameHeader.FrameSize; - int rows = Math.Min(BlockLength, frameSize.FrameHeight - blockOrigin.Y); - int columns = Math.Min(BlockLength, frameSize.FrameWidth - blockOrigin.X); + // Palette samples and symbols extend to the coded mode-info boundary, including replicated source + // alignment. The signed edge distances use eighth-sample units and exclude only blocks beyond that boundary. + int rows = BlockLength + (Math.Min(0, macroBlock.ToBottomEdge) >> 3); + int columns = BlockLength + (Math.Min(0, macroBlock.ToRightEdge) >> 3); int sampleCount = rows * columns; Span samples = workspace.GetSamples(0)[..sampleCount]; TOperator.CopyPaletteSamples(sourcePlane, blockOrigin, rows, columns, samples); diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs index a757f23c94..235d0f27d9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs @@ -1082,8 +1082,11 @@ internal partial class Av1TileWriter int blockHeight = blockSize.GetHeight(); int planeWidth = blockWidth >> subX; int planeHeight = blockHeight >> subY; - int columns = Math.Min(blockWidth, frm_hdr.FrameSize.FrameWidth - blockOrigin.X) >> subX; - int rows = Math.Min(blockHeight, frm_hdr.FrameSize.FrameHeight - blockOrigin.Y) >> subY; + + // Palette syntax covers coded alignment samples too. Visible-frame clipping would omit symbols + // that the decoder consumes before transform syntax and corrupt the remainder of the tile. + int columns = (blockWidth + (Math.Min(0, macroBlock.ToRightEdge) >> 3)) >> subX; + int rows = (blockHeight + (Math.Min(0, macroBlock.ToBottomEdge) >> 3)) >> subY; Buffer2DRegion colorIndexMap = tb_ptr.Workspace .GetPaletteMaps() .GetMap(planeType, planeWidth, planeHeight); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs index 36f982c1bb..7d5c558aea 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs @@ -1131,6 +1131,41 @@ public class Av1IntraSuperblockEncoderTests effort: 5)); } + [Theory] + [InlineData(5, 3)] + [InlineData(3, 5)] + [InlineData(1, 5)] + public void ProductionTileSelectsExactLumaPaletteAtClippedHighBitDepths(int width, int height) + { + AssertProductionTileSelectsExactLumaPalette( + Av1BitDepth.TenBit, + 10, + width, + height, + false, + (ushort)128, + (ushort)896, + 128, + 896, + static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new Av1TileEncoder( + writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace, effort: 5)); + + AssertProductionTileSelectsExactLumaPalette( + Av1BitDepth.TwelveBit, + 12, + width, + height, + false, + (ushort)512, + (ushort)3584, + 512, + 3584, + static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new Av1TileEncoder( + writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace, effort: 5)); + } + [Fact] public void ProductionTileSelectsLumaPaletteWithFourByFourTransforms() { @@ -1180,61 +1215,92 @@ public class Av1IntraSuperblockEncoderTests [Fact] public void ProductionTileSelectsExactPairedChromaPalette() { - AssertProductionTileSelectsExactPairedChromaPalette(useLumaPalette: false); - AssertProductionTileSelectsExactPairedChromaPalette(useLumaPalette: true); + AssertProductionTileSelectsExactPairedChromaPalette(false, 8, 8, false, false); + AssertProductionTileSelectsExactPairedChromaPalette(true, 8, 8, false, false); } - private static void AssertProductionTileSelectsExactPairedChromaPalette(bool useLumaPalette) + [Theory] + [InlineData(5, 3, false, false)] + [InlineData(3, 5, false, false)] + [InlineData(5, 3, true, false)] + [InlineData(3, 5, true, false)] + [InlineData(5, 3, true, true)] + [InlineData(3, 5, true, true)] + [InlineData(1, 5, true, false)] + [InlineData(5, 1, true, false)] + [InlineData(1, 5, true, true)] + [InlineData(5, 1, true, true)] + public void ProductionTileSelectsExactPairedChromaPaletteAtClippedSizes(int width, int height, bool subX, bool subY) + => AssertProductionTileSelectsExactPairedChromaPalette(false, width, height, subX, subY); + + private static void AssertProductionTileSelectsExactPairedChromaPalette( + bool useLumaPalette, + int width, + int height, + bool subX, + bool subY) { - const int Width = 8; - const int Height = 8; const int QIndex = 37; ObuColorConfig colorConfig = new() { IsMonochrome = false, - SubSamplingX = false, - SubSamplingY = false, + SubSamplingX = subX, + SubSamplingY = subY, BitDepth = Av1BitDepth.EightBit }; + Av1ColorFormat colorFormat = subY ? Av1ColorFormat.Yuv420 : subX ? Av1ColorFormat.Yuv422 : Av1ColorFormat.Yuv444; using Av1EncoderFrameBuffer source = new( Configuration.Default, - Width, - Height, + width, + height, 8, - Av1ColorFormat.Yuv444, + colorFormat, 0, 0); using Av1EncoderFrameBuffer reconstruction = new( Configuration.Default, - Width, - Height, + width, + height, 8, - Av1ColorFormat.Yuv444, + colorFormat, 0, 0); Buffer2DRegion lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion blueSource = source.Frame.CodedView.GetPlane(Av1Plane.U); Buffer2DRegion redSource = source.Frame.CodedView.GetPlane(Av1Plane.V); - for (int row = 0; row < Height; row++) + for (int row = 0; row < lumaSource.Height; row++) { Span lumaRow = lumaSource.DangerousGetRowSpan(row); if (useLumaPalette) { - for (int column = 0; column < Width; column++) + for (int column = 0; column < lumaRow.Length; column++) { - lumaRow[column] = column < Width / 2 ? (byte)64 : (byte)192; + lumaRow[column] = column < width / 2 ? (byte)64 : (byte)192; } } else { lumaRow.Fill(128); } + } - blueSource.DangerousGetRowSpan(row).Fill(row < Height / 2 ? (byte)32 : (byte)224); - redSource.DangerousGetRowSpan(row).Fill(row < Height / 2 ? (byte)200 : (byte)40); + int chromaWidth = (width + (subX ? 1 : 0)) >> (subX ? 1 : 0); + int chromaHeight = (height + (subY ? 1 : 0)) >> (subY ? 1 : 0); + for (int row = 0; row < blueSource.Height; row++) + { + for (int column = 0; column < blueSource.Width; column++) + { + // Repeat the last visible sample into coded alignment, including one-pixel source axes. + bool firstColor = chromaHeight > 1 + ? Math.Min(row, chromaHeight - 1) < chromaHeight / 2 + : Math.Min(column, chromaWidth - 1) < chromaWidth / 2; + + blueSource.DangerousGetRowSpan(row)[column] = firstColor ? (byte)32 : (byte)224; + redSource.DangerousGetRowSpan(row)[column] = firstColor ? (byte)200 : (byte)40; + } } ClearPlane(reconstruction.Luma); @@ -1242,8 +1308,8 @@ public class Av1IntraSuperblockEncoderTests ClearPlane(Assert.IsType>(reconstruction.ChromaRed)); using Av1EncoderModeInfoBuffer modeInfo = new( Configuration.Default, - Width, - Height, + width, + height, disallow4x4AllFrames: true); Av1PictureControlSet pictureTemplate = CreatePicture( @@ -1253,21 +1319,21 @@ public class Av1IntraSuperblockEncoderTests QIndex); pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true; - pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = Width; - pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = Height; + pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = width; + pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = height; using Av1EncoderPictureBuffer picture = new( Configuration.Default, pictureTemplate.Sequence.SequenceHeader, pictureTemplate.Parent.FrameHeader, - Width, - Height, + width, + height, disallow4x4AllFrames: true); using Av1EncoderCoefficientBuffer coefficients = new( Configuration.Default, pictureTemplate.Sequence.SequenceHeader, - Width, - Height); + width, + height); using Av1EncoderSuperblockWorkspace superblockWorkspace = new(Configuration.Default); using Av1EncoderBlockWorkspace blockWorkspace = new(Configuration.Default); @@ -1296,22 +1362,55 @@ public class Av1IntraSuperblockEncoderTests Buffer2DRegion colorIndexMap = superblockWorkspace .GetPaletteMaps() - .GetMap(Av1PlaneType.Uv, Width, Height); + .GetMap(Av1PlaneType.Uv, blueSource.Width, blueSource.Height); Buffer2DRegion blueReconstruction = reconstruction.Frame.CodedView.GetPlane(Av1Plane.U); Buffer2DRegion redReconstruction = reconstruction.Frame.CodedView.GetPlane(Av1Plane.V); - for (int row = 0; row < Height; row++) + for (int row = 0; row < blueSource.Height; row++) { - byte expectedIndex = (byte)(row < Height / 2 ? 0 : 1); - foreach (byte index in colorIndexMap.DangerousGetRowSpan(row)) + for (int column = 0; column < blueSource.Width; column++) { - Assert.Equal(expectedIndex, index); + byte expectedIndex = blueSource.DangerousGetRowSpan(row)[column] == 32 ? (byte)0 : (byte)1; + Assert.Equal(expectedIndex, colorIndexMap.DangerousGetRowSpan(row)[column]); } Assert.True(blueSource.DangerousGetRowSpan(row).SequenceEqual(blueReconstruction.DangerousGetRowSpan(row))); Assert.True(redSource.DangerousGetRowSpan(row).SequenceEqual(redReconstruction.DangerousGetRowSpan(row))); } + byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, width, height); + using Av1Decoder decoder = new(Configuration.Default); + using Av1FrameBuffer decodedFrame = decoder.DecodeFrameBuffer(payload, null, null, out _); + Assert.Equal(width, decodedFrame.Width); + Assert.Equal(height, decodedFrame.Height); + Assert.NotNull(decoder.FrameInfo); + Assert.Equal(2, decoder.FrameInfo.GetModeInfoAt(default).GetPaletteSize(Av1PlaneType.Uv)); + for (int plane = 0; plane < 3; plane++) + { + int planeSubX = plane > 0 && subX ? 1 : 0; + int planeSubY = plane > 0 && subY ? 1 : 0; + Buffer2DRegion actual = decodedFrame.DeriveBlockPointer((Av1Plane)plane, planeSubX, planeSubY); + Buffer2DRegion expected = reconstruction.Frame.View.GetPlane((Av1Plane)plane); + for (int row = 0; row < expected.Height; row++) + { + Assert.Equal(expected.DangerousGetRowSpan(row), actual.DangerousGetRowSpan(row)); + } + } + + string outputDirectory = Path.Combine(TestEnvironment.ActualOutputDirectoryFullPath, "Formats", "Heif", "Av1"); + string outputName = $"encoder-palette-chroma-{width}x{height}-{subX}-{subY}-{useLumaPalette}"; + Directory.CreateDirectory(outputDirectory); + File.WriteAllBytes(Path.Combine(outputDirectory, outputName + ".obu"), payload); + using FileStream raw = File.Create(Path.Combine(outputDirectory, outputName + ".retained.yuv")); + for (int plane = 0; plane < 3; plane++) + { + Buffer2DRegion retained = reconstruction.Frame.View.GetPlane((Av1Plane)plane); + for (int row = 0; row < retained.Height; row++) + { + raw.Write(retained.DangerousGetRowSpan(row)); + } + } + Assert.NotEqual(0, tileWriter.GetTileData(0).Length); } @@ -3319,29 +3418,40 @@ public class Av1IntraSuperblockEncoderTests } } + // Decode every payload, including clipped maps: retained reconstruction alone cannot reveal missing map symbols. + byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, width, height); + using Av1Decoder decoder = new(Configuration.Default); + using Image decoded = decoder.Decode(payload); + Assert.NotNull(decoder.FrameInfo); + Av1BlockModeInfo decodedBlock = decoder.FrameInfo.GetModeInfoAt(default); + Assert.True(decodedBlock.GetPaletteSize(Av1Plane.Y) > 0); + Assert.Equal(new Size(width, height), decoded.Size); + using Av1FrameBuffer decodedFrame = decoder.DecodeFrameBuffer(payload, null, null, out _); + Buffer2DRegion decodedPlane = decodedFrame.DeriveBlockPointer(Av1Plane.Y, 0, 0); + for (int row = 0; row < height; row++) + { + ReadOnlySpan decodedSamples = MemoryMarshal.Cast(decodedPlane.DangerousGetRowSpan(row)); + Assert.Equal(reconstructionPlane.DangerousGetRowSpan(row)[..width], decodedSamples); + } + if (useSplitTransform) { Assert.True(predictionOnlyError > 0); Assert.True(reconstructionError < predictionOnlyError); - byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, width, height); - using Av1Decoder decoder = new(Configuration.Default); - using Image decoded = decoder.Decode(payload); - Assert.NotNull(decoder.FrameInfo); - Av1BlockModeInfo decodedBlock = decoder.FrameInfo.GetModeInfoAt(default); - Assert.True(decodedBlock.GetPaletteSize(Av1Plane.Y) > 0); Assert.Equal(4, decodedBlock.GetTransformUnitCount(Av1Plane.Y)); - Assert.Equal(new Size(width, height), decoded.Size); + } - string outputDirectory = Path.Combine( - TestEnvironment.ActualOutputDirectoryFullPath, - "Formats", - "Heif", - "Av1"); + string outputDirectory = Path.Combine(TestEnvironment.ActualOutputDirectoryFullPath, "Formats", "Heif", "Av1"); + string outputName = useSplitTransform + ? $"encoder-palette-transform-size-select-{bitDepthValue}b" + : $"encoder-palette-luma-{bitDepthValue}b-{width}x{height}"; - Directory.CreateDirectory(outputDirectory); - File.WriteAllBytes( - Path.Combine(outputDirectory, $"encoder-palette-transform-size-select-{bitDepthValue}b.obu"), - payload); + Directory.CreateDirectory(outputDirectory); + File.WriteAllBytes(Path.Combine(outputDirectory, outputName + ".obu"), payload); + using FileStream raw = File.Create(Path.Combine(outputDirectory, outputName + ".retained.yuv")); + for (int row = 0; row < height; row++) + { + raw.Write(MemoryMarshal.AsBytes(reconstructionPlane.DangerousGetRowSpan(row)[..width])); } Assert.NotEqual(0, tileWriter.GetTileData(0).Length);