diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index c6708c6933..f454053ffe 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -65,8 +65,17 @@ verification before advancing. The following dependency result does not close th `Av1EncoderBlockWorkspace` appends exactly 131,072 integer elements (512 KiB) only for inter workers. The sequence owner retains this storage across frames; still-image workers retain their previous allocation. Native evidence: `block.h:763-793`, `encoder_alloc.h:57-75`, `encodemv.c:125-250`, `rd.c:687-705`, - and `encodeframe_utils.c:1663-1675`. Per-superblock refresh is connected; speed-dependent row/set refresh - remains to be integrated with the new configuration controller. + and `encodeframe_utils.c:1663-1675`. Motion-cost refresh now follows the configured superblock, row, or + evenly spaced row-set cadence. Levels 3 and above use rows; levels 5 and above below 720p use row sets. + Evidence: `speed_features.c:1002,1315`, `encodeframe_utils.c:1556-1589,1628-1630,1663-1675`. + `Av1MotionSearchSettings` resolves the policy and `Av1TileEncoder.ProcessTiles` applies it to the actual + serial traversal, initializing each tile even when CDF adaptation is disabled. The frame-height rounding + yields evenly distributed update rows for both 64- and 128-sample superblocks, including short final tiles. + Release .NET 11 passed with zero errors and 1,009 existing warnings after the final edit. VSTest passed + 324/324 focused frame, motion-policy, and superblock cases. Optimized native decoding matched all 39 + compact streams (86,859 samples), plus three 129x273 multi-row streams (477,243 samples): maximum error + zero, differing samples zero, samples exceeding one zero. These checks do not establish encoder parity + or a measured performance improvement. Mode and coefficient cost-refresh lifecycles remain unresolved. - Inter-mode rate calculation now applies the missing rounded 108/128 weight to motion-vector rate alone. Evidence: `mcomp.c:306-312`, `rd.h:46`, `motion_search_facade.c:535-542`; managed owning method is `Av1IntraSuperblockEncoder.ReferenceModeDecision.GetInterModeRate`. Search still requires the separate @@ -272,7 +281,7 @@ verification before advancing. The following dependency result does not close th The fixture now supplies an explicit entropy history favoring GLOBALMV and checks its cost against every competing mode; all preexisting expected skip/rate/distortion/coefficient/pixel assertions remain intact. - This remains the same active production milestone. Scaled references, production temporal analysis, - broader block/reference support, complete mode pruning and ordering, and speed-dependent entropy-cost refresh + broader block/reference support, complete mode pruning and ordering, and mode/coefficient cost refresh remain open. Still-image and intra/global-motion Effort policies are not migrated. The production decoder comparisons establish same-bitstream equality only, not separate-encoder parity, performance, or codec completion. diff --git a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionSearchSettings.cs b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionSearchSettings.cs index 9827bcf680..97571ad193 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionSearchSettings.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionSearchSettings.cs @@ -94,6 +94,7 @@ internal readonly struct Av1MotionSearchSettings this.SecondCandidateSelection = CandidateSelection.FirstOnly; this.MeshPruningLevel = 1; this.AllowIntraBlockCopy = false; + this.MotionCostUpdate = CostUpdateFrequency.SuperblockRow; } if (speed >= HeifEncodingSpeed.Level4) @@ -140,6 +141,7 @@ internal readonly struct Av1MotionSearchSettings if (!is720pOrLarger) { this.DownsampledSadLevel = 1; + this.MotionCostUpdate = CostUpdateFrequency.SuperblockRowSet; } } @@ -185,6 +187,27 @@ internal readonly struct Av1MotionSearchSettings } } + /// + /// The serial tile traversal boundaries at which motion costs are refreshed. + /// + public enum CostUpdateFrequency + { + /// + /// Refresh before each superblock. + /// + Superblock, + + /// + /// Refresh at the first superblock of each tile row. + /// + SuperblockRow, + + /// + /// Refresh at evenly spaced sets of superblock rows within a tile. + /// + SuperblockRowSet + } + /// /// The full-pixel search pattern. /// @@ -304,6 +327,11 @@ internal readonly struct Av1MotionSearchSettings FirstOnly } + /// + /// Gets the frequency at which selected motion symbols refresh the search cost tables. + /// + public CostUpdateFrequency MotionCostUpdate { get; } + /// /// Gets the adaptation level for the initial full-pixel step. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs index 38588b0d47..80b0cab266 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs @@ -327,6 +327,7 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter ObuTileGroupHeader tileLayout = frameHeader.TilesInfo; Span tileDataOffsets = picture.TileDataOffsets.Span; Span tileDataLengths = picture.TileDataLengths.Span; + Av1MotionSearchSettings.CostUpdateFrequency motionCostUpdate = picture.Parent.MotionSearchSettings.MotionCostUpdate; if (!TSymbolOperation.WritesOutput && frameHeader.AllowIntraBlockCopy) { // Hash the visible source once before reconstruction begins so candidate discovery never depends @@ -348,6 +349,17 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter // advances the output offset; the analysis operation does not touch range-coder state. writer.Reset(tileDataEnd); + int motionCostRowInterval = 1; + if (motionCostUpdate == Av1MotionSearchSettings.CostUpdateFrequency.SuperblockRowSet) + { + // Target one update per 256 luma rows, then distribute those updates evenly over the + // tile's superblock rows. Two rounded divisions keep short final tiles evenly spaced. + int tileHeight = (tile.ModeInfoRowEnd - tile.ModeInfoRowStart) << Av1Constants.ModeInfoSizeLog2; + int updateCount = (tileHeight + 255) / 256; + int updateSpan = updateCount << sequenceHeader.SuperblockSizeLog2; + motionCostRowInterval = (tileHeight + updateSpan - 1) / updateSpan; + } + Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart); entropyContext.MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition); for (int modeInfoRow = tile.ModeInfoRowStart; @@ -379,10 +391,17 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter } else { - if (!frameHeader.IsIntra) + bool firstColumn = modeInfoColumn == tile.ModeInfoColumnStart; + bool firstSuperblock = firstColumn && modeInfoRow == tile.ModeInfoRowStart; + int tileSuperblockRow = (modeInfoRow - tile.ModeInfoRowStart) >> superblockShift; + bool refreshMotionCosts = motionCostUpdate == Av1MotionSearchSettings.CostUpdateFrequency.Superblock || + (firstColumn && (tileSuperblockRow % motionCostRowInterval) == 0); + + if (!frameHeader.IsIntra && (firstSuperblock || (!frameHeader.DisableCdfUpdate && refreshMotionCosts))) { - // Candidates within a superblock share one entropy snapshot. Updating while - // trying partitions would make the search depend on discarded alternatives. + // Initialize from each tile's starting CDF even when adaptation is disabled. + // Later updates consume only preceding selected blocks at the configured boundary; + // all candidate trials between boundaries share the same cost snapshot. writer.FillMotionVectorCosts(blockWorkspace.GetMotionVectorCosts(frameHeader.MotionVectorPrecision)); } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs index 64bf6bd712..cca45bdd63 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs @@ -392,26 +392,46 @@ public class Av1EncoderFrameTests int colorFormatValue, int effort, HeifEncodingSpeed speed) + => VerifySequenceEncoderColorPlanes(bitDepthValue, colorFormatValue, effort, speed, 23, 19); + + [Theory] + [InlineData(HeifEncodingSpeed.Level0)] + [InlineData(HeifEncodingSpeed.Level3)] + [InlineData(HeifEncodingSpeed.Level5)] + public void SequenceEncoderPreservesNativeColorPlanesAcrossMotionCostRefreshRows(HeifEncodingSpeed speed) + => VerifySequenceEncoderColorPlanes(EightBit, Yuv420, 8, speed, 129, 273); + + private static void VerifySequenceEncoderColorPlanes( + int bitDepthValue, + int colorFormatValue, + int effort, + HeifEncodingSpeed speed, + int width, + int height) { - const int Width = 23; - const int Height = 19; + // A 129x273 frame crosses both columns and uneven row sets of 64- or 128-sample superblocks. + // The compact fixture retains its odd visible edges and all precision/subsampling combinations. const int QIndex = 17; const int ByteToUInt16Scale = ushort.MaxValue / byte.MaxValue; Av1BitDepth bitDepth = (Av1BitDepth)bitDepthValue; Av1ColorFormat colorFormat = (Av1ColorFormat)colorFormatValue; ObuColorConfig colorConfig = CreateColorConfig(bitDepth, colorFormat); ReadOnlySpan period = [0, 28, 40, 28, 0, -28, -40, -12]; - using Image source = new(Width, Height); + using Image source = new(width, height); using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( Configuration.Default, - Width, - Height, + width, + height, colorConfig, QIndex, effort, speed); - string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion)); + string outputFolder = width == 23 ? nameof(SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion) + : nameof(SequenceEncoderPreservesNativeColorPlanesAcrossMotionCostRefreshRows); + + string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", outputFolder); + string outputName = $"{bitDepth.GetBitCount()}-{colorFormat}-effort{effort}-speed{(int)speed}"; using FileStream output = File.Create(Path.Combine(outputDirectory, outputName + ".obu")); using BinaryWriter rawOutput = new(File.Create(Path.Combine(outputDirectory, outputName + ".managed.yuv"))); @@ -422,13 +442,13 @@ public class Av1EncoderFrameTests // The second source translates all three channels by one luma sample on each axis. Chroma is // converted independently by the production converter, so 4:2:0 and 4:2:2 cannot hide behind // constant neutral planes. Odd dimensions also exercise each plane's visible-edge clipping. - for (int y = 0; y < Height; y++) + for (int y = 0; y < height; y++) { Span row = source.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); - int referenceY = Math.Min(y + frameIndex, Height - 1); - for (int x = 0; x < Width; x++) + int referenceY = Math.Min(y + frameIndex, height - 1); + for (int x = 0; x < width; x++) { - int referenceX = Math.Min(x + frameIndex, Width - 1); + int referenceX = Math.Min(x + frameIndex, width - 1); row[x] = new Rgb48( (ushort)((128 + period[referenceX % period.Length]) * ByteToUInt16Scale), (ushort)((128 + period[referenceY % period.Length]) * ByteToUInt16Scale), @@ -451,8 +471,8 @@ public class Av1EncoderFrameTests decoder.DecodeSequenceReference(sample.ToArray(), null, null); Assert.True(Assert.IsType(decoder.SequenceHeader).EnableIntraEdgeFilter); Av1FrameBuffer decoded = Assert.IsType>(decoder.FrameBuffer); - Assert.Equal(Width, decoded.Width); - Assert.Equal(Height, decoded.Height); + Assert.Equal(width, decoded.Width); + Assert.Equal(height, decoded.Height); Assert.Equal(bitDepth, decoded.BitDepth); Av1FrameInfo decodedFrameInfo = Assert.IsType(decoder.FrameInfo); foreach (Av1BlockModeInfo mode in decodedFrameInfo.GetModeInfos(Point.Empty, decodedFrameInfo.GetModeInfoCount(Point.Empty))) @@ -470,7 +490,7 @@ public class Av1EncoderFrameTests Av1Plane plane = (Av1Plane)planeIndex; int subsamplingX = plane == Av1Plane.Y || !colorConfig.SubSamplingX ? 0 : 1; int subsamplingY = plane == Av1Plane.Y || !colorConfig.SubSamplingY ? 0 : 1; - int planeHeight = (Height + subsamplingY) >> subsamplingY; + int planeHeight = (height + subsamplingY) >> subsamplingY; if (bitDepth == Av1BitDepth.EightBit) { Buffer2DRegion planeSamples = decoded.DeriveBlockPointer(plane, subsamplingX, subsamplingY);