diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 988620acfc..11d36c31c6 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -868,7 +868,7 @@ Encoder verification contract: - [x] Operation-wide allocation tracking now exercises a real 64x64 12-bit 4:4:4 frame through packed-pixel conversion, both native frame owners, picture and coefficient state, reusable block workspaces, entropy coding, OBU framing, and a non-seekable destination. It proves exactly one 60 KiB tile-output reservation from current libaom's all-intra 2.5x rule and balanced exactly-once returns for every tracked allocation before the operation completes. The focused ownership case passes 1 of 1 and the complete HEIF/AV1 namespace passes 8,863 of 8,863 direct net11 VSTest cases with zero failures or skips. - [x] HEIF box offsets are now counted from the start of the encoded file instead of reading `Stream.Position`. This preserves ISO BMFF file-relative `iloc` offsets when the destination begins at a nonzero position and permits non-seekable output. Decoder item extents and image-sequence chunk offsets now resolve from that same file origin rather than the backing stream origin. Real legacy-JPEG HEIF round trips cover non-seekable output and a prefixed destination, while current-position AV1 decode covers both a still item and a five-frame sequence. All 96 encoder/decoder cases and all 38 sequence-parser cases pass direct net11 Release VSTest; the Release build remains at the established 1,005-warning baseline with zero errors. -- [~] Effort-six uniform luma transform selection now compares the winning ordinary spatial mode as one 8x8 transform against four raster-ordered 4x4 transforms. Each 4x4 transform searches every legal type with coefficient contexts derived from the already retained transform edges and the preceding trial blocks, while reconstructed top-right and bottom-left references follow production coding order. The strided transform operator writes each candidate directly into its 8x8 reconstruction mosaic. Prepared prediction and residual data are reused across transform trials, and the existing aligned block-workspace owner retains four final transform states, local coefficient contexts, coefficients, and compact trial reconstruction; no allocator rent, managed array, best-candidate re-transform, or full-block intermediate copy was added. Non-skipped intra-block copy under `TX_MODE_SELECT` emits and costs the unsplit variable-transform root required by current libaom, while skipped intra-block copy emits no transform-partition symbol. Uniform transform-size contexts use coding-block extents for intra-block-copy neighbors and residual-transform extents for intra neighbors. The focused contract passes 21 of 21 direct net11 Release VSTest cases, including an independently decoded stream that proves at least one real four-transform luma block. The complete non-HEVC HEIF/AV1 namespace passes 9,294 of 9,294 cases with zero failures or skips. The exact Release build remains at 1,005 baseline warnings and zero errors, Roslynk reports no touched-file diagnostics, and current-main `aomdec` accepts both generated effort-six streams with decoded-frame MD5 values `2dd1cbe449fe2d0471dc2c15c50acb69` and `677435e5af39c930af1178f91c34af6a`. Transform-size integration for filter-intra and palette, broader joint mode/transform refinement, partition search, and effort-dependent pruning remain. +- [~] Effort-six uniform luma transform selection now compares the winning ordinary spatial or filter-intra mode as one 8x8 transform against four raster-ordered 4x4 transforms. Each 4x4 transform searches every legal type with coefficient contexts derived from retained transform edges and preceding trial blocks, while reconstructed top-right and bottom-left references follow production coding order. Filter-intra trials rebuild each recursive prediction from those reconstructed edges and charge the retained filter mode in both header and coefficient syntax. The strided transform operator writes each candidate directly into its 8x8 reconstruction mosaic, and the existing aligned block-workspace owner retains prediction, residual, coefficients, contexts, compact reconstruction, and four final states; no allocator rent, managed array, best-candidate re-transform, or full-block intermediate copy was added. Non-skipped intra-block copy under `TX_MODE_SELECT` emits and costs the unsplit variable-transform root required by current libaom, while skipped intra-block copy emits no transform-partition symbol. Uniform transform-size contexts use coding-block extents for intra-block-copy neighbors and residual-transform extents for intra neighbors. The 18-case filter-intra group passes at 8, 10, and 12 bits, including full decoder syntax checks for a selected four-transform filter block, and the complete non-HEVC HEIF/AV1 namespace passes 9,297 of 9,297 cases with zero failures or skips. The exact Release build remains at 1,005 baseline warnings and zero errors. Current-main `aomdec` at `a40ed1ea9e4ecc3df58a5bccb76623f2c94ae727` accepts the three new filter-intra streams and both earlier effort-six streams; the earlier decoded-frame MD5 values remain `2dd1cbe449fe2d0471dc2c15c50acb69` and `677435e5af39c930af1178f91c34af6a`. Transform-size integration for palette, broader joint mode/transform refinement, partition search, and effort-dependent pruning remain. ### 7. Write complete AVIF output diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs index 15459befb1..fcfc36c29e 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs @@ -815,8 +815,7 @@ internal static partial class Av1IntraSuperblockEncoder selectedTransformSize = TransformSize; if (this.effort >= 6 && this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select && - paletteInfo.PaletteSizes[0] == 0 && - selectedFilterIntraMode == Av1FilterIntraMode.AllFilterIntraModes) + paletteInfo.PaletteSizes[0] == 0) { long splitCost = this.GetSplitLumaCandidateCost( writer, @@ -827,6 +826,7 @@ internal static partial class Av1IntraSuperblockEncoder tileIndex, bestMode, selectedAngleDelta, + selectedFilterIntraMode, paletteDisabledCost, transformSizeContext, bestTransformCost, @@ -863,6 +863,7 @@ internal static partial class Av1IntraSuperblockEncoder ushort tileIndex, Av1PredictionMode mode, int angleDelta, + Av1FilterIntraMode filterIntraMode, int paletteDisabledCost, int transformSizeContext, long costLimit, @@ -909,7 +910,7 @@ internal static partial class Av1IntraSuperblockEncoder if (this.picture.Sequence.SequenceHeader.EnableFilterIntra) { rate += writer.GetFilterIntraModeCost( - Av1FilterIntraMode.AllFilterIntraModes, + filterIntraMode, BlockSize); } } @@ -941,20 +942,39 @@ internal static partial class Av1IntraSuperblockEncoder out bool hasLeft, out bool hasAbove); - TOperator.PrepareIntra( - this.blockWorkspace, - sourcePlane, - transformOrigin, - prediction, - aboveStorage.Slice(1, TransformWidth * 2), - leftStorage.Slice(1, TransformWidth * 2), - hasLeft, - hasAbove, - mode, - angleDelta, - residual, - TransformSize, - this.bitDepth); + if (filterIntraMode == Av1FilterIntraMode.AllFilterIntraModes) + { + TOperator.PrepareIntra( + this.blockWorkspace, + sourcePlane, + transformOrigin, + prediction, + aboveStorage.Slice(1, TransformWidth * 2), + leftStorage.Slice(1, TransformWidth * 2), + hasLeft, + hasAbove, + mode, + angleDelta, + residual, + TransformSize, + this.bitDepth); + } + else + { + // Filter-intra prediction is recursive within each transform unit, so rebuild it from + // the reconstructed edges established by the preceding 4x4 candidate. + TOperator.PrepareFilterIntra( + this.blockWorkspace, + sourcePlane, + transformOrigin, + prediction, + aboveStorage.Slice(1, TransformWidth * 2), + leftStorage.Slice(1, TransformWidth * 2), + residual, + filterIntraMode, + TransformSize, + this.bitDepth); + } Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts( Av1ComponentType.Luminance, @@ -1009,7 +1029,7 @@ internal static partial class Av1IntraSuperblockEncoder blockContext, candidateState.EndOfBlock, useReducedTransformSet, - Av1FilterIntraMode.AllFilterIntraModes); + filterIntraMode); long candidateCost = Av1RateDistortion.GetCost( this.rateMultiplier, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs index e9377db1b1..15beae25d0 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs @@ -13,6 +13,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -1520,6 +1521,7 @@ public class Av1IntraSuperblockEncoderTests => VerifyProductionTileSelectsFilterIntraMode( filterIntraModeValue, 8, + false, static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => new( Configuration.Default, @@ -1531,9 +1533,9 @@ public class Av1IntraSuperblockEncoderTests blockWorkspace, initialSize: 512, effort: 5), - static (mode, destination, above, left, _, scratch) => + static (mode, destination, stride, above, left, width, height, _, scratch) => Av1FilterIntraPredictorBase.GetPredictor(mode) - .Predict(destination, 8, above, left, 8, 8, scratch)); + .Predict(destination, stride, above, left, width, height, scratch)); [Theory] [InlineData((int)Av1FilterIntraMode.DC, 10)] @@ -1552,6 +1554,7 @@ public class Av1IntraSuperblockEncoderTests => VerifyProductionTileSelectsFilterIntraMode( filterIntraModeValue, bitDepth, + false, static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => new( Configuration.Default, @@ -1563,28 +1566,81 @@ public class Av1IntraSuperblockEncoderTests blockWorkspace, initialSize: 512, effort: 5), - static (mode, destination, above, left, sampleBitDepth, scratch) => + static (mode, destination, stride, above, left, width, height, sampleBitDepth, scratch) => Av1FilterIntraPredictorBase.GetPredictor(mode) .Predict( MemoryMarshal.Cast(destination), - 8, + stride, MemoryMarshal.Cast(above), MemoryMarshal.Cast(left), - 8, - 8, + width, + height, + sampleBitDepth, + MemoryMarshal.Cast(scratch))); + + [Fact] + public void ProductionTileSelectsFilterIntraWithFourByFourTransforms() + => VerifyProductionTileSelectsFilterIntraMode( + (int)Av1FilterIntraMode.DC, + 8, + true, + static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new( + Configuration.Default, + source, + reconstruction, + picture, + coefficients, + superblockWorkspace, + blockWorkspace, + initialSize: 512, + effort: 6), + static (mode, destination, stride, above, left, width, height, _, scratch) => + Av1FilterIntraPredictorBase.GetPredictor(mode) + .Predict(destination, stride, above, left, width, height, scratch)); + + [Theory] + [InlineData(10)] + [InlineData(12)] + public void ProductionTileSelectsFilterIntraWithFourByFourTransformsHighBitDepth(int bitDepth) + => VerifyProductionTileSelectsFilterIntraMode( + (int)Av1FilterIntraMode.DC, + bitDepth, + true, + static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => + new( + Configuration.Default, + source, + reconstruction, + picture, + coefficients, + superblockWorkspace, + blockWorkspace, + initialSize: 512, + effort: 6), + static (mode, destination, stride, above, left, width, height, sampleBitDepth, scratch) => + Av1FilterIntraPredictorBase.GetPredictor(mode) + .Predict( + MemoryMarshal.Cast(destination), + stride, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + width, + height, sampleBitDepth, MemoryMarshal.Cast(scratch))); private static void VerifyProductionTileSelectsFilterIntraMode( int filterIntraModeValue, int bitDepth, + bool useSplitTransform, TileWriterFactory createWriter, FilterPrediction predictFilter) where TSample : unmanaged, IBinaryInteger { const int Width = 16; const int Height = 16; - const int QIndex = 1; + const int QIndex = 37; const int TargetX = 8; const int TargetY = 8; const Av1TransformSize TransformSize = Av1TransformSize.Size8x8; @@ -1631,6 +1687,10 @@ public class Av1IntraSuperblockEncoderTests using Av1EncoderModeInfoBuffer pilotModeInfo = new(Configuration.Default, Width, Height, disallow4x4AllFrames: true); Av1PictureControlSet pilotTemplate = CreatePicture(pilotModeInfo, colorConfig, use128x128Superblock: false, QIndex); pilotTemplate.Sequence.SequenceHeader.EnableFilterIntra = true; + pilotTemplate.Parent.FrameHeader.TransformMode = useSplitTransform + ? Av1TransformMode.Select + : Av1TransformMode.Largest; + using Av1EncoderPictureBuffer pilotPicture = new( Configuration.Default, pilotTemplate.Sequence.SequenceHeader, @@ -1668,7 +1728,87 @@ public class Av1IntraSuperblockEncoderTests Span target = stackalloc TSample[TransformSize.GetSize2d()]; Span filterScratch = stackalloc TSample[Av1FilterIntraPredictorBase.ScratchLength]; - predictFilter(filterIntraMode, target, above, left, bitDepth, filterScratch); + if (useSplitTransform) + { + Span transformAboveStorage = stackalloc TSample[5]; + Span transformAbove = transformAboveStorage[1..]; + Span transformLeft = stackalloc TSample[4]; + for (int transformRow = 0; transformRow < 2; transformRow++) + { + int rowOffset = transformRow * 4; + for (int transformColumn = 0; transformColumn < 2; transformColumn++) + { + int columnOffset = transformColumn * 4; + ReadOnlySpan availableAbove = transformRow == 0 + ? above.Slice(columnOffset, 4) + : target.Slice(((rowOffset - 1) * 8) + columnOffset, 4); + + // The predictor consumes the corner through the element immediately before the top-edge span. + // Later transforms therefore use already reconstructed samples from the same 8-by-8 block. + transformAboveStorage[0] = transformRow == 0 + ? transformColumn == 0 ? aboveStorage[0] : above[columnOffset - 1] + : transformColumn == 0 ? left[rowOffset - 1] : target[((rowOffset - 1) * 8) + columnOffset - 1]; + + availableAbove.CopyTo(transformAbove); + for (int row = 0; row < 4; row++) + { + transformLeft[row] = transformColumn == 0 + ? left[rowOffset + row] + : target[((rowOffset + row) * 8) + columnOffset - 1]; + } + + int destinationOffset = (rowOffset * 8) + columnOffset; + predictFilter( + filterIntraMode, + target[destinationOffset..], + 8, + transformAbove, + transformLeft, + 4, + 4, + bitDepth, + filterScratch); + } + } + } + else + { + predictFilter(filterIntraMode, target, 8, above, left, 8, 8, bitDepth, filterScratch); + } + + if (useSplitTransform) + { + for (int transformRow = 0; transformRow < 2; transformRow++) + { + for (int transformColumn = 0; transformColumn < 2; transformColumn++) + { + int transformIndex = (transformRow * 2) + transformColumn; + + // Distinct transform-local frequency patterns remain compact in separate 4-by-4 bases but spread + // across coefficients when a single 8-by-8 transform spans the discontinuities between quadrants. + for (int row = 0; row < 4; row++) + { + Span targetRow = target.Slice( + (((transformRow * 4) + row) * 8) + (transformColumn * 4), + 4); + + for (int column = 0; column < targetRow.Length; column++) + { + int residualSign = transformIndex switch + { + 0 => row < 2 ? -1 : 1, + 1 => column < 2 ? -1 : 1, + 2 => (row < 2) == (column < 2) ? -1 : 1, + _ => ((row + column) & 1) == 0 ? -1 : 1 + }; + + targetRow[column] = TSample.CreateChecked( + int.CreateChecked(targetRow[column]) + (residualSign * 40 * sampleScale)); + } + } + } + } + } using Av1EncoderFrameBuffer source = new( Configuration.Default, @@ -1703,6 +1843,10 @@ public class Av1IntraSuperblockEncoderTests using Av1EncoderModeInfoBuffer modeInfo = new(Configuration.Default, Width, Height, disallow4x4AllFrames: true); Av1PictureControlSet pictureTemplate = CreatePicture(modeInfo, colorConfig, use128x128Superblock: false, QIndex); pictureTemplate.Sequence.SequenceHeader.EnableFilterIntra = true; + pictureTemplate.Parent.FrameHeader.TransformMode = useSplitTransform + ? Av1TransformMode.Select + : Av1TransformMode.Largest; + using Av1EncoderPictureBuffer picture = new( Configuration.Default, pictureTemplate.Sequence.SequenceHeader, @@ -1726,28 +1870,126 @@ public class Av1IntraSuperblockEncoderTests superblockWorkspace, blockWorkspace); + ref Av1MacroBlockModeInfo targetBlock = ref picture.Picture.GetMacroBlockModeInfo(new Point(2, 2)); + Assert.Equal(Av1PredictionMode.DC, targetBlock.Block.Mode); + Assert.Equal(filterIntraMode, superblockWorkspace.FinalBlocks[3].FilterIntraMode); + Assert.Equal( + useSplitTransform ? Av1TransformSize.Size4x4 : Av1TransformSize.Size8x8, + targetBlock.Block.TransformSize); + + int targetTransformIndex = (3 * TransformSize.GetSize2d()) / + Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + + int targetTransformCount = useSplitTransform ? 4 : 1; + Span targetStates = coefficients + .GetTransformBlockSpan(0, Av1Plane.Y) + .Slice(targetTransformIndex, targetTransformCount); + + foreach (Av1EncoderTransformBlockState targetState in targetStates) + { + if (useSplitTransform) + { + Assert.NotEqual((ushort)0, targetState.EndOfBlock); + } + else + { + Assert.Equal((ushort)0, targetState.EndOfBlock); + Assert.Equal(Av1TransformType.DctDct, targetState.TransformType); + } + } + Buffer2DRegion actualLuma = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y); Assert.Equal(above, actualLuma.DangerousGetRowSpan(TargetY - 1).Slice(TargetX, 8)); + long reconstructionError = 0; for (int row = 0; row < 8; row++) { Assert.Equal(left[row], actualLuma.DangerousGetRowSpan(TargetY + row)[TargetX - 1]); - Assert.Equal( - target.Slice(row * 8, 8), - actualLuma.DangerousGetRowSpan(TargetY + row).Slice(TargetX, 8)); + ReadOnlySpan targetRow = target.Slice(row * 8, 8); + ReadOnlySpan actualRow = actualLuma.DangerousGetRowSpan(TargetY + row).Slice(TargetX, 8); + if (useSplitTransform) + { + for (int column = 0; column < targetRow.Length; column++) + { + long difference = long.CreateChecked(targetRow[column]) - long.CreateChecked(actualRow[column]); + reconstructionError += difference * difference; + } + } + else + { + Assert.Equal(targetRow, actualRow); + } } - ref Av1MacroBlockModeInfo targetBlock = ref picture.Picture.GetMacroBlockModeInfo(new Point(2, 2)); - Assert.Equal(Av1PredictionMode.DC, targetBlock.Block.Mode); - Assert.Equal(filterIntraMode, superblockWorkspace.FinalBlocks[3].FilterIntraMode); - - int targetTransformIndex = (3 * TransformSize.GetSize2d()) / - Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount; + if (useSplitTransform) + { + // The chosen transforms must reduce the source error below leaving the known residual entirely uncoded. + long predictionOnlyError = 64L * 40 * 40 * sampleScale * sampleScale; + Assert.InRange(reconstructionError, 1, predictionOnlyError - 1); + + // Tile fixtures initialize only entropy state. Complete the same still-picture headers as the frame + // encoder before serializing so the independent decoder validates the real OBU syntax. + ObuSequenceHeader sequenceHeader = pictureTemplate.Sequence.SequenceHeader; + sequenceHeader.IsStillPicture = true; + sequenceHeader.IsReducedStillPictureHeader = true; + sequenceHeader.SequenceProfile = bitDepth == 12 + ? ObuSequenceProfile.Professional + : ObuSequenceProfile.Main; + + sequenceHeader.OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }]; + sequenceHeader.FrameWidthBits = 4; + sequenceHeader.FrameHeightBits = 4; + sequenceHeader.MaxFrameWidth = Width; + sequenceHeader.MaxFrameHeight = Height; + sequenceHeader.ForceScreenContentTools = 2; + sequenceHeader.ForceIntegerMotionVector = 2; + + ObuFrameHeader frameHeader = pictureTemplate.Parent.FrameHeader; + frameHeader.FrameType = ObuFrameType.KeyFrame; + frameHeader.ShowFrame = true; + frameHeader.ErrorResilientMode = true; + frameHeader.RefreshFrameFlags = byte.MaxValue; + frameHeader.DisableFrameEndUpdateCdf = true; + frameHeader.FrameSize = new ObuFrameSize + { + FrameWidth = Width, + FrameHeight = Height, + SuperResolutionDenominator = Av1Constants.ScaleNumerator, + SuperResolutionUpscaledWidth = Width, + RenderWidth = Width, + RenderHeight = Height + }; - Av1EncoderTransformBlockState targetState = - coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[targetTransformIndex]; + frameHeader.TilesInfo.HasUniformTileSpacing = true; + using MemoryStream stream = new(); + new ObuWriter().WriteAll( + Configuration.Default, + stream, + sequenceHeader, + frameHeader, + tileWriter); + + byte[] payload = stream.ToArray(); + using Av1Decoder decoder = new(Configuration.Default); + using Image decoded = decoder.Decode(payload); + Assert.NotNull(decoder.FrameInfo); + Av1BlockModeInfo decodedBlock = decoder.FrameInfo.GetModeInfoAt(new Point(2, 2)); + Assert.True(decodedBlock.UseFilterIntra); + Assert.Equal(filterIntraMode, decodedBlock.FilterIntraMode); + Assert.Equal(4, decodedBlock.GetTransformUnitCount(Av1Plane.Y)); + Assert.Equal(new Size(Width, Height), decoded.Size); + + string outputDirectory = Path.Combine( + TestEnvironment.ActualOutputDirectoryFullPath, + "Formats", + "Heif", + "Av1"); + + Directory.CreateDirectory(outputDirectory); + File.WriteAllBytes( + Path.Combine(outputDirectory, $"encoder-filter-intra-transform-size-select-{bitDepth}b.obu"), + payload); + } - Assert.Equal((ushort)0, targetState.EndOfBlock); - Assert.Equal(Av1TransformType.DctDct, targetState.TransformType); Assert.NotEqual(0, pilotWriter.GetTileData(0).Length); Assert.NotEqual(0, tileWriter.GetTileData(0).Length); } @@ -2571,8 +2813,11 @@ public class Av1IntraSuperblockEncoderTests private delegate void FilterPrediction( Av1FilterIntraMode mode, Span destination, + int destinationStride, ReadOnlySpan above, ReadOnlySpan left, + int width, + int height, int bitDepth, Span scratch) where TSample : unmanaged;