From f0fe10bdd7d95f57062fc73291edf576270a08ec Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Tue, 25 Aug 2026 01:37:34 +1000 Subject: [PATCH] Integrate AV1 loop filtering --- HEIF_IMPLEMENTATION_PLAN.md | 2 +- .../Heif/Av1/Pipeline/Av1FrameDecoder.cs | 66 +--- .../LoopFilter/Av1LoopFilterContext.cs | 81 ++++- .../LoopFilter/Av1LoopFilterDecoder.cs | 321 +++++++++++++++--- .../Quantification/Av1InverseQuantizer.cs | 5 +- .../Formats/Heif/Av1/Tiling/Av1FrameInfo.cs | 19 +- .../Heif/Av1/Tiling/Av1SuperblockInfo.cs | 4 +- .../Formats/Heif/Av1/Tiling/Av1TileReader.cs | 39 ++- .../Heif/Av1/Transform/Av1BlockDecoder.cs | 39 ++- 9 files changed, 429 insertions(+), 147 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 59e32f71a9..43ea1e8139 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -119,7 +119,7 @@ This assessment is based on the current source after the upstream ImageSharp mer ### AV1 decoder - The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one independently decodable frame, and it disposes the reconstruction planes after pixel conversion. It deliberately does not retain animation/video reference frames or implement `show_existing_frame` playback state. -- The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. The exact scalar low/high-bit-depth AV1 4-, 6-, 8-, and 14-tap deblocking kernels are now present, but filter-level derivation, transform-edge traversal, and pipeline wiring remain required before loop filtering can be enabled. +- The reconstruction pipeline now records plane-relative transform geometry, preserves tile-local delta-Q and delta-LF predictors, derives segmentation and reference-adjusted filter levels, and runs the exact scalar low/high-bit-depth AV1 4-, 6-, 8-, and 14-tap deblocking kernels in normative vertical-then-horizontal order. The available full-decode AV1 fixture has no internal edge, so independent low/high-bit-depth AVIF vectors with filtered transform boundaries remain required before deblocking has external pixel-level verification. CDEF, super-resolution, loop restoration, and padding remain disabled even though they are normative when signaled, not optional quality improvements. - Palette mode now reads the normative luma/chroma mode and size CDFs, neighbor color caches, high-bit-depth color syntax, diagonal color-index maps, clipped-edge padding, and direct sample prediction through the existing reconstruction pipeline. The scalar implementation matches the pinned libaom source, but an independently encoded palette AVIF fixture is still required before this path is independently verified. - Loop restoration and other normative independently decodable still-image syntax paths still contain `NotImplementedException` or equivalent unsupported branches. Tile-local palette CDF adaptation is now present; the remaining still-image frame-context behavior requires a separate source audit without introducing sequence playback state. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Independently encoded high-bit-depth AVIF conformance files are still required; chroma-from-luma storage and the other normative reconstruction stages listed below remain incomplete. diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs index 95947f29f2..a9e66f84af 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs @@ -2,6 +2,7 @@ // Licensed under the Six Labors Split License. using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; @@ -43,6 +44,11 @@ internal class Av1FrameDecoder : IAv1FrameDecoder /// private readonly Av1DeQuantizationContext deQuants; + /// + /// The transform-size map populated during reconstruction and consumed by deblocking. + /// + private readonly Av1LoopFilterContext loopFilterContext; + /// /// The block reconstruction stage that applies prediction and inverse transforms. /// @@ -63,11 +69,12 @@ internal class Av1FrameDecoder : IAv1FrameDecoder this.frameBuffer = frameBuffer; this.inverseQuantizer = new(sequenceHeader, frameHeader); this.deQuants = new(sequenceHeader, frameHeader); - this.blockDecoder = new(this.sequenceHeader, this.frameHeader, this.frameBuffer); + this.loopFilterContext = new(sequenceHeader); + this.blockDecoder = new(this.sequenceHeader, this.frameHeader, this.frameBuffer, this.loopFilterContext); } /// - /// Reconstructs every coded tile of the frame; in-loop post-processing stages remain disabled until implemented. + /// Reconstructs every coded tile and applies the implemented in-loop frame stages in normative order. /// public void DecodeFrame() { @@ -77,16 +84,17 @@ internal class Av1FrameDecoder : IAv1FrameDecoder this.DecodeFrameTiles(column); } - bool doLoopFilterFlag = false; bool doLoopRestoration = false; bool doUpscale = false; - // These flags remain false until the corresponding normative stages have complete scalar implementations - // and independent still-image vectors; silently running partial filters would corrupt reconstructed pixels. - if (doLoopFilterFlag) - { - this.DecodeLoopFilterForFrame(); - } + Av1LoopFilterDecoder loopFilterDecoder = new( + this.sequenceHeader, + this.frameHeader, + this.frameInfo, + this.frameBuffer, + this.loopFilterContext); + + loopFilterDecoder.DecodeFrame(); if (doLoopRestoration) { @@ -186,44 +194,4 @@ internal class Av1FrameDecoder : IAv1FrameDecoder this.blockDecoder.DecodeBlock(modeInfo, globalPosition, subSize, superblockInfo, tileInfo); } } - - /// - /// Traverses frame superblocks in the order required by the not-yet-implemented deblocking stage. - /// - private void DecodeLoopFilterForFrame() - { - int superblockSizeLog2 = this.sequenceHeader.SuperblockSizeLog2; - int pictureWidthInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameWidth, this.sequenceHeader.SuperblockSizeLog2); - int pictureHeightInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameHeight, this.sequenceHeader.SuperblockSizeLog2); - - // Deblocking uses raster traversal so each block can consume already reconstructed top and left edges. - for (int superblockIndexY = 0; superblockIndexY < pictureHeightInSuperblocks; ++superblockIndexY) - { - for (int superblockIndexX = 0; superblockIndexX < pictureWidthInSuperblocks; ++superblockIndexX) - { - int superblockOriginX = superblockIndexX << superblockSizeLog2; - int superblockOriginY = superblockIndexY << superblockSizeLog2; - bool endOfRowFlag = superblockIndexX == pictureWidthInSuperblocks - 1; - - Point superblockPoint = new(superblockOriginX, superblockOriginY); - Av1SuperblockInfo superblockInfo = this.frameInfo.GetSuperblock(superblockPoint); - - // Superblock filtering remains disabled until its complete plane and edge-strength implementation is available. - /* - DecodeLoopFilterForSuperblock( - superblockInfo, - this.frameHeader, - this.sequenceHeader, - reconstructionFrameBuffer, - loopFilterContext, - superblockOriginY >> 2, - superblockOriginX >> 2, - Av1Plane.Y, - 3, - endOfRowFlag, - superblockInfo.SuperblockDeltaLoopFilter); - */ - } - } - } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterContext.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterContext.cs index fbc79ae5d4..046918ab6d 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterContext.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterContext.cs @@ -1,11 +1,90 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; /// -/// Defines the per-frame state boundary used while applying the AV1 deblocking loop filter. +/// Stores the transform-size map consumed by the AV1 deblocking loop filter. /// internal class Av1LoopFilterContext { + /// + /// Stores luma and shared-chroma transform sizes at plane-relative 4x4 granularity. + /// + private readonly Av1TransformSize[][] transformSizes = new Av1TransformSize[2][]; + + /// + /// Stores the row stride of each transform-size map. + /// + private readonly int[] transformSizeStrides = new int[2]; + + /// + /// Initializes a new instance of the class. + /// + /// The sequence header defining aligned frame and chroma dimensions. + public Av1LoopFilterContext(ObuSequenceHeader sequenceHeader) + { + int alignedModeInfoWidth = Av1Math.AlignPowerOf2(sequenceHeader.MaxFrameWidth, sequenceHeader.SuperblockSizeLog2) >> + Av1Constants.ModeInfoSizeLog2; + + int alignedModeInfoHeight = Av1Math.AlignPowerOf2(sequenceHeader.MaxFrameHeight, sequenceHeader.SuperblockSizeLog2) >> + Av1Constants.ModeInfoSizeLog2; + + this.transformSizeStrides[(int)Av1PlaneType.Y] = alignedModeInfoWidth; + this.transformSizes[(int)Av1PlaneType.Y] = new Av1TransformSize[alignedModeInfoWidth * alignedModeInfoHeight]; + + if (!sequenceHeader.ColorConfig.IsMonochrome) + { + int subX = sequenceHeader.ColorConfig.SubSamplingX ? 1 : 0; + int subY = sequenceHeader.ColorConfig.SubSamplingY ? 1 : 0; + int chromaWidth = Av1Math.DivideLog2Ceiling(alignedModeInfoWidth, subX); + int chromaHeight = Av1Math.DivideLog2Ceiling(alignedModeInfoHeight, subY); + + this.transformSizeStrides[(int)Av1PlaneType.Uv] = chromaWidth; + this.transformSizes[(int)Av1PlaneType.Uv] = new Av1TransformSize[chromaWidth * chromaHeight]; + } + else + { + this.transformSizes[(int)Av1PlaneType.Uv] = []; + } + } + + /// + /// Stores a transform size across every 4x4 position covered by one transform block. + /// + /// The luma or chroma plane. + /// The transform origin in plane-relative 4x4 units. + /// The transform size. + public void SetTransformSize(Av1Plane plane, Point position, Av1TransformSize transformSize) + { + int planeType = Math.Min((int)plane, (int)Av1PlaneType.Uv); + Span transformSizeMap = this.transformSizes[planeType]; + int stride = this.transformSizeStrides[planeType]; + int width = transformSize.Get4x4WideCount(); + int height = transformSize.Get4x4HighCount(); + + // Loop filtering addresses every covered 4x4 position, not only the transform origin. Replication keeps + // edge lookup independent of the transform traversal order used while reconstructing the coded block. + for (int y = 0; y < height; y++) + { + transformSizeMap.Slice(((position.Y + y) * stride) + position.X, width).Fill(transformSize); + } + } + + /// + /// Gets the transform size covering a plane-relative 4x4 position. + /// + /// The luma or chroma plane. + /// The position in plane-relative 4x4 units. + /// The transform size covering the position. + public Av1TransformSize GetTransformSize(Av1Plane plane, Point position) + { + int planeType = Math.Min((int)plane, (int)Av1PlaneType.Uv); + int stride = this.transformSizeStrides[planeType]; + return this.transformSizes[planeType][(position.Y * stride) + position.X]; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterDecoder.cs index 1a05902dec..fdc9f7ae7a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterDecoder.cs @@ -1,114 +1,329 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; /// -/// Provides frame traversal for the AV1 in-loop deblocking stage. +/// Applies the AV1 in-loop deblocking stage to a reconstructed still-image frame. /// internal class Av1LoopFilterDecoder { /// - /// The sequence-level superblock configuration. + /// The sequence-level superblock and color configuration. /// private readonly ObuSequenceHeader sequenceHeader; /// - /// The frame dimensions and loop-filter parameters. + /// The frame dimensions, segmentation state, and loop-filter parameters. /// private readonly ObuFrameHeader frameHeader; /// - /// The decoded block-mode information addressed by superblock origin. + /// The decoded mode and superblock delta information. /// private readonly Av1FrameInfo frameInfo; /// - /// The reconstructed plane samples supplied to in-loop filtering. + /// The reconstructed plane samples modified by deblocking. /// private readonly Av1FrameBuffer frameBuffer; /// - /// The per-frame filter context retained across superblocks. + /// The per-plane transform-size map populated during reconstruction. /// private readonly Av1LoopFilterContext loopFilterContext; /// /// Initializes a new instance of the class. /// - /// The sequence header that supplies the superblock size. - /// The frame header that supplies dimensions and filter parameters. - /// The decoded block-mode information for the frame. + /// The sequence header defining superblock size and color layout. + /// The frame header defining dimensions and filter parameters. + /// The decoded block-mode and superblock delta information. /// The reconstructed frame samples to filter. - public Av1LoopFilterDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Av1FrameInfo frameInfo, Av1FrameBuffer frameBuffer) + /// The transform-size map populated during reconstruction. + public Av1LoopFilterDecoder( + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1FrameInfo frameInfo, + Av1FrameBuffer frameBuffer, + Av1LoopFilterContext loopFilterContext) { this.sequenceHeader = sequenceHeader; this.frameHeader = frameHeader; this.frameInfo = frameInfo; this.frameBuffer = frameBuffer; - this.loopFilterContext = new(); + this.loopFilterContext = loopFilterContext; } /// - /// Traverses each superblock in raster order when loop filtering is enabled for the decode pass. + /// Filters every enabled plane, processing all vertical boundaries before horizontal boundaries. /// - /// Whether the frame should run the deblocking stage. - /// The superblock filtering operation has not been implemented. - public void DecodeFrame(bool doLoopFilterFlag) + public void DecodeFrame() { - Guard.NotNull(this.sequenceHeader); - Guard.NotNull(this.frameHeader); - Guard.NotNull(this.frameInfo); + ObuLoopFilterParameters filterParameters = this.frameHeader.LoopFilterParameters; + if (filterParameters.FilterLevel[0] == 0 && filterParameters.FilterLevel[1] == 0) + { + return; + } + + ObuColorConfig colorConfig = this.sequenceHeader.ColorConfig; + for (int planeIndex = 0; planeIndex < colorConfig.PlaneCount; planeIndex++) + { + Av1Plane plane = (Av1Plane)planeIndex; + int planeFilterLevel = plane switch + { + Av1Plane.U => filterParameters.FilterLevelU, + Av1Plane.V => filterParameters.FilterLevelV, + _ => Math.Max(filterParameters.FilterLevel[0], filterParameters.FilterLevel[1]) + }; + + if (planeFilterLevel == 0) + { + continue; + } + + int subX = plane != Av1Plane.Y && colorConfig.SubSamplingX ? 1 : 0; + int subY = plane != Av1Plane.Y && colorConfig.SubSamplingY ? 1 : 0; + Span lowBitDepthSamples = default; + Span highBitDepthSamples = default; + int stride; + + if (this.frameBuffer.BytesPerSample == 2) + { + Span signedSamples = this.frameBuffer.DeriveBlockPointer16(plane, Point.Empty, subX, subY, out stride); + highBitDepthSamples = MemoryMarshal.Cast(signedSamples); + } + else + { + lowBitDepthSamples = this.frameBuffer.DeriveBlockPointer(plane, Point.Empty, subX, subY, out stride); + } + + this.FilterPlane(plane, subX, subY, stride, lowBitDepthSamples, highBitDepthSamples); + } + } + + /// + /// Filters one plane in the AV1 vertical-then-horizontal boundary order. + /// + /// The color plane to filter. + /// The horizontal chroma subsampling shift. + /// The vertical chroma subsampling shift. + /// The plane stride in logical samples. + /// The low-bit-depth plane storage, when active. + /// The high-bit-depth plane storage, when active. + private void FilterPlane( + Av1Plane plane, + int subX, + int subY, + int stride, + Span lowBitDepthSamples, + Span highBitDepthSamples) + { + int rowStep = 1 << subY; + int columnStep = 1 << subX; + + // The AV1 result is independent of ordering within a pass, but vertical filtering must finish before any + // horizontal filtering begins because the two directions modify intersecting sample neighborhoods. + for (int pass = 0; pass < 2; pass++) + { + for (int row = 0; row < this.frameHeader.ModeInfoRowCount; row += rowStep) + { + for (int column = 0; column < this.frameHeader.ModeInfoColumnCount; column += columnStep) + { + this.FilterEdge( + plane, + pass, + row, + column, + subX, + subY, + stride, + lowBitDepthSamples, + highBitDepthSamples); + } + } + } + } + + /// + /// Derives and applies the filter for one 4x4 luma-grid boundary. + /// + /// The color plane to filter. + /// Zero for a vertical boundary; one for a horizontal boundary. + /// The boundary row in luma 4x4 units. + /// The boundary column in luma 4x4 units. + /// The horizontal chroma subsampling shift. + /// The vertical chroma subsampling shift. + /// The plane stride in logical samples. + /// The low-bit-depth plane storage, when active. + /// The high-bit-depth plane storage, when active. + private void FilterEdge( + Av1Plane plane, + int pass, + int row, + int column, + int subX, + int subY, + int stride, + Span lowBitDepthSamples, + Span highBitDepthSamples) + { + int x = column << Av1Constants.ModeInfoSizeLog2; + int y = row << Av1Constants.ModeInfoSizeLog2; + bool verticalBoundary = pass == 0; + if (x >= this.frameHeader.FrameSize.FrameWidth || + y >= this.frameHeader.FrameSize.FrameHeight || + (verticalBoundary ? x == 0 : y == 0)) + { + return; + } + + int adjustedRow = row | subY; + int adjustedColumn = column | subX; + int previousRow = adjustedRow - (verticalBoundary ? 0 : 1 << subY); + int previousColumn = adjustedColumn - (verticalBoundary ? 1 << subX : 0); + Point modeInfoPosition = new(adjustedColumn, adjustedRow); + Point previousModeInfoPosition = new(previousColumn, previousRow); + Point planeTransformPosition = new(adjustedColumn >> subX, adjustedRow >> subY); + Point previousPlaneTransformPosition = new(previousColumn >> subX, previousRow >> subY); + Av1BlockModeInfo modeInfo = this.frameInfo.GetModeInfoAt(modeInfoPosition); + Av1TransformSize transformSize = this.loopFilterContext.GetTransformSize(plane, planeTransformPosition); + Av1TransformSize previousTransformSize = this.loopFilterContext.GetTransformSize(plane, previousPlaneTransformPosition); + Av1BlockSize planeBlockSize = modeInfo.BlockSize.GetSubsampled(subX, subY); + int planeX = x >> subX; + int planeY = y >> subY; + bool isBlockEdge = verticalBoundary + ? planeX % planeBlockSize.GetWidth() == 0 + : planeY % planeBlockSize.GetHeight() == 0; + + bool isTransformEdge = verticalBoundary + ? planeX % transformSize.GetWidth() == 0 + : planeY % transformSize.GetHeight() == 0; - if (!doLoopFilterFlag) + // The still-image decoder accepts key and intra-only frames, so every decoded block satisfies the AV1 + // isIntra condition. Retaining the other predicates mirrors the normative edge decision without inter state. + bool applyFilter = isTransformEdge && (isBlockEdge || !modeInfo.Skip || this.frameHeader.IsIntra); + if (!applyFilter) { return; } - int superblockSizeLog2 = this.sequenceHeader.SuperblockSizeLog2; - int frameWidthInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameWidth, this.sequenceHeader.SuperblockSizeLog2); - int frameHeightInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameHeight, this.sequenceHeader.SuperblockSizeLog2); + int currentLevel = this.GetFilterLevel(modeInfo, modeInfoPosition, plane, pass); + int filterLevel = currentLevel != 0 + ? currentLevel + : this.GetFilterLevel(this.frameInfo.GetModeInfoAt(previousModeInfoPosition), previousModeInfoPosition, plane, pass); - // Filtering proceeds in raster order because vertical and horizontal edges depend on already reconstructed - // neighboring blocks, while the final superblock in each row requires distinct delayed-edge handling. - for (int superblockIndexY = 0; superblockIndexY < frameHeightInSuperblocks; ++superblockIndexY) + if (filterLevel == 0) { - for (int superblockIndexX = 0; superblockIndexX < frameWidthInSuperblocks; ++superblockIndexX) + return; + } + + int baseFilterSize = verticalBoundary + ? Math.Min(transformSize.GetWidth(), previousTransformSize.GetWidth()) + : Math.Min(transformSize.GetHeight(), previousTransformSize.GetHeight()); + + int maximumFilterSize = plane == Av1Plane.Y ? 16 : 8; + int filterSize = Math.Min(maximumFilterSize, baseFilterSize); + int kernelLength = plane == Av1Plane.Y + ? filterSize switch { - int superblockOriginX = superblockIndexX << superblockSizeLog2; - int superblockOriginY = superblockIndexY << superblockSizeLog2; - bool endOfRowFlag = superblockIndexX == frameWidthInSuperblocks - 1; - - Point superblockPoint = new(superblockOriginX, superblockOriginY); - Av1SuperblockInfo superblockInfo = this.frameInfo.GetSuperblock(superblockPoint); - - // Mode-info coordinates are measured in 4x4 units, whereas the frame and superblock origins are pixels. - Point superblockOriginInModeInfo = new(superblockOriginX >> 2, superblockOriginY >> 2); - - this.DecodeForSuperblock( - superblockInfo, - superblockOriginInModeInfo, - Av1Plane.Y, - 3, - endOfRowFlag, - superblockInfo.SuperblockDeltaLoopFilter); + 4 => 4, + 8 => 8, + _ => 14 } + : filterSize == 4 ? 4 : 6; + + int sharpness = this.frameHeader.LoopFilterParameters.SharpnessLevel; + int shift = sharpness > 4 ? 2 : sharpness > 0 ? 1 : 0; + int limit = sharpness > 0 + ? Av1Math.Clip3(1, 9 - sharpness, filterLevel >> shift) + : Math.Max(1, filterLevel >> shift); + + int boundaryLimit = (2 * (filterLevel + 2)) + limit; + int highEdgeVarianceThreshold = filterLevel >> 4; + int q0Offset = stride + (planeY * stride) + planeX; + int pixelStep = verticalBoundary ? 1 : stride; + int lineStep = verticalBoundary ? stride : 1; + + if (this.frameBuffer.BytesPerSample == 2) + { + Av1LoopFilterKernels.FilterHighBitDepthEdge( + highBitDepthSamples, + q0Offset, + pixelStep, + lineStep, + kernelLength, + limit, + boundaryLimit, + highEdgeVarianceThreshold, + this.frameBuffer.BitDepth.GetBitCount()); + } + else + { + Av1LoopFilterKernels.FilterLowBitDepthEdge( + lowBitDepthSamples, + q0Offset, + pixelStep, + lineStep, + kernelLength, + limit, + boundaryLimit, + highEdgeVarianceThreshold); } } /// - /// Represents the not-yet-implemented deblocking operation for one superblock and plane range. + /// Derives the adaptive filter level for one block, plane, and boundary direction. /// - /// The decoded modes and delta values for the superblock. - /// The superblock origin in 4x4 mode-info units. - /// The first color plane to filter. - /// The exclusive color-plane index at which filtering stops. - /// Whether the superblock is the final block in its raster row. - /// The per-superblock loop-filter strength adjustments. - /// Always thrown because superblock deblocking has not been implemented. - private void DecodeForSuperblock(Av1SuperblockInfo superblockInfo, Point modeInfoLocation, Av1Plane startPlane, int endPlane, bool endOfRowFlag, Span superblockDeltaLoopFilter) - => throw new NotImplementedException(); + /// The decoded mode and segment information. + /// The frame-relative position in luma 4x4 units. + /// The color plane. + /// Zero for a vertical boundary; one for a horizontal boundary. + /// The filter level in the AV1 zero-to-63 domain. + private int GetFilterLevel(Av1BlockModeInfo modeInfo, Point modeInfoPosition, Av1Plane plane, int pass) + { + int filterIndex = plane == Av1Plane.Y ? pass : (int)plane + 1; + ObuLoopFilterParameters parameters = this.frameHeader.LoopFilterParameters; + int baseLevel = filterIndex switch + { + 0 => parameters.FilterLevel[0], + 1 => parameters.FilterLevel[1], + 2 => parameters.FilterLevelU, + _ => parameters.FilterLevelV + }; + + int superblockShift = this.sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2; + Point superblockPosition = new(modeInfoPosition.X >> superblockShift, modeInfoPosition.Y >> superblockShift); + Span deltaLoopFilter = this.frameInfo.GetSuperblock(superblockPosition).SuperblockDeltaLoopFilter; + int delta = this.frameHeader.DeltaLoopFilterParameters.IsMulti ? deltaLoopFilter[filterIndex] : deltaLoopFilter[0]; + int level = Av1Math.Clip3(0, Av1Constants.MaxLoopFilter, baseLevel + delta); + ObuSegmentationLevelFeature feature = (ObuSegmentationLevelFeature)((int)ObuSegmentationLevelFeature.AlternativeLoopFilterYVertical + filterIndex); + ObuSegmentationParameters segmentation = this.frameHeader.SegmentationParameters; + + if (segmentation.IsFeatureActive(modeInfo.SegmentId, feature)) + { + level = Av1Math.Clip3( + 0, + Av1Constants.MaxLoopFilter, + level + segmentation.FeatureData[modeInfo.SegmentId, (int)feature]); + } + + if (parameters.ReferenceDeltaModeEnabled) + { + // Every supported AVIF still-picture block uses INTRA_FRAME, whose reference delta is index zero and + // whose prediction mode does not consume either inter mode delta. + int referenceScale = 1 << (level >> 5); + level = Av1Math.Clip3( + 0, + Av1Constants.MaxLoopFilter, + level + (parameters.ReferenceDeltas[0] * referenceScale)); + } + + return level; + } } diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs index b1983438fa..a318f53896 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Quantification/Av1InverseQuantizer.cs @@ -53,7 +53,10 @@ internal class Av1InverseQuantizer { for (int i = 0; i < Av1Constants.MaxSegmentCount; i++) { - int currentQIndex = Av1QuantizationLookup.GetQIndex(this.frameHeader.SegmentationParameters, i, superblockInfo.SuperblockDeltaQ); + int currentQIndex = Av1QuantizationLookup.GetQIndex( + this.frameHeader.SegmentationParameters, + i, + superblockInfo.SuperblockQuantizerIndex); for (Av1Plane plane = 0; (int)plane < Av1Constants.MaxPlanes; plane++) { diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs index f8ff7b6919..29d7777ccf 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1FrameInfo.cs @@ -81,9 +81,9 @@ internal partial class Av1FrameInfo private readonly Av1TransformInfo[] transformInfosUv; /// - /// Stores the quantizer-index delta for each frame superblock. + /// Stores the active base quantizer index for each frame superblock. /// - private readonly int[] deltaQ; + private readonly int[] quantizerIndices; /// /// The base-2 number of constrained directional enhancement filter entries allocated per superblock. @@ -154,7 +154,7 @@ internal partial class Av1FrameInfo this.coefficientsY = new int[superblockCount * lumaCoefficientCountPerSuperblock]; this.coefficientsU = new int[superblockCount * chromaCoefficientCountPerSuperblock]; this.coefficientsV = new int[superblockCount * chromaCoefficientCountPerSuperblock]; - this.deltaQ = new int[superblockCount]; + this.quantizerIndices = new int[superblockCount]; // A 128x128 superblock contains four 64x64 CDEF filter blocks; a 64x64 superblock contains one. this.cdefStrengthFactorLog2 = (superblockSizeLog2 - 6) << 2; @@ -305,13 +305,13 @@ internal partial class Av1FrameInfo } /// - /// Gets a reference to the quantizer-index delta for a specified superblock. + /// Gets a reference to the active base quantizer index for a specified superblock. /// /// The position in the frame superblock grid. - /// A reference to the superblock quantizer-index delta. - public ref int GetDeltaQuantizationIndex(Point index) + /// A reference to the superblock base quantizer index. + public ref int GetQuantizerIndex(Point index) { - Span span = this.deltaQ; + Span span = this.quantizerIndices; int i = (index.Y * this.superblockColumnCount) + index.X; return ref span[i]; } @@ -353,11 +353,6 @@ internal partial class Av1FrameInfo return span.Slice(i, 1 << this.deltaLoopFactorLog2); } - /// - /// Resets all frame loop-filter delta values to zero. - /// - public void ClearDeltaLoopFilter() => Array.Fill(this.deltaLoopFilter, 0); - /// /// Stores decoded mode information and maps every 4x4 position covered by its block. /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockInfo.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockInfo.cs index 610b0698c1..9d79ba007a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockInfo.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1SuperblockInfo.cs @@ -35,9 +35,9 @@ internal class Av1SuperblockInfo public Point ModeInfoPosition => this.Position * this.frameInfo.SuperblockModeInfoSize; /// - /// Gets a reference to the superblock quantizer-index delta. + /// Gets a reference to the active base quantizer index for this superblock. /// - public ref int SuperblockDeltaQ => ref this.frameInfo.GetDeltaQuantizationIndex(this.Position); + public ref int SuperblockQuantizerIndex => ref this.frameInfo.GetQuantizerIndex(this.Position); /// /// Gets the mode information that covers the superblock origin. diff --git a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs index fc22e0f2c9..706b4626e7 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs @@ -74,6 +74,11 @@ internal class Av1TileReader : IAv1TileReader /// private int currentQuantizerIndex; + /// + /// Stores the loop-filter delta values carried between superblocks in the current tile. + /// + private readonly int[] currentDeltaLoopFilter = new int[Av1Constants.FrameLoopFilterCount]; + /// /// Stores the segment identifier covering each 4x4 frame position. /// @@ -189,6 +194,7 @@ internal class Av1TileReader : IAv1TileReader int modeInfoRowStart = this.FrameHeader.TilesInfo.TileRowStartModeInfo[tileRowIndex]; int modeInfoRowEnd = this.FrameHeader.TilesInfo.TileRowStartModeInfo[tileRowIndex + 1]; this.aboveNeighborContext.Clear(this.SequenceHeader, modeInfoColumnStart, modeInfoColumnEnd); + this.currentQuantizerIndex = this.FrameHeader.QuantizationParameters.BaseQIndex; this.ClearLoopFilterDelta(); int planesCount = this.SequenceHeader.ColorConfig.PlaneCount; @@ -236,10 +242,10 @@ internal class Av1TileReader : IAv1TileReader } /// - /// Resets all frame loop-filter delta state before parsing a tile. + /// Resets the loop-filter delta predictors before parsing a tile. /// private void ClearLoopFilterDelta() - => this.FrameInfo.ClearDeltaLoopFilter(); + => this.currentDeltaLoopFilter.AsSpan().Clear(); /// /// Reads loop-restoration unit syntax that begins at a superblock location. @@ -1982,14 +1988,13 @@ internal class Av1TileReader : IAv1TileReader /// The current coding block and superblock delta storage. private void ReadDeltaLoopFilter(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) { - Av1BlockSize superBlockSize = this.SequenceHeader.Use128x128Superblock ? Av1BlockSize.Block128x128 : Av1BlockSize.Block64x64; - if (this.FrameHeader.DeltaLoopFilterParameters.IsPresent || - (partitionInfo.ModeInfo.BlockSize == superBlockSize && partitionInfo.ModeInfo.Skip)) + if (!this.FrameHeader.DeltaLoopFilterParameters.IsPresent || partitionInfo.ModeInfo.PositionInSuperblock != Point.Empty) { return; } - if (this.FrameHeader.DeltaLoopFilterParameters.IsPresent) + Av1BlockSize superBlockSize = this.SequenceHeader.Use128x128Superblock ? Av1BlockSize.Block128x128 : Av1BlockSize.Block64x64; + if (partitionInfo.ModeInfo.BlockSize != superBlockSize || !partitionInfo.ModeInfo.Skip) { int frameLoopFilterCount = 1; if (this.FrameHeader.DeltaLoopFilterParameters.IsMulti) @@ -1997,14 +2002,20 @@ internal class Av1TileReader : IAv1TileReader frameLoopFilterCount = this.SequenceHeader.ColorConfig.PlaneCount > 1 ? Av1Constants.FrameLoopFilterCount : Av1Constants.FrameLoopFilterCount - 2; } - Span currentDeltaLoopFilter = partitionInfo.SuperblockInfo.SuperblockDeltaLoopFilter; for (int i = 0; i < frameLoopFilterCount; i++) { int reducedDeltaLoopFilterLevel = reader.ReadDeltaLoopFilter(); int deltaLoopFilterResolution = this.FrameHeader.DeltaLoopFilterParameters.Resolution; - currentDeltaLoopFilter[i] = Av1Math.Clip3(-Av1Constants.MaxLoopFilter, Av1Constants.MaxLoopFilter, currentDeltaLoopFilter[i] + (reducedDeltaLoopFilterLevel << deltaLoopFilterResolution)); + this.currentDeltaLoopFilter[i] = Av1Math.Clip3( + -Av1Constants.MaxLoopFilter, + Av1Constants.MaxLoopFilter, + this.currentDeltaLoopFilter[i] + (reducedDeltaLoopFilterLevel * deltaLoopFilterResolution)); } } + + // Delta-LF values are predicted across superblocks within a tile, but every block in one superblock observes + // the same resulting values. Snapshot the predictors so later filtering does not depend on parse order. + this.currentDeltaLoopFilter.AsSpan().CopyTo(partitionInfo.SuperblockInfo.SuperblockDeltaLoopFilter); } /// @@ -2037,9 +2048,7 @@ internal class Av1TileReader : IAv1TileReader /// Corresponds to read_delta_qindex in SVT-AV1. private void ReadDeltaQuantizerIndex(ref Av1SymbolDecoder reader, Av1PartitionInfo partitionInfo) { - Av1BlockSize superBlockSize = this.SequenceHeader.Use128x128Superblock ? Av1BlockSize.Block128x128 : Av1BlockSize.Block64x64; - if (!this.FrameHeader.DeltaQParameters.IsPresent || - (partitionInfo.ModeInfo.BlockSize == superBlockSize && partitionInfo.ModeInfo.Skip)) + if (!this.FrameHeader.DeltaQParameters.IsPresent || partitionInfo.ModeInfo.PositionInSuperblock != Point.Empty) { return; } @@ -2048,9 +2057,13 @@ internal class Av1TileReader : IAv1TileReader { int reducedDeltaQuantizerIndex = reader.ReadDeltaQuantizerIndex(); int deltaQuantizerResolution = this.FrameHeader.DeltaQParameters.Resolution; - this.currentQuantizerIndex = Av1Math.Clip3(1, 255, this.currentQuantizerIndex + (reducedDeltaQuantizerIndex << deltaQuantizerResolution)); - partitionInfo.SuperblockInfo.SuperblockDeltaQ = this.currentQuantizerIndex; + this.currentQuantizerIndex = Av1Math.Clip3( + 1, + Av1Constants.MaxQ, + this.currentQuantizerIndex + (reducedDeltaQuantizerIndex * deltaQuantizerResolution)); } + + partitionInfo.SuperblockInfo.SuperblockQuantizerIndex = this.currentQuantizerIndex; } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index 990b98ecef..cfab7229b0 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -3,6 +3,7 @@ using System.Runtime.CompilerServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; +using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantification; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; @@ -30,6 +31,11 @@ internal class Av1BlockDecoder /// private readonly Av1FrameBuffer frameBuffer; + /// + /// The per-plane transform-size map consumed after reconstruction by the deblocking stage. + /// + private readonly Av1LoopFilterContext loopFilterContext; + /// /// Indicates whether transform traversal must also populate loop-filter parameters. /// @@ -51,11 +57,17 @@ internal class Av1BlockDecoder /// The decoded sequence header. /// The decoded frame header. /// The frame buffer receiving reconstructed samples. - public Av1BlockDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Av1FrameBuffer frameBuffer) + /// The transform-size map populated while reconstructing blocks. + public Av1BlockDecoder( + ObuSequenceHeader sequenceHeader, + ObuFrameHeader frameHeader, + Av1FrameBuffer frameBuffer, + Av1LoopFilterContext loopFilterContext) { this.sequenceHeader = sequenceHeader; this.frameHeader = frameHeader; this.frameBuffer = frameBuffer; + this.loopFilterContext = loopFilterContext; int ySize = (1 << this.sequenceHeader.SuperblockSizeLog2) * (1 << this.sequenceHeader.SuperblockSizeLog2); // One scratch plane is reused for every transform unit. Its maximum size must cover a complete superblock @@ -65,7 +77,9 @@ internal class Av1BlockDecoder (this.sequenceHeader.ColorConfig.SubSamplingY ? ySize >> 2 : ySize); this.CurrentInverseQuantizationCoefficients = new int[inverseQuantizationSize]; - this.isLoopFilterEnabled = false; + this.isLoopFilterEnabled = frameHeader.LoopFilterParameters.FilterLevel[0] != 0 || + frameHeader.LoopFilterParameters.FilterLevel[1] != 0; + this.currentCoefficientIndex = new int[3]; this.chromaFromLumaContext = new(sequenceHeader.ColorConfig); } @@ -140,7 +154,6 @@ internal class Av1BlockDecoder : modeInfo.TransformUnitsCount[(int)Av1Plane.U]; bool highBitDepth = this.frameBuffer.BytesPerSample == 2; - int loopFilterStride = this.frameHeader.ModeInfoStride; Av1PredictionDecoder predictionDecoder = new(this.sequenceHeader, this.frameHeader); Av1InverseQuantizer inverseQuantizer = new(this.sequenceHeader, this.frameHeader); @@ -220,20 +233,16 @@ internal class Av1BlockDecoder if (this.isLoopFilterEnabled) { - /* + // U and V share transform geometry. Store the chroma map once so the later plane passes consume + // identical sizes without retaining duplicate state. if (plane != 2) { - // SVT: svt_aom_fill_4x4_lf_param - Fill4x4LoopFilterParameters( - this.loopFilterContext, - (modeInfoPosition.X & (~subX)) + (transformInfo.OffsetX << subX), - (modeInfoPosition.Y & (~subY)) + (transformInfo.OffsetY << subY), - loopFilterStride, - transformSize, - subX, - subY, - plane); - }*/ + Point transformPosition = new( + (modeInfoPosition.X >> subX) + transformInfo[0].OffsetX, + (modeInfoPosition.Y >> subY) + transformInfo[0].OffsetY); + + this.loopFilterContext.SetTransformSize((Av1Plane)plane, transformPosition, transformSize); + } } // if (!inter_block)