diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 0e50c724d..7f898dc5a 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -29,7 +29,7 @@ Checkboxes may be marked complete only when the implementation and the verificat ## Delivery dashboard -Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `63c4d302a`, including the completed AV1 transform, OBU-framing, intra-block-copy, and 12-profile reconstruction checkpoints. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. +Last reconciled with the source tree on 2026-08-27 against the worktree based on commit `32d5e3b51`, including the completed AV1 transform, OBU-framing, intra-block-copy, 12-profile reconstruction, and layered-item property checkpoints. This dashboard is the authoritative delivery order. The detailed phase checklists below provide subsystem evidence; they do not override the current-stage marker or permit work to skip ahead. Status meanings: @@ -61,7 +61,8 @@ Immediate checkpoint: **complete layered AV1 image-item decoding through the exi - [ ] **Current:** implement layered AV1 image-item properties and stateful dependency reconstruction, then verify default final-layer output against the two pinned libavif progressive fixtures. - [x] Parse and associate `a1op`, `lsel`, and `a1lx` through the bounded image-item property model, including normative essential flags, duplicate handling, exact property lengths, and the four-layer limit. - [x] Validate `a1lx` layer boundaries against the logical item size and restrict concrete `lsel` decoding to the cumulative payload through the selected spatial layer without copying item bytes. - - [ ] Apply the selected `a1op` operating-point mask while consuming extended OBUs and validate the selected index against the parsed sequence header. + - [x] Apply the selected `a1op` operating-point mask while consuming extended OBUs and validate the selected index against the parsed sequence header. + - [x] Store the eight fixed reference-validity, order-hint, and map-index tables inline on the frame header, retaining complete multi-bit order hints without per-header array allocations. - [ ] Preserve reconstruction, reference-frame, primary-CDF, segmentation, loop-filter, and motion state across every dependent layer in one image-item decoder session. - [ ] Implement the complete inter-frame entropy, mode, motion-vector, compound-prediction, inter-prediction, and warped/global-motion paths permitted by the image profile. - [ ] Return the explicitly selected spatial layer or the final displayed layer, keeping reference reconstruction separate from display-only film grain. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index aeb74c07c..b30b5de9c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -37,9 +37,19 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable /// /// The configuration used for image and scratch-memory allocation. public Av1Decoder(Configuration configuration) + : this(configuration, 0) + { + } + + /// + /// Initializes a new instance of the class for one selected AV1 operating point. + /// + /// The configuration used for image and scratch-memory allocation. + /// The zero-based sequence-header operating-point index to decode. + public Av1Decoder(Configuration configuration, byte operatingPointIndex) { this.configuration = configuration; - this.obuReader = new(); + this.obuReader = new(operatingPointIndex); } /// diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs index 7fb868d42..73084c3df 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs @@ -10,6 +10,21 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// internal class ObuFrameHeader { + /// + /// Stores the validity state of the eight reference-frame slots without a per-header array allocation. + /// + private InlineArray8 referenceValid; + + /// + /// Stores the multi-bit order hint associated with each of the eight reference-frame slots. + /// + private InlineArray8 referenceOrderHint; + + /// + /// Stores the reference-map index selected for each of the eight inter references. + /// + private InlineArray8 referenceFrameIndex; + /// /// Gets or sets a value indicating whether motion vectors use integer-sample precision. /// @@ -156,14 +171,16 @@ internal class ObuFrameHeader internal ObuFrameType FrameType { get; set; } /// - /// Gets or sets the validity state of each reference-frame slot. + /// Gets the validity state of each reference-frame slot. /// - internal bool[] ReferenceValid { get; set; } = new bool[Av1Constants.ReferenceFrameCount]; + /// The mutable eight-entry reference-validity table. + public Span GetReferenceValidity() => this.referenceValid; /// - /// Gets or sets the stored order-hint state for each reference-frame slot. + /// Gets the multi-bit order hint associated with each reference-frame slot. /// - internal bool[] ReferenceOrderHint { get; set; } = new bool[Av1Constants.ReferenceFrameCount]; + /// The mutable eight-entry reference-order-hint table. + public Span GetReferenceOrderHints() => this.referenceOrderHint; /// /// Gets or sets a value indicating whether the decoded frame is immediately displayed. @@ -206,9 +223,10 @@ internal class ObuFrameHeader internal uint CurrentFrameId { get; set; } /// - /// Gets or sets the reference-map index selected for each inter reference. + /// Gets the reference-map index selected for each inter reference. /// - internal uint[] ReferenceFrameIndex { get; set; } = new uint[Av1Constants.ReferenceFrameCount]; + /// The mutable eight-entry reference-frame-index table. + public Span GetReferenceFrameIndices() => this.referenceFrameIndex; /// /// Gets or sets the frame order hint. diff --git a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs index 457a4df51..9036cfb1c 100644 --- a/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs +++ b/src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs @@ -11,11 +11,36 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; /// internal class ObuReader { + /// + /// The zero-based sequence-header operating-point index selected by the container. + /// + private readonly byte operatingPointIndex; + /// /// The tile reader created for the current coded frame. /// private IAv1TileReader? decoder; + /// + /// The temporal- and spatial-layer mask for the selected operating point. + /// + private uint currentOperatingPointIdc; + + /// + /// Initializes a new instance of the class using operating-point index zero. + /// + public ObuReader() + : this(0) + { + } + + /// + /// Initializes a new instance of the class for one selected AV1 operating point. + /// + /// The zero-based sequence-header operating-point index to decode. + public ObuReader(byte operatingPointIndex) + => this.operatingPointIndex = operatingPointIndex; + /// /// Gets or sets the most recently parsed sequence header. /// @@ -97,6 +122,22 @@ internal class ObuReader // A dedicated payload reader prevents malformed syntax from consuming the following OBU. The parent // advances once here, so ignored metadata, padding, and reserved OBUs are skipped without copying. Span obuPayload = reader.ReadBytes(payloadSize); + + // AV1 operating_point_idc uses bits 0-7 for temporal IDs and bits 8-11 for spatial IDs. libaom + // requires both selected bits for an extended OBU, while an all-zero mask and unextended OBUs apply + // universally. Sequence headers establish the mask and temporal delimiters define framing, so neither + // can be filtered even when their extension identifies a layer outside the selected operating point. + bool isOperatingPointIndependent = header.Type is ObuType.SequenceHeader or ObuType.TemporalDelimiter; + bool isInCurrentOperatingPoint = this.currentOperatingPointIdc == 0 + || !header.HasExtension + || (((this.currentOperatingPointIdc >> header.TemporalId) & 1U) != 0 + && ((this.currentOperatingPointIdc >> (header.SpatialId + 8)) & 1U) != 0); + + if (!isOperatingPointIndependent && !isInCurrentOperatingPoint) + { + continue; + } + Av1BitStreamReader payloadReader = new(obuPayload); int decodedPayloadSize; @@ -105,6 +146,14 @@ internal class ObuReader case ObuType.SequenceHeader: this.SequenceHeader = new(); ReadSequenceHeader(ref payloadReader, this.SequenceHeader); + if (this.operatingPointIndex >= this.SequenceHeader.OperatingPoint.Length) + { + throw new InvalidImageContentException( + $"The AV1 operating-point selector requests index {this.operatingPointIndex}, " + + $"but the sequence header declares {this.SequenceHeader.OperatingPoint.Length} operating points."); + } + + this.currentOperatingPointIdc = this.SequenceHeader.OperatingPoint[this.operatingPointIndex].Idc; decodedPayloadSize = Av1Math.DivideBy8Floor(payloadReader.BitPosition); break; case ObuType.FrameHeader: @@ -1064,10 +1113,8 @@ internal class ObuReader if (frameHeader.FrameType == ObuFrameType.KeyFrame && frameHeader.ShowFrame) { - frameHeader.ReferenceValid = new bool[Av1Constants.ReferenceFrameCount]; - frameHeader.ReferenceOrderHint = new bool[Av1Constants.ReferenceFrameCount]; - Array.Fill(frameHeader.ReferenceValid, false); - Array.Fill(frameHeader.ReferenceOrderHint, false); + frameHeader.GetReferenceValidity().Clear(); + frameHeader.GetReferenceOrderHints().Clear(); } frameHeader.DisableCdfUpdate = reader.ReadBoolean(); @@ -1122,20 +1169,22 @@ internal class ObuReader } int diffLength = sequenceHeader.DeltaFrameIdLength; + Span referenceFrameIndices = frameHeader.GetReferenceFrameIndices(); + Span referenceValidity = frameHeader.GetReferenceValidity(); for (int i = 0; i < Av1Constants.ReferenceFrameCount; i++) { if (frameHeader.CurrentFrameId > (1U << diffLength)) { - if ((frameHeader.ReferenceFrameIndex[i] > frameHeader.CurrentFrameId) || - frameHeader.ReferenceFrameIndex[i] > (frameHeader.CurrentFrameId - (1 - diffLength))) + if ((referenceFrameIndices[i] > frameHeader.CurrentFrameId) || + referenceFrameIndices[i] > (frameHeader.CurrentFrameId - (1 - diffLength))) { - frameHeader.ReferenceValid[i] = false; + referenceValidity[i] = false; } } - else if (frameHeader.ReferenceFrameIndex[i] > frameHeader.CurrentFrameId && - frameHeader.ReferenceFrameIndex[i] < ((1 << idLength) + (frameHeader.CurrentFrameId - (1 << diffLength)))) + else if (referenceFrameIndices[i] > frameHeader.CurrentFrameId && + referenceFrameIndices[i] < ((1 << idLength) + (frameHeader.CurrentFrameId - (1 << diffLength)))) { - frameHeader.ReferenceValid[i] = false; + referenceValidity[i] = false; } } } @@ -1211,12 +1260,14 @@ internal class ObuReader { if (frameHeader.ErrorResilientMode && sequenceHeader.OrderHintInfo != null) { + Span referenceOrderHints = frameHeader.GetReferenceOrderHints(); + Span referenceValidity = frameHeader.GetReferenceValidity(); for (int i = 0; i < Av1Constants.ReferenceFrameCount; i++) { int referenceOrderHint = (int)reader.ReadLiteral(sequenceHeader.OrderHintInfo.OrderHintBits); - if (referenceOrderHint != (frameHeader.ReferenceOrderHint[i] ? 1U : 0U)) + if (referenceOrderHint != referenceOrderHints[i]) { - frameHeader.ReferenceValid[i] = false; + referenceValidity[i] = false; } } } diff --git a/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs index 4441c3a8c..479ad8931 100644 --- a/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1HeifItemDecoder.cs @@ -52,7 +52,9 @@ internal class Av1HeifItemDecoder : IHeifItemDecoder, IHeifAlpha out HeifContentLightLevel? obuContentLightLevel, out HeifMasteringDisplayColorVolume? obuMasteringDisplayColorVolume); - using Av1Decoder decoder = new(options.Configuration); + byte operatingPointIndex = item.Av1OperatingPointSelector?.Index ?? 0; + + using Av1Decoder decoder = new(options.Configuration, operatingPointIndex); Image image = decoder.Decode(itemData, colorProfile, codecConfiguration); HeifMetadata metadata = image.Metadata.GetHeifMetadata(); metadata.CompressionMethod = this.CompressionMethod; @@ -82,7 +84,9 @@ internal class Av1HeifItemDecoder : IHeifItemDecoder, IHeifAlpha throw new InvalidImageContentException($"AV1 alpha image item {item.Id} is not monochrome."); } - using Av1Decoder decoder = new(options.Configuration); + byte operatingPointIndex = item.Av1OperatingPointSelector?.Index ?? 0; + + using Av1Decoder decoder = new(options.Configuration, operatingPointIndex); decoder.DecodeAlpha( itemData, item.CicpProfile, diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs index b2bb1004e..e937a1c78 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs @@ -12,6 +12,14 @@ public class ObuFrameHeaderTests private static readonly byte[] DefaultSequenceHeaderBitStream = [0x0a, 0x06, 0b001_1_1_000, 0b00_1000_01, 0b11_110101, 0b001_11101, 0b111_1_1_1_0_1, 0b1_0_0_1_1_1_10]; + // This complete temporal-delimiter and sequence-header prefix comes from the color item in libavif's + // draw_points_idat_progressive.avif. Its operating points select spatial layers 0+1 and layer 0 respectively. + private static ReadOnlySpan ProgressiveSequenceHeaderBitStream => + [ + 0x12, 0x00, + 0x0A, 0x0F, 0x20, 0x13, 0x01, 0x00, 0x80, 0x81, 0x4E, 0x0A, 0x36, 0xBE, 0x48, 0x08, 0x20, 0x34, 0x80 + ]; + // Bits Syntax element Value // 1 obu_forbidden_bit 0 // 4 obu_type 2 (OBU_TEMPORAL_DELIMITER) @@ -196,6 +204,88 @@ public class ObuFrameHeaderTests Assert.Equal(ObuPrettyPrint.PrettyPrintProperties(expected), ObuPrettyPrint.PrettyPrintProperties(obuReader.SequenceHeader)); } + /// + /// Verifies that an item cannot select an operating-point index absent from its sequence header. + /// + [Fact] + public void ReadOperatingPointRejectsIndexOutsideSequenceHeader() + { + byte[] bitStream = [.. ProgressiveSequenceHeaderBitStream]; + + Assert.Throws(() => ReadObuStream(bitStream, 2)); + } + + /// + /// Verifies that an extended OBU belongs to an operating point only when both of its layer identifiers are selected. + /// + /// The temporal-layer identifier carried by the test OBU. + /// The spatial-layer identifier carried by the test OBU. + /// Whether operating point one selects both identifiers. + [Theory] + [InlineData(0, 0, true)] + [InlineData(0, 1, false)] + [InlineData(1, 0, false)] + public void ReadOperatingPointRequiresBothLayerBits(byte temporalId, byte spatialId, bool isIncluded) + { + byte extension = (byte)((temporalId << 5) | (spatialId << 3)); + byte[] bitStream = [.. ProgressiveSequenceHeaderBitStream, 0x2E, extension, 0x01, 0x00]; + + // A selected metadata OBU reaches ignored-payload validation, where an all-zero payload is invalid. A filtered + // OBU has nevertheless had its complete header, size and payload boundary consumed before syntax is skipped. + if (isIncluded) + { + Assert.Throws(() => ReadObuStream(bitStream, 1)); + } + else + { + ReadObuStream(bitStream, 1); + } + } + + /// + /// Verifies that an all-zero operating-point mask includes every extended OBU. + /// + [Fact] + public void ReadZeroOperatingPointMaskIncludesExtendedObu() + { + byte[] bitStream = [.. DefaultSequenceHeaderBitStream, 0x2E, 0x08, 0x01, 0x00]; + + Assert.Throws(() => ReadObuStream(bitStream)); + } + + /// + /// Verifies that an OBU without an extension header applies to every operating point. + /// + [Fact] + public void ReadOperatingPointIncludesUnextendedObu() + { + byte[] bitStream = [.. ProgressiveSequenceHeaderBitStream, 0x2A, 0x01, 0x00]; + + Assert.Throws(() => ReadObuStream(bitStream, 1)); + } + + /// + /// Verifies that temporal delimiters remain part of stream framing even when their extension is outside the selected mask. + /// + [Fact] + public void ReadOperatingPointDoesNotFilterTemporalDelimiter() + { + byte[] bitStream = [.. ProgressiveSequenceHeaderBitStream, 0x16, 0x08, 0x01, 0x01]; + + Assert.Throws(() => ReadObuStream(bitStream, 1)); + } + + /// + /// Verifies that an excluded OBU cannot escape validation of its declared payload boundary. + /// + [Fact] + public void ReadFilteredOperatingPointObuStillValidatesBoundary() + { + byte[] bitStream = [.. ProgressiveSequenceHeaderBitStream, 0x2E, 0x08, 0x02, 0x80]; + + Assert.Throws(() => ReadObuStream(bitStream, 1)); + } + /// /// Verifies that the reduced sequence syntax cannot be used without declaring a still picture. /// @@ -438,10 +528,10 @@ public class ObuFrameHeaderTests /// Reads one complete OBU stream for malformed-input assertions that cannot capture a ref-struct reader. /// /// The complete encoded OBU stream. - private static void ReadObuStream(byte[] bitStream) + private static void ReadObuStream(byte[] bitStream, byte operatingPointIndex = 0) { Av1BitStreamReader reader = new(bitStream); - ObuReader obuReader = new(); + ObuReader obuReader = new(operatingPointIndex); IAv1TileReader tileDecoder = new Av1TileDecoderStub(); obuReader.ReadAll(ref reader, bitStream.Length, () => tileDecoder);