Parse HEIF sequence composition timing

HEIF_IMPLEMENTATION_PLAN.md

@@ -53,7 +53,9 @@ Checkboxes may be marked complete only when the implementation and the verificat
     - [x] Select one enabled `pict` master track without materializing unrelated tracks, then parse its `mvhd`/`tkhd`/`mdhd`/`hdlr`, self-contained `dref`, `av01` or `hvc1` sample entry, codec configuration, mandatory `ccst`, and bounded repetition edit.
     - [x] Resolve `stsc`, `stco`/`co64`, `stsz`/`stz2`, `stts`, and `stss` into one exact value-type descriptor array capped by `DecoderOptions.MaxFrames`; validate complete run/count syntax through one allocator-owned sequential scratch buffer.
       - Release verification passes the libavif-shaped two-sample parser fixture, the one-frame retention boundary, and a sample offset/length beyond the file. The parser performs no per-entry allocation and does not buffer `moov`, `mdat`, or complete attacker-sized tables.
-    - [ ] Parse composition offsets and hidden samples from `ctts`/`cslg`, optional direct dependencies from `refs` sample groups, track presentation/color/HDR properties, and bounded sequence metadata items.
+    - [x] Parse HEVC composition offsets and hidden samples from `ctts`/`cslg`, while rejecting the `ctts` box prohibited for AV1 tracks.
+      - Release verification covers signed and unsigned composition-offset syntax, hidden-sample visibility, composition-time calculation, required `cslg` and edit-list signaling, complete run counts, and the AV1 prohibition without buffering either table.
+    - [ ] Parse optional direct dependencies from `refs` sample groups, track presentation/color/HDR properties, and bounded sequence metadata items.
     - [ ] Connect the parsed sequence index to HEIF detection, Identify, frame decode, alpha matching, and frame metadata without changing still-image source selection.
     - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames.
   - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model.

src/ImageSharp/Formats/Heif/HeifSequenceParser.cs

@@ -645,6 +645,8 @@ internal sealed class HeifSequenceParser
         BoxReference sampleSizes = default;
         BoxReference chunkOffsets = default;
         BoxReference syncSamples = default;
+        BoxReference compositionOffsets = default;
+        BoxReference compositionToDecode = default;
 
         while (stream.Position < tableEnd)
         {
@@ -675,9 +677,11 @@ internal sealed class HeifSequenceParser
                     SetUnique(ref syncSamples, childStart, childLength, "sample table", childType);
                     break;
                 case Heif4CharCode.Ctts:
+                    SetUnique(ref compositionOffsets, childStart, childLength, "sample table", childType);
+                    break;
                 case Heif4CharCode.Cslg:
-                    throw new InvalidImageContentException(
+                    SetUnique(ref compositionToDecode, childStart, childLength, "sample table", childType);
-                        "Composition-offset image sequences are not yet supported by the bounded HEIF presentation model.");
+                    break;
             }
 
             stream.Position = checked(childStart + childLength);
@@ -730,6 +734,34 @@ internal sealed class HeifSequenceParser
                 samples[i].IsSync = true;
             }
         }
+
+        if (compositionOffsets.IsPresent)
+        {
+            if (track.CodecType == Heif4CharCode.Av01)
+            {
+                // AV1-ISOBMFF defines AV1 sample composition time as decode time and explicitly prohibits ctts.
+                throw new InvalidImageContentException("An AV1 image-sequence track contains a prohibited composition-offset box.");
+            }
+
+            stream.Position = compositionOffsets.Offset;
+            CompositionSummary composition = ParseCompositionOffsets(stream, compositionOffsets.Length, track, scratch);
+            if (composition.HasHiddenSamples && (!compositionToDecode.IsPresent || !track.HasEditList))
+            {
+                throw new InvalidImageContentException("A HEVC image-sequence track has hidden samples without the required composition and edit boxes.");
+            }
+
+            if (compositionToDecode.IsPresent)
+            {
+                stream.Position = compositionToDecode.Offset;
+                ParseCompositionToDecode(stream, compositionToDecode.Length, composition, scratch);
+            }
+        }
+        else if (compositionToDecode.IsPresent)
+        {
+            throw new InvalidImageContentException("The composition-to-decode box has no composition-offset table.");
+        }
+
+        SetCompositionTimes(track);
     }
 
     /// <summary>
@@ -1290,6 +1322,139 @@ internal sealed class HeifSequenceParser
         }
     }
 
+    /// <summary>
+    /// Parses HEVC decode-to-composition offsets and marks non-output reference samples.
+    /// </summary>
+    /// <param name="stream">The stream positioned at the composition-offset payload.</param>
+    /// <param name="boxLength">The validated composition-offset payload length.</param>
+    /// <param name="track">The selected HEVC track receiving retained composition offsets.</param>
+    /// <param name="scratch">The parser-owned reusable scratch span.</param>
+    /// <returns>The complete visible-offset range and hidden-sample state.</returns>
+    private static CompositionSummary ParseCompositionOffsets(Stream stream, long boxLength, HeifSequenceTrack track, Span<byte> scratch)
+    {
+        ReadOnlySpan<byte> prefix = ReadPrefix(stream, boxLength, scratch, 8, "composition offsets");
+        byte version = prefix[0];
+        if (version is not 0 and not 1 || ReadFlags(prefix) != 0)
+        {
+            throw new InvalidImageContentException("The composition-offset box has an unsupported version or flags.");
+        }
+
+        uint entryCount = BinaryPrimitives.ReadUInt32BigEndian(prefix[4..]);
+        long entryBytes = checked((long)entryCount * 8);
+        if (entryCount == 0 || boxLength != 8 + entryBytes)
+        {
+            throw new InvalidImageContentException("The composition-offset table is empty or has an invalid length.");
+        }
+
+        TableReader reader = new(stream, entryBytes, scratch, "composition offsets");
+        ulong describedSamples = 0;
+        int retainedOffset = 0;
+        long leastOffset = long.MaxValue;
+        long greatestOffset = long.MinValue;
+        bool hasHiddenSamples = false;
+        for (uint entry = 0; entry < entryCount; entry++)
+        {
+            uint sampleCount = reader.ReadUInt32();
+            uint rawOffset = reader.ReadUInt32();
+            if (sampleCount == 0)
+            {
+                throw new InvalidImageContentException("The composition-offset table contains a zero-length run.");
+            }
+
+            bool hidden = version == 1 && rawOffset == 0x80000000;
+            long compositionOffset = version == 0 ? rawOffset : unchecked((int)rawOffset);
+            describedSamples = checked(describedSamples + sampleCount);
+            hasHiddenSamples |= hidden;
+            if (!hidden)
+            {
+                leastOffset = Math.Min(leastOffset, compositionOffset);
+                greatestOffset = Math.Max(greatestOffset, compositionOffset);
+            }
+
+            int retainedRun = Math.Min((int)Math.Min(sampleCount, int.MaxValue), track.Samples.Length - retainedOffset);
+            Span<HeifSequenceSample> samples = track.Samples;
+            for (int i = 0; i < retainedRun; i++)
+            {
+                samples[retainedOffset + i].CompositionOffset = compositionOffset;
+                samples[retainedOffset + i].IsHidden = hidden;
+            }
+
+            retainedOffset += retainedRun;
+        }
+
+        if (describedSamples != track.TotalSampleCount || leastOffset == long.MaxValue)
+        {
+            throw new InvalidImageContentException("The composition-offset table does not describe every sample or contains no output sample.");
+        }
+
+        return new CompositionSummary(leastOffset, greatestOffset, hasHiddenSamples);
+    }
+
+    /// <summary>
+    /// Validates the track-wide composition bounds associated with HEVC non-output and reordered samples.
+    /// </summary>
+    /// <param name="stream">The stream positioned at the composition-to-decode payload.</param>
+    /// <param name="boxLength">The validated composition-to-decode payload length.</param>
+    /// <param name="composition">The offset range derived from the complete composition-offset table.</param>
+    /// <param name="scratch">The parser-owned reusable scratch span.</param>
+    private static void ParseCompositionToDecode(Stream stream, long boxLength, CompositionSummary composition, Span<byte> scratch)
+    {
+        ReadOnlySpan<byte> prefix = ReadPrefix(stream, boxLength, scratch, 4, "composition-to-decode");
+        byte version = prefix[0];
+        int fieldSize = version switch
+        {
+            0 => 4,
+            1 => 8,
+            _ => throw new InvalidImageContentException($"The composition-to-decode box has unsupported version {version}.")
+        };
+
+        int requiredLength = 4 + (fieldSize * 5);
+        prefix = ReadPrefixFromStart(stream, boxLength, scratch, requiredLength, "composition-to-decode");
+        if (boxLength != requiredLength || ReadFlags(prefix) != 0)
+        {
+            throw new InvalidImageContentException("The composition-to-decode box has unsupported flags or length.");
+        }
+
+        long shift = ReadSignedInteger(prefix[4..], fieldSize);
+        long leastOffset = ReadSignedInteger(prefix[(4 + fieldSize)..], fieldSize);
+        long greatestOffset = ReadSignedInteger(prefix[(4 + (fieldSize * 2))..], fieldSize);
+        long compositionStart = ReadSignedInteger(prefix[(4 + (fieldSize * 3))..], fieldSize);
+        long compositionEnd = ReadSignedInteger(prefix[(4 + (fieldSize * 4))..], fieldSize);
+        long requiredShift = composition.LeastOffset < 0 ? checked(-composition.LeastOffset) : 0;
+        if (shift < requiredShift
+            || leastOffset != composition.LeastOffset
+            || greatestOffset != composition.GreatestOffset
+            || (compositionEnd != 0 && compositionEnd < compositionStart))
+        {
+            throw new InvalidImageContentException("The composition-to-decode box does not match the track's composition offsets.");
+        }
+    }
+
+    /// <summary>
+    /// Computes retained sample composition times while preserving decode-order storage.
+    /// </summary>
+    /// <param name="track">The selected track whose durations and offsets have been validated.</param>
+    private static void SetCompositionTimes(HeifSequenceTrack track)
+    {
+        long decodeTime = 0;
+        Span<HeifSequenceSample> samples = track.Samples;
+        for (int i = 0; i < samples.Length; i++)
+        {
+            ref HeifSequenceSample sample = ref samples[i];
+            sample.CompositionTime = sample.IsHidden ? long.MinValue : checked(decodeTime + sample.CompositionOffset);
+            decodeTime = checked(decodeTime + sample.Duration);
+        }
+    }
+
+    /// <summary>
+    /// Reads one signed composition field of the version-selected fixed width.
+    /// </summary>
+    /// <param name="data">The field bytes.</param>
+    /// <param name="fieldSize">The four-byte or eight-byte field width.</param>
+    /// <returns>The signed field value.</returns>
+    private static long ReadSignedInteger(ReadOnlySpan<byte> data, int fieldSize)
+        => fieldSize == 4 ? BinaryPrimitives.ReadInt32BigEndian(data) : BinaryPrimitives.ReadInt64BigEndian(data);
+
     /// <summary>
     /// Parses the single normal-rate edit list used to signal image-sequence repetition.
     /// </summary>
@@ -1378,6 +1543,8 @@ internal sealed class HeifSequenceParser
             // indefinite rather than wrapping the observable ushort play count.
             track.RepeatCount = plays is 0 or > ushort.MaxValue ? (ushort)0 : (ushort)plays;
         }
+
+        track.HasEditList = true;
     }
 
     /// <summary>
@@ -1652,6 +1819,40 @@ internal sealed class HeifSequenceParser
         public uint SamplesPerChunk { get; }
     }
 
+    /// <summary>
+    /// Contains the visible composition-offset range derived from a complete HEVC track.
+    /// </summary>
+    private readonly struct CompositionSummary
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="CompositionSummary"/> struct.
+        /// </summary>
+        /// <param name="leastOffset">The smallest visible composition offset.</param>
+        /// <param name="greatestOffset">The greatest visible composition offset.</param>
+        /// <param name="hasHiddenSamples">Whether the track contains non-output samples.</param>
+        public CompositionSummary(long leastOffset, long greatestOffset, bool hasHiddenSamples)
+        {
+            this.LeastOffset = leastOffset;
+            this.GreatestOffset = greatestOffset;
+            this.HasHiddenSamples = hasHiddenSamples;
+        }
+
+        /// <summary>
+        /// Gets the smallest visible composition offset.
+        /// </summary>
+        public long LeastOffset { get; }
+
+        /// <summary>
+        /// Gets the greatest visible composition offset.
+        /// </summary>
+        public long GreatestOffset { get; }
+
+        /// <summary>
+        /// Gets a value indicating whether the track contains non-output samples.
+        /// </summary>
+        public bool HasHiddenSamples { get; }
+    }
+
     /// <summary>
     /// Reads fixed-width sample-table values through one bounded reusable buffer.
     /// </summary>

src/ImageSharp/Formats/Heif/HeifSequenceSample.cs

@@ -23,6 +23,21 @@ internal struct HeifSequenceSample
     /// </summary>
     public uint Duration { get; set; }
 
+    /// <summary>
+    /// Gets or sets the signed offset from decode time to composition time in media-time-scale units.
+    /// </summary>
+    public long CompositionOffset { get; set; }
+
+    /// <summary>
+    /// Gets or sets the computed composition time in media-time-scale units.
+    /// </summary>
+    public long CompositionTime { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the sample is decoded only as a reference and is not presented.
+    /// </summary>
+    public bool IsHidden { get; set; }
+
     /// <summary>
     /// Gets or sets a value indicating whether decoding can begin at this sample.
     /// </summary>

src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs

@@ -127,4 +127,9 @@ internal sealed class HeifSequenceTrack
     /// Gets or sets the track duration in movie-time-scale units.
     /// </summary>
     public ulong TrackDuration { get; set; }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the track contains the edit list required for hidden samples.
+    /// </summary>
+    public bool HasEditList { get; set; }
 }

tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs

@@ -79,7 +79,36 @@ public class HeifSequenceParserTests
         Assert.Throws<InvalidImageContentException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
     }
 
-    private static byte[] CreateSequenceFile(uint chunkOffset)
+    [Fact]
+    public void ParseMarksHiddenHevcSamples()
+    {
+        byte[] data = CreateSequenceFile(1024, hevc: true, compositionOffsets: true);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = new(Configuration.Default.MemoryAllocator, 2);
+        stream.Position = 8;
+
+        HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data));
+
+        Assert.Equal(Heif4CharCode.Hvc1, sequence.ColorTrack.CodecType);
+        Assert.NotNull(sequence.ColorTrack.HevcCodecConfiguration);
+        Assert.True(sequence.ColorTrack.Samples[0].IsHidden);
+        Assert.Equal(long.MinValue, sequence.ColorTrack.Samples[0].CompositionTime);
+        Assert.False(sequence.ColorTrack.Samples[1].IsHidden);
+        Assert.Equal(100, sequence.ColorTrack.Samples[1].CompositionTime);
+    }
+
+    [Fact]
+    public void ParseRejectsCompositionOffsetsForAv1()
+    {
+        byte[] data = CreateSequenceFile(1024, compositionOffsets: true);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = new(Configuration.Default.MemoryAllocator, 2);
+        stream.Position = 8;
+
+        Assert.Throws<InvalidImageContentException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+    }
+
+    private static byte[] CreateSequenceFile(uint chunkOffset, bool hevc = false, bool compositionOffsets = false)
     {
         using MemoryStream stream = new();
         using BinaryWriter writer = new(stream, Encoding.UTF8, true);
@@ -104,7 +133,7 @@ public class HeifSequenceParserTests
 
         long mediaInformation = BeginBox(writer, Heif4CharCode.Minf);
         WriteDataInformation(writer);
-        WriteSampleTable(writer, chunkOffset);
+        WriteSampleTable(writer, chunkOffset, hevc, compositionOffsets);
         EndBox(writer, mediaInformation);
         EndBox(writer, media);
         EndBox(writer, track);
@@ -192,10 +221,10 @@ public class HeifSequenceParserTests
         EndBox(writer, dataInformation);
     }
 
-    private static void WriteSampleTable(BinaryWriter writer, uint chunkOffset)
+    private static void WriteSampleTable(BinaryWriter writer, uint chunkOffset, bool hevc, bool compositionOffsets)
     {
         long sampleTable = BeginBox(writer, Heif4CharCode.Stbl);
-        WriteSampleDescription(writer);
+        WriteSampleDescription(writer, hevc);
 
         long timing = BeginBox(writer, Heif4CharCode.Stts);
         WriteFullBoxHeader(writer, 0, 0);
@@ -231,15 +260,37 @@ public class HeifSequenceParserTests
         WriteUInt32(writer, 1);
         WriteUInt32(writer, 1);
         EndBox(writer, syncSamples);
+
+        if (compositionOffsets)
+        {
+            long offsets = BeginBox(writer, Heif4CharCode.Ctts);
+            WriteFullBoxHeader(writer, 1, 0);
+            WriteUInt32(writer, 2);
+            WriteUInt32(writer, 1);
+            WriteUInt32(writer, 0x80000000);
+            WriteUInt32(writer, 1);
+            WriteUInt32(writer, 0);
+            EndBox(writer, offsets);
+
+            long compositionToDecode = BeginBox(writer, Heif4CharCode.Cslg);
+            WriteFullBoxHeader(writer, 0, 0);
+            WriteUInt32(writer, 0);
+            WriteUInt32(writer, 0);
+            WriteUInt32(writer, 0);
+            WriteUInt32(writer, 100);
+            WriteUInt32(writer, 200);
+            EndBox(writer, compositionToDecode);
+        }
+
         EndBox(writer, sampleTable);
     }
 
-    private static void WriteSampleDescription(BinaryWriter writer)
+    private static void WriteSampleDescription(BinaryWriter writer, bool hevc)
     {
         long description = BeginBox(writer, Heif4CharCode.Stsd);
         WriteFullBoxHeader(writer, 0, 0);
         WriteUInt32(writer, 1);
-        long sampleEntry = BeginBox(writer, Heif4CharCode.Av01);
+        long sampleEntry = BeginBox(writer, hevc ? Heif4CharCode.Hvc1 : Heif4CharCode.Av01);
         WriteZeros(writer, 6);
         WriteUInt16(writer, 1);
         WriteZeros(writer, 16);
@@ -253,9 +304,16 @@ public class HeifSequenceParserTests
         WriteUInt16(writer, 0x18);
         WriteUInt16(writer, ushort.MaxValue);
 
-        long configuration = BeginBox(writer, Heif4CharCode.Av1C);
+        if (hevc)
-        writer.Write(new byte[] { 0x81, 0, 0, 0 });
+        {
-        EndBox(writer, configuration);
+            WriteHevcConfiguration(writer);
+        }
+        else
+        {
+            long configuration = BeginBox(writer, Heif4CharCode.Av1C);
+            writer.Write(new byte[] { 0x81, 0, 0, 0 });
+            EndBox(writer, configuration);
+        }
 
         long codingConstraints = BeginBox(writer, Heif4CharCode.Ccst);
         WriteFullBoxHeader(writer, 0, 0);
@@ -265,6 +323,25 @@ public class HeifSequenceParserTests
         EndBox(writer, description);
     }
 
+    private static void WriteHevcConfiguration(BinaryWriter writer)
+    {
+        long configuration = BeginBox(writer, Heif4CharCode.HvcC);
+        writer.Write((byte)1);
+        writer.Write((byte)1);
+        WriteUInt32(writer, 0);
+        WriteZeros(writer, 6);
+        writer.Write((byte)0);
+        WriteUInt16(writer, 0xF000);
+        writer.Write((byte)0xFC);
+        writer.Write((byte)0xFD);
+        writer.Write((byte)0xF8);
+        writer.Write((byte)0xF8);
+        WriteUInt16(writer, 0);
+        writer.Write((byte)3);
+        writer.Write((byte)0);
+        EndBox(writer, configuration);
+    }
+
     private static long BeginBox(BinaryWriter writer, Heif4CharCode type)
     {
         long start = writer.BaseStream.Position;