diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 1ee8f0d4c..5b261f806 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -53,7 +53,9 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Select one enabled `pict` master track without materializing unrelated tracks, then parse its `mvhd`/`tkhd`/`mdhd`/`hdlr`, self-contained `dref`, `av01` or `hvc1` sample entry, codec configuration, mandatory `ccst`, and bounded repetition edit. - [x] Resolve `stsc`, `stco`/`co64`, `stsz`/`stz2`, `stts`, and `stss` into one exact value-type descriptor array capped by `DecoderOptions.MaxFrames`; validate complete run/count syntax through one allocator-owned sequential scratch buffer. - Release verification passes the libavif-shaped two-sample parser fixture, the one-frame retention boundary, and a sample offset/length beyond the file. The parser performs no per-entry allocation and does not buffer `moov`, `mdat`, or complete attacker-sized tables. - - [ ] Parse composition offsets and hidden samples from `ctts`/`cslg`, optional direct dependencies from `refs` sample groups, track presentation/color/HDR properties, and bounded sequence metadata items. + - [x] Parse HEVC composition offsets and hidden samples from `ctts`/`cslg`, while rejecting the `ctts` box prohibited for AV1 tracks. + - Release verification covers signed and unsigned composition-offset syntax, hidden-sample visibility, composition-time calculation, required `cslg` and edit-list signaling, complete run counts, and the AV1 prohibition without buffering either table. + - [ ] Parse optional direct dependencies from `refs` sample groups, track presentation/color/HDR properties, and bounded sequence metadata items. - [ ] Connect the parsed sequence index to HEIF detection, Identify, frame decode, alpha matching, and frame metadata without changing still-image source selection. - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames. - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model. diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs index ff9e71114..949087d43 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs @@ -645,6 +645,8 @@ internal sealed class HeifSequenceParser BoxReference sampleSizes = default; BoxReference chunkOffsets = default; BoxReference syncSamples = default; + BoxReference compositionOffsets = default; + BoxReference compositionToDecode = default; while (stream.Position < tableEnd) { @@ -675,9 +677,11 @@ internal sealed class HeifSequenceParser SetUnique(ref syncSamples, childStart, childLength, "sample table", childType); break; case Heif4CharCode.Ctts: + SetUnique(ref compositionOffsets, childStart, childLength, "sample table", childType); + break; case Heif4CharCode.Cslg: - throw new InvalidImageContentException( - "Composition-offset image sequences are not yet supported by the bounded HEIF presentation model."); + SetUnique(ref compositionToDecode, childStart, childLength, "sample table", childType); + break; } stream.Position = checked(childStart + childLength); @@ -730,6 +734,34 @@ internal sealed class HeifSequenceParser samples[i].IsSync = true; } } + + if (compositionOffsets.IsPresent) + { + if (track.CodecType == Heif4CharCode.Av01) + { + // AV1-ISOBMFF defines AV1 sample composition time as decode time and explicitly prohibits ctts. + throw new InvalidImageContentException("An AV1 image-sequence track contains a prohibited composition-offset box."); + } + + stream.Position = compositionOffsets.Offset; + CompositionSummary composition = ParseCompositionOffsets(stream, compositionOffsets.Length, track, scratch); + if (composition.HasHiddenSamples && (!compositionToDecode.IsPresent || !track.HasEditList)) + { + throw new InvalidImageContentException("A HEVC image-sequence track has hidden samples without the required composition and edit boxes."); + } + + if (compositionToDecode.IsPresent) + { + stream.Position = compositionToDecode.Offset; + ParseCompositionToDecode(stream, compositionToDecode.Length, composition, scratch); + } + } + else if (compositionToDecode.IsPresent) + { + throw new InvalidImageContentException("The composition-to-decode box has no composition-offset table."); + } + + SetCompositionTimes(track); } /// @@ -1290,6 +1322,139 @@ internal sealed class HeifSequenceParser } } + /// + /// Parses HEVC decode-to-composition offsets and marks non-output reference samples. + /// + /// The stream positioned at the composition-offset payload. + /// The validated composition-offset payload length. + /// The selected HEVC track receiving retained composition offsets. + /// The parser-owned reusable scratch span. + /// The complete visible-offset range and hidden-sample state. + private static CompositionSummary ParseCompositionOffsets(Stream stream, long boxLength, HeifSequenceTrack track, Span scratch) + { + ReadOnlySpan prefix = ReadPrefix(stream, boxLength, scratch, 8, "composition offsets"); + byte version = prefix[0]; + if (version is not 0 and not 1 || ReadFlags(prefix) != 0) + { + throw new InvalidImageContentException("The composition-offset box has an unsupported version or flags."); + } + + uint entryCount = BinaryPrimitives.ReadUInt32BigEndian(prefix[4..]); + long entryBytes = checked((long)entryCount * 8); + if (entryCount == 0 || boxLength != 8 + entryBytes) + { + throw new InvalidImageContentException("The composition-offset table is empty or has an invalid length."); + } + + TableReader reader = new(stream, entryBytes, scratch, "composition offsets"); + ulong describedSamples = 0; + int retainedOffset = 0; + long leastOffset = long.MaxValue; + long greatestOffset = long.MinValue; + bool hasHiddenSamples = false; + for (uint entry = 0; entry < entryCount; entry++) + { + uint sampleCount = reader.ReadUInt32(); + uint rawOffset = reader.ReadUInt32(); + if (sampleCount == 0) + { + throw new InvalidImageContentException("The composition-offset table contains a zero-length run."); + } + + bool hidden = version == 1 && rawOffset == 0x80000000; + long compositionOffset = version == 0 ? rawOffset : unchecked((int)rawOffset); + describedSamples = checked(describedSamples + sampleCount); + hasHiddenSamples |= hidden; + if (!hidden) + { + leastOffset = Math.Min(leastOffset, compositionOffset); + greatestOffset = Math.Max(greatestOffset, compositionOffset); + } + + int retainedRun = Math.Min((int)Math.Min(sampleCount, int.MaxValue), track.Samples.Length - retainedOffset); + Span samples = track.Samples; + for (int i = 0; i < retainedRun; i++) + { + samples[retainedOffset + i].CompositionOffset = compositionOffset; + samples[retainedOffset + i].IsHidden = hidden; + } + + retainedOffset += retainedRun; + } + + if (describedSamples != track.TotalSampleCount || leastOffset == long.MaxValue) + { + throw new InvalidImageContentException("The composition-offset table does not describe every sample or contains no output sample."); + } + + return new CompositionSummary(leastOffset, greatestOffset, hasHiddenSamples); + } + + /// + /// Validates the track-wide composition bounds associated with HEVC non-output and reordered samples. + /// + /// The stream positioned at the composition-to-decode payload. + /// The validated composition-to-decode payload length. + /// The offset range derived from the complete composition-offset table. + /// The parser-owned reusable scratch span. + private static void ParseCompositionToDecode(Stream stream, long boxLength, CompositionSummary composition, Span scratch) + { + ReadOnlySpan prefix = ReadPrefix(stream, boxLength, scratch, 4, "composition-to-decode"); + byte version = prefix[0]; + int fieldSize = version switch + { + 0 => 4, + 1 => 8, + _ => throw new InvalidImageContentException($"The composition-to-decode box has unsupported version {version}.") + }; + + int requiredLength = 4 + (fieldSize * 5); + prefix = ReadPrefixFromStart(stream, boxLength, scratch, requiredLength, "composition-to-decode"); + if (boxLength != requiredLength || ReadFlags(prefix) != 0) + { + throw new InvalidImageContentException("The composition-to-decode box has unsupported flags or length."); + } + + long shift = ReadSignedInteger(prefix[4..], fieldSize); + long leastOffset = ReadSignedInteger(prefix[(4 + fieldSize)..], fieldSize); + long greatestOffset = ReadSignedInteger(prefix[(4 + (fieldSize * 2))..], fieldSize); + long compositionStart = ReadSignedInteger(prefix[(4 + (fieldSize * 3))..], fieldSize); + long compositionEnd = ReadSignedInteger(prefix[(4 + (fieldSize * 4))..], fieldSize); + long requiredShift = composition.LeastOffset < 0 ? checked(-composition.LeastOffset) : 0; + if (shift < requiredShift + || leastOffset != composition.LeastOffset + || greatestOffset != composition.GreatestOffset + || (compositionEnd != 0 && compositionEnd < compositionStart)) + { + throw new InvalidImageContentException("The composition-to-decode box does not match the track's composition offsets."); + } + } + + /// + /// Computes retained sample composition times while preserving decode-order storage. + /// + /// The selected track whose durations and offsets have been validated. + private static void SetCompositionTimes(HeifSequenceTrack track) + { + long decodeTime = 0; + Span samples = track.Samples; + for (int i = 0; i < samples.Length; i++) + { + ref HeifSequenceSample sample = ref samples[i]; + sample.CompositionTime = sample.IsHidden ? long.MinValue : checked(decodeTime + sample.CompositionOffset); + decodeTime = checked(decodeTime + sample.Duration); + } + } + + /// + /// Reads one signed composition field of the version-selected fixed width. + /// + /// The field bytes. + /// The four-byte or eight-byte field width. + /// The signed field value. + private static long ReadSignedInteger(ReadOnlySpan data, int fieldSize) + => fieldSize == 4 ? BinaryPrimitives.ReadInt32BigEndian(data) : BinaryPrimitives.ReadInt64BigEndian(data); + /// /// Parses the single normal-rate edit list used to signal image-sequence repetition. /// @@ -1378,6 +1543,8 @@ internal sealed class HeifSequenceParser // indefinite rather than wrapping the observable ushort play count. track.RepeatCount = plays is 0 or > ushort.MaxValue ? (ushort)0 : (ushort)plays; } + + track.HasEditList = true; } /// @@ -1652,6 +1819,40 @@ internal sealed class HeifSequenceParser public uint SamplesPerChunk { get; } } + /// + /// Contains the visible composition-offset range derived from a complete HEVC track. + /// + private readonly struct CompositionSummary + { + /// + /// Initializes a new instance of the struct. + /// + /// The smallest visible composition offset. + /// The greatest visible composition offset. + /// Whether the track contains non-output samples. + public CompositionSummary(long leastOffset, long greatestOffset, bool hasHiddenSamples) + { + this.LeastOffset = leastOffset; + this.GreatestOffset = greatestOffset; + this.HasHiddenSamples = hasHiddenSamples; + } + + /// + /// Gets the smallest visible composition offset. + /// + public long LeastOffset { get; } + + /// + /// Gets the greatest visible composition offset. + /// + public long GreatestOffset { get; } + + /// + /// Gets a value indicating whether the track contains non-output samples. + /// + public bool HasHiddenSamples { get; } + } + /// /// Reads fixed-width sample-table values through one bounded reusable buffer. /// diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceSample.cs b/src/ImageSharp/Formats/Heif/HeifSequenceSample.cs index a9d562263..c5a8c2207 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceSample.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceSample.cs @@ -23,6 +23,21 @@ internal struct HeifSequenceSample /// public uint Duration { get; set; } + /// + /// Gets or sets the signed offset from decode time to composition time in media-time-scale units. + /// + public long CompositionOffset { get; set; } + + /// + /// Gets or sets the computed composition time in media-time-scale units. + /// + public long CompositionTime { get; set; } + + /// + /// Gets or sets a value indicating whether the sample is decoded only as a reference and is not presented. + /// + public bool IsHidden { get; set; } + /// /// Gets or sets a value indicating whether decoding can begin at this sample. /// diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs index 5659fdc62..ced4d846d 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs @@ -127,4 +127,9 @@ internal sealed class HeifSequenceTrack /// Gets or sets the track duration in movie-time-scale units. /// public ulong TrackDuration { get; set; } + + /// + /// Gets or sets a value indicating whether the track contains the edit list required for hidden samples. + /// + public bool HasEditList { get; set; } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs index 2abcb7ef2..54634b9f7 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs @@ -79,7 +79,36 @@ public class HeifSequenceParserTests Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); } - private static byte[] CreateSequenceFile(uint chunkOffset) + [Fact] + public void ParseMarksHiddenHevcSamples() + { + byte[] data = CreateSequenceFile(1024, hevc: true, compositionOffsets: true); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = new(Configuration.Default.MemoryAllocator, 2); + stream.Position = 8; + + HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data)); + + Assert.Equal(Heif4CharCode.Hvc1, sequence.ColorTrack.CodecType); + Assert.NotNull(sequence.ColorTrack.HevcCodecConfiguration); + Assert.True(sequence.ColorTrack.Samples[0].IsHidden); + Assert.Equal(long.MinValue, sequence.ColorTrack.Samples[0].CompositionTime); + Assert.False(sequence.ColorTrack.Samples[1].IsHidden); + Assert.Equal(100, sequence.ColorTrack.Samples[1].CompositionTime); + } + + [Fact] + public void ParseRejectsCompositionOffsetsForAv1() + { + byte[] data = CreateSequenceFile(1024, compositionOffsets: true); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = new(Configuration.Default.MemoryAllocator, 2); + stream.Position = 8; + + Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); + } + + private static byte[] CreateSequenceFile(uint chunkOffset, bool hevc = false, bool compositionOffsets = false) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.UTF8, true); @@ -104,7 +133,7 @@ public class HeifSequenceParserTests long mediaInformation = BeginBox(writer, Heif4CharCode.Minf); WriteDataInformation(writer); - WriteSampleTable(writer, chunkOffset); + WriteSampleTable(writer, chunkOffset, hevc, compositionOffsets); EndBox(writer, mediaInformation); EndBox(writer, media); EndBox(writer, track); @@ -192,10 +221,10 @@ public class HeifSequenceParserTests EndBox(writer, dataInformation); } - private static void WriteSampleTable(BinaryWriter writer, uint chunkOffset) + private static void WriteSampleTable(BinaryWriter writer, uint chunkOffset, bool hevc, bool compositionOffsets) { long sampleTable = BeginBox(writer, Heif4CharCode.Stbl); - WriteSampleDescription(writer); + WriteSampleDescription(writer, hevc); long timing = BeginBox(writer, Heif4CharCode.Stts); WriteFullBoxHeader(writer, 0, 0); @@ -231,15 +260,37 @@ public class HeifSequenceParserTests WriteUInt32(writer, 1); WriteUInt32(writer, 1); EndBox(writer, syncSamples); + + if (compositionOffsets) + { + long offsets = BeginBox(writer, Heif4CharCode.Ctts); + WriteFullBoxHeader(writer, 1, 0); + WriteUInt32(writer, 2); + WriteUInt32(writer, 1); + WriteUInt32(writer, 0x80000000); + WriteUInt32(writer, 1); + WriteUInt32(writer, 0); + EndBox(writer, offsets); + + long compositionToDecode = BeginBox(writer, Heif4CharCode.Cslg); + WriteFullBoxHeader(writer, 0, 0); + WriteUInt32(writer, 0); + WriteUInt32(writer, 0); + WriteUInt32(writer, 0); + WriteUInt32(writer, 100); + WriteUInt32(writer, 200); + EndBox(writer, compositionToDecode); + } + EndBox(writer, sampleTable); } - private static void WriteSampleDescription(BinaryWriter writer) + private static void WriteSampleDescription(BinaryWriter writer, bool hevc) { long description = BeginBox(writer, Heif4CharCode.Stsd); WriteFullBoxHeader(writer, 0, 0); WriteUInt32(writer, 1); - long sampleEntry = BeginBox(writer, Heif4CharCode.Av01); + long sampleEntry = BeginBox(writer, hevc ? Heif4CharCode.Hvc1 : Heif4CharCode.Av01); WriteZeros(writer, 6); WriteUInt16(writer, 1); WriteZeros(writer, 16); @@ -253,9 +304,16 @@ public class HeifSequenceParserTests WriteUInt16(writer, 0x18); WriteUInt16(writer, ushort.MaxValue); - long configuration = BeginBox(writer, Heif4CharCode.Av1C); - writer.Write(new byte[] { 0x81, 0, 0, 0 }); - EndBox(writer, configuration); + if (hevc) + { + WriteHevcConfiguration(writer); + } + else + { + long configuration = BeginBox(writer, Heif4CharCode.Av1C); + writer.Write(new byte[] { 0x81, 0, 0, 0 }); + EndBox(writer, configuration); + } long codingConstraints = BeginBox(writer, Heif4CharCode.Ccst); WriteFullBoxHeader(writer, 0, 0); @@ -265,6 +323,25 @@ public class HeifSequenceParserTests EndBox(writer, description); } + private static void WriteHevcConfiguration(BinaryWriter writer) + { + long configuration = BeginBox(writer, Heif4CharCode.HvcC); + writer.Write((byte)1); + writer.Write((byte)1); + WriteUInt32(writer, 0); + WriteZeros(writer, 6); + writer.Write((byte)0); + WriteUInt16(writer, 0xF000); + writer.Write((byte)0xFC); + writer.Write((byte)0xFD); + writer.Write((byte)0xF8); + writer.Write((byte)0xF8); + WriteUInt16(writer, 0); + writer.Write((byte)3); + writer.Write((byte)0); + EndBox(writer, configuration); + } + private static long BeginBox(BinaryWriter writer, Heif4CharCode type) { long start = writer.BaseStream.Position;