Bound HEIF sequence presentation matrices

HEIF_IMPLEMENTATION_PLAN.md

@@ -66,7 +66,8 @@ Checkboxes may be marked complete only when the implementation and the verificat
       - [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state.
       - [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers.
       - [x] Match auxiliary alpha samples by exact decode duration, visibility, and presentation time, and validate premultiplication track identity.
-      - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, track-matrix presentation, and independent sequence vectors.
+      - [x] Require unity movie and track matrices so image presentation remains on the optimized `clap`/`irot`/`imir` path without a movie compositor.
+      - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction and independent sequence vectors.
     - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames.
   - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model.
   - [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences.
@@ -123,7 +124,7 @@ The sequence reader and writer may retain only the following syntax and the reso
 | Syntax | Required image behavior |
 | --- | --- |
 | `ftyp` sequence and structural brands | Recognize `avis` AV1 sequences and the non-layered `hevc`/`hevx` HEVC sequence profiles. `avio` can additionally signal an all-sync AV1 sequence. Layered `hevm`/`hevs`, JPEG `jpgs`, arbitrary video brands, and brands for unimplemented codecs remain unsupported until their image payload and presentation requirements are implemented. |
-| `moov`/`mvhd`, `trak`/`tkhd`, and `mdia`/`mdhd`/`hdlr` | Select an enabled, in-movie `pict` master image-sequence track; retain its displayed dimensions, presentation matrix, media time scale, media duration, and movie-time-scale track duration. Ignore unrelated tracks rather than exposing them. |
+| `moov`/`mvhd`, `trak`/`tkhd`, and `mdia`/`mdhd`/`hdlr` | Select an enabled, in-movie `pict` master image-sequence track; retain its displayed dimensions, media time scale, media duration, and movie-time-scale track duration. Require unity movie and track matrices because arbitrary movie-canvas composition is outside image-format scope; use `clap`/`irot`/`imir` for supported image presentation. Ignore unrelated tracks rather than exposing them. |
 | `minf`/`dinf`/`dref` and `stbl` | Accept only self-contained sample data references and own the bounded sample-table state for one selected image sequence plus an optional linked auxiliary-alpha sequence. No reusable data-reference, media-information, or sample-table API is created. |
 | `stsd` and one supported visual sample entry | Require exactly one `av01` entry for AVIF or one non-layered `hvc1` entry for HEIC. Retain only its dimensions, codec configuration (`av1C` or `hvcC`), image presentation/color/HDR properties, and mandatory version-zero `ccst` coding constraints. Reject an unsupported essential configuration rather than treating it as generic video. |
 | `stsc`, `stco`/`co64`, and `stsz`/`stz2` | Resolve each declared image sample directly to a validated file offset and length. Expand run tables once into a compact frame-owned descriptor array bounded by `DecoderOptions.MaxFrames`; never buffer the movie or complete `mdat`. |

src/ImageSharp/Formats/Heif/HeifSequenceParser.cs

@@ -264,7 +264,6 @@ internal sealed class HeifSequenceParser
             Id = identity.Id,
             Width = identity.Width,
             Height = identity.Height,
-            Matrix = identity.Matrix,
             HandlerType = identity.HandlerType,
             TrackDuration = identity.TrackDuration,
             AuxiliaryForTrackId = identity.AuxiliaryForTrackId,
@@ -292,11 +291,6 @@ internal sealed class HeifSequenceParser
 
         stream.Position = media.Offset;
         this.ParseMedia(stream, media.Length, track, scratch);
-        if (track.Matrix.HasPerspective)
-        {
-            throw new InvalidImageContentException("The HEIF image-sequence track uses an unsupported perspective matrix.");
-        }
-
         if (track.TotalSampleCount == 0 || track.Samples.Length == 0)
         {
             throw new InvalidImageContentException("The HEIF image-sequence track contains no retained image samples.");
@@ -306,7 +300,7 @@ internal sealed class HeifSequenceParser
     }
 
     /// <summary>
-    /// Parses the movie time scale required to interpret track edit durations.
+    /// Parses the movie time scale and validates that no general movie-canvas transformation is required.
     /// </summary>
     /// <param name="stream">The stream positioned at the movie-header payload.</param>
     /// <param name="boxLength">The validated movie-header payload length.</param>
@@ -318,8 +312,8 @@ internal sealed class HeifSequenceParser
         byte version = prefix[0];
         int requiredLength = version switch
         {
-            0 => 20,
-            1 => 32,
+            0 => 100,
+            1 => 112,
             _ => throw new InvalidImageContentException($"The movie header has unsupported version {version}.")
         };
 
@@ -331,6 +325,13 @@ internal sealed class HeifSequenceParser
             throw new InvalidImageContentException("The movie header has a zero time scale.");
         }
 
+        int matrixOffset = version == 0 ? 36 : 48;
+        HeifTrackMatrix matrix = HeifTrackMatrix.Parse(prefix.Slice(matrixOffset, 36));
+        if (!matrix.IsIdentity)
+        {
+            throw new NotSupportedException("The HEIF image sequence requires an unsupported movie presentation matrix.");
+        }
+
         return timescale;
     }
 
@@ -380,18 +381,13 @@ internal sealed class HeifSequenceParser
             throw new InvalidImageContentException("A HEIF image-sequence track has zero dimensions.");
         }
 
-        HeifTrackMatrix matrix = new(
-            BinaryPrimitives.ReadInt32BigEndian(prefix[matrixOffset..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 4)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 8)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 12)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 16)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 20)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 24)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 28)..]),
-            BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 32)..]));
+        HeifTrackMatrix matrix = HeifTrackMatrix.Parse(prefix.Slice(matrixOffset, 36));
+        if (!matrix.IsIdentity)
+        {
+            throw new NotSupportedException("The HEIF image-sequence track requires an unsupported movie presentation matrix.");
+        }
 
-        return new TrackIdentity(id, (flags & 3) == 3, width, height, duration, matrix);
+        return new TrackIdentity(id, (flags & 3) == 3, width, height, duration);
     }
 
     /// <summary>
@@ -2371,15 +2367,13 @@ internal sealed class HeifSequenceParser
         /// <param name="width">The displayed track width.</param>
         /// <param name="height">The displayed track height.</param>
         /// <param name="trackDuration">The track duration in movie-time-scale units.</param>
-        /// <param name="matrix">The track presentation matrix.</param>
-        public TrackIdentity(uint id, bool isEnabledInMovie, int width, int height, ulong trackDuration, HeifTrackMatrix matrix)
+        public TrackIdentity(uint id, bool isEnabledInMovie, int width, int height, ulong trackDuration)
         {
             this.Id = id;
             this.IsEnabledInMovie = isEnabledInMovie;
             this.Width = width;
             this.Height = height;
             this.TrackDuration = trackDuration;
-            this.Matrix = matrix;
             this.HandlerType = default;
             this.AuxiliaryForTrackId = 0;
             this.PremultipliedByTrackId = 0;
@@ -2410,11 +2404,6 @@ internal sealed class HeifSequenceParser
         /// </summary>
         public ulong TrackDuration { get; }
 
-        /// <summary>
-        /// Gets the track presentation matrix.
-        /// </summary>
-        public HeifTrackMatrix Matrix { get; }
-
         /// <summary>
         /// Gets or sets the media handler type.
         /// </summary>

src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs

@@ -45,11 +45,6 @@ internal sealed class HeifSequenceTrack
     /// </summary>
     public int CodedHeight { get; set; }
 
-    /// <summary>
-    /// Gets or sets the track transformation matrix.
-    /// </summary>
-    public HeifTrackMatrix Matrix { get; set; }
-
     /// <summary>
     /// Gets or sets the media time scale in units per second.
     /// </summary>

src/ImageSharp/Formats/Heif/HeifTrackMatrix.cs

@@ -1,10 +1,12 @@
 // Copyright (c) Six Labors.
 // Licensed under the Six Labors Split License.
 
+using System.Buffers.Binary;
+
 namespace SixLabors.ImageSharp.Formats.Heif;
 
 /// <summary>
-/// Contains the fixed-point transformation matrix of a HEIF image-sequence track.
+/// Contains a fixed-point movie or track transformation matrix from a HEIF image sequence.
 /// </summary>
 internal readonly struct HeifTrackMatrix
 {
@@ -79,7 +81,25 @@ internal readonly struct HeifTrackMatrix
     public int W { get; }
 
     /// <summary>
-    /// Gets a value indicating whether the matrix contains unsupported perspective projection.
+    /// Gets a value indicating whether the matrix leaves the image coordinate system unchanged.
+    /// </summary>
+    public bool IsIdentity => this.A == 0x00010000 && this.B == 0 && this.U == 0 && this.C == 0 && this.D == 0x00010000 &&
+        this.V == 0 && this.X == 0 && this.Y == 0 && this.W == 0x40000000;
+
+    /// <summary>
+    /// Reads the nine fixed-point coefficients from a matrix payload in file byte order.
     /// </summary>
-    public bool HasPerspective => this.U != 0 || this.V != 0 || this.W != 0x40000000;
+    /// <param name="data">The 36-byte matrix payload.</param>
+    /// <returns>The decoded transformation matrix.</returns>
+    public static HeifTrackMatrix Parse(ReadOnlySpan<byte> data)
+        => new(
+            BinaryPrimitives.ReadInt32BigEndian(data),
+            BinaryPrimitives.ReadInt32BigEndian(data[4..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[8..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[12..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[16..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[20..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[24..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[28..]),
+            BinaryPrimitives.ReadInt32BigEndian(data[32..]));
 }

tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs

@@ -184,6 +184,23 @@ public class HeifSequenceParserTests
         Assert.Throws<InvalidImageContentException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
     }
 
+    [Theory]
+    [InlineData(true, false)]
+    [InlineData(false, true)]
+    public void ParseRejectsNonIdentityMoviePresentationMatrix(bool nonIdentityMovieMatrix, bool nonIdentityTrackMatrix)
+    {
+        byte[] data = CreateSequenceFile(
+            1024,
+            nonIdentityMovieMatrix: nonIdentityMovieMatrix,
+            nonIdentityTrackMatrix: nonIdentityTrackMatrix);
+
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2);
+        stream.Position = 8;
+
+        Assert.Throws<NotSupportedException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+    }
+
     [Fact]
     public void ParseMarksHiddenHevcSamples()
     {
@@ -429,7 +446,9 @@ public class HeifSequenceParserTests
         byte[] av1Configuration = null,
         int? sampleSize = null,
         bool allSamplesSync = false,
-        uint premultipliedByTrackId = 0)
+        uint premultipliedByTrackId = 0,
+        bool nonIdentityMovieMatrix = false,
+        bool nonIdentityTrackMatrix = false)
     {
         using MemoryStream stream = new();
         using BinaryWriter writer = new(stream, Encoding.UTF8, true);
@@ -441,11 +460,17 @@ public class HeifSequenceParserTests
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 1000);
         WriteUInt32(writer, 600);
-        WriteZeros(writer, 80);
+        WriteUInt32(writer, 0x00010000);
+        WriteUInt16(writer, 0x0100);
+        WriteUInt16(writer, 0);
+        WriteZeros(writer, 8);
+        WritePresentationMatrix(writer, nonIdentityMovieMatrix);
+        WriteZeros(writer, 24);
+        WriteUInt32(writer, 3);
         EndBox(writer, movieHeader);
 
         long track = BeginBox(writer, Heif4CharCode.Trak);
-        WriteTrackHeader(writer, width, height);
+        WriteTrackHeader(writer, width, height, 1, nonIdentityTrackMatrix);
         if (premultipliedByTrackId != 0)
         {
             WriteTrackReference(writer, Heif4CharCode.Prem, premultipliedByTrackId);
@@ -625,7 +650,7 @@ public class HeifSequenceParserTests
         return data;
     }
 
-    private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1)
+    private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1, bool nonIdentityMatrix = false)
     {
         long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd);
         WriteFullBoxHeader(writer, 0, 3);
@@ -635,7 +660,15 @@ public class HeifSequenceParserTests
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 600);
         WriteZeros(writer, 16);
-        WriteUInt32(writer, 0x00010000);
+        WritePresentationMatrix(writer, nonIdentityMatrix);
+        WriteUInt32(writer, (uint)width << 16);
+        WriteUInt32(writer, (uint)height << 16);
+        EndBox(writer, trackHeader);
+    }
+
+    private static void WritePresentationMatrix(BinaryWriter writer, bool nonIdentityMatrix)
+    {
+        WriteUInt32(writer, nonIdentityMatrix ? 0x00020000U : 0x00010000U);
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 0);
@@ -644,9 +677,6 @@ public class HeifSequenceParserTests
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 0);
         WriteUInt32(writer, 0x40000000);
-        WriteUInt32(writer, (uint)width << 16);
-        WriteUInt32(writer, (uint)height << 16);
-        EndBox(writer, trackHeader);
     }
 
     private static void WriteTrackReference(BinaryWriter writer, Heif4CharCode referenceType, uint trackId)