diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 02cd6bd9e..232301f61 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -66,7 +66,8 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state. - [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers. - [x] Match auxiliary alpha samples by exact decode duration, visibility, and presentation time, and validate premultiplication track identity. - - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, track-matrix presentation, and independent sequence vectors. + - [x] Require unity movie and track matrices so image presentation remains on the optimized `clap`/`irot`/`imir` path without a movie compositor. + - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction and independent sequence vectors. - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames. - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model. - [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences. @@ -123,7 +124,7 @@ The sequence reader and writer may retain only the following syntax and the reso | Syntax | Required image behavior | | --- | --- | | `ftyp` sequence and structural brands | Recognize `avis` AV1 sequences and the non-layered `hevc`/`hevx` HEVC sequence profiles. `avio` can additionally signal an all-sync AV1 sequence. Layered `hevm`/`hevs`, JPEG `jpgs`, arbitrary video brands, and brands for unimplemented codecs remain unsupported until their image payload and presentation requirements are implemented. | -| `moov`/`mvhd`, `trak`/`tkhd`, and `mdia`/`mdhd`/`hdlr` | Select an enabled, in-movie `pict` master image-sequence track; retain its displayed dimensions, presentation matrix, media time scale, media duration, and movie-time-scale track duration. Ignore unrelated tracks rather than exposing them. | +| `moov`/`mvhd`, `trak`/`tkhd`, and `mdia`/`mdhd`/`hdlr` | Select an enabled, in-movie `pict` master image-sequence track; retain its displayed dimensions, media time scale, media duration, and movie-time-scale track duration. Require unity movie and track matrices because arbitrary movie-canvas composition is outside image-format scope; use `clap`/`irot`/`imir` for supported image presentation. Ignore unrelated tracks rather than exposing them. | | `minf`/`dinf`/`dref` and `stbl` | Accept only self-contained sample data references and own the bounded sample-table state for one selected image sequence plus an optional linked auxiliary-alpha sequence. No reusable data-reference, media-information, or sample-table API is created. | | `stsd` and one supported visual sample entry | Require exactly one `av01` entry for AVIF or one non-layered `hvc1` entry for HEIC. Retain only its dimensions, codec configuration (`av1C` or `hvcC`), image presentation/color/HDR properties, and mandatory version-zero `ccst` coding constraints. Reject an unsupported essential configuration rather than treating it as generic video. | | `stsc`, `stco`/`co64`, and `stsz`/`stz2` | Resolve each declared image sample directly to a validated file offset and length. Expand run tables once into a compact frame-owned descriptor array bounded by `DecoderOptions.MaxFrames`; never buffer the movie or complete `mdat`. | diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs index 5f9d94a6a..d6371ec3a 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs @@ -264,7 +264,6 @@ internal sealed class HeifSequenceParser Id = identity.Id, Width = identity.Width, Height = identity.Height, - Matrix = identity.Matrix, HandlerType = identity.HandlerType, TrackDuration = identity.TrackDuration, AuxiliaryForTrackId = identity.AuxiliaryForTrackId, @@ -292,11 +291,6 @@ internal sealed class HeifSequenceParser stream.Position = media.Offset; this.ParseMedia(stream, media.Length, track, scratch); - if (track.Matrix.HasPerspective) - { - throw new InvalidImageContentException("The HEIF image-sequence track uses an unsupported perspective matrix."); - } - if (track.TotalSampleCount == 0 || track.Samples.Length == 0) { throw new InvalidImageContentException("The HEIF image-sequence track contains no retained image samples."); @@ -306,7 +300,7 @@ internal sealed class HeifSequenceParser } /// - /// Parses the movie time scale required to interpret track edit durations. + /// Parses the movie time scale and validates that no general movie-canvas transformation is required. /// /// The stream positioned at the movie-header payload. /// The validated movie-header payload length. @@ -318,8 +312,8 @@ internal sealed class HeifSequenceParser byte version = prefix[0]; int requiredLength = version switch { - 0 => 20, - 1 => 32, + 0 => 100, + 1 => 112, _ => throw new InvalidImageContentException($"The movie header has unsupported version {version}.") }; @@ -331,6 +325,13 @@ internal sealed class HeifSequenceParser throw new InvalidImageContentException("The movie header has a zero time scale."); } + int matrixOffset = version == 0 ? 36 : 48; + HeifTrackMatrix matrix = HeifTrackMatrix.Parse(prefix.Slice(matrixOffset, 36)); + if (!matrix.IsIdentity) + { + throw new NotSupportedException("The HEIF image sequence requires an unsupported movie presentation matrix."); + } + return timescale; } @@ -380,18 +381,13 @@ internal sealed class HeifSequenceParser throw new InvalidImageContentException("A HEIF image-sequence track has zero dimensions."); } - HeifTrackMatrix matrix = new( - BinaryPrimitives.ReadInt32BigEndian(prefix[matrixOffset..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 4)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 8)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 12)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 16)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 20)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 24)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 28)..]), - BinaryPrimitives.ReadInt32BigEndian(prefix[(matrixOffset + 32)..])); + HeifTrackMatrix matrix = HeifTrackMatrix.Parse(prefix.Slice(matrixOffset, 36)); + if (!matrix.IsIdentity) + { + throw new NotSupportedException("The HEIF image-sequence track requires an unsupported movie presentation matrix."); + } - return new TrackIdentity(id, (flags & 3) == 3, width, height, duration, matrix); + return new TrackIdentity(id, (flags & 3) == 3, width, height, duration); } /// @@ -2371,15 +2367,13 @@ internal sealed class HeifSequenceParser /// The displayed track width. /// The displayed track height. /// The track duration in movie-time-scale units. - /// The track presentation matrix. - public TrackIdentity(uint id, bool isEnabledInMovie, int width, int height, ulong trackDuration, HeifTrackMatrix matrix) + public TrackIdentity(uint id, bool isEnabledInMovie, int width, int height, ulong trackDuration) { this.Id = id; this.IsEnabledInMovie = isEnabledInMovie; this.Width = width; this.Height = height; this.TrackDuration = trackDuration; - this.Matrix = matrix; this.HandlerType = default; this.AuxiliaryForTrackId = 0; this.PremultipliedByTrackId = 0; @@ -2410,11 +2404,6 @@ internal sealed class HeifSequenceParser /// public ulong TrackDuration { get; } - /// - /// Gets the track presentation matrix. - /// - public HeifTrackMatrix Matrix { get; } - /// /// Gets or sets the media handler type. /// diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs index c2e932b3a..311139f8a 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs @@ -45,11 +45,6 @@ internal sealed class HeifSequenceTrack /// public int CodedHeight { get; set; } - /// - /// Gets or sets the track transformation matrix. - /// - public HeifTrackMatrix Matrix { get; set; } - /// /// Gets or sets the media time scale in units per second. /// diff --git a/src/ImageSharp/Formats/Heif/HeifTrackMatrix.cs b/src/ImageSharp/Formats/Heif/HeifTrackMatrix.cs index c6f151ffa..989b4803f 100644 --- a/src/ImageSharp/Formats/Heif/HeifTrackMatrix.cs +++ b/src/ImageSharp/Formats/Heif/HeifTrackMatrix.cs @@ -1,10 +1,12 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Buffers.Binary; + namespace SixLabors.ImageSharp.Formats.Heif; /// -/// Contains the fixed-point transformation matrix of a HEIF image-sequence track. +/// Contains a fixed-point movie or track transformation matrix from a HEIF image sequence. /// internal readonly struct HeifTrackMatrix { @@ -79,7 +81,25 @@ internal readonly struct HeifTrackMatrix public int W { get; } /// - /// Gets a value indicating whether the matrix contains unsupported perspective projection. + /// Gets a value indicating whether the matrix leaves the image coordinate system unchanged. + /// + public bool IsIdentity => this.A == 0x00010000 && this.B == 0 && this.U == 0 && this.C == 0 && this.D == 0x00010000 && + this.V == 0 && this.X == 0 && this.Y == 0 && this.W == 0x40000000; + + /// + /// Reads the nine fixed-point coefficients from a matrix payload in file byte order. /// - public bool HasPerspective => this.U != 0 || this.V != 0 || this.W != 0x40000000; + /// The 36-byte matrix payload. + /// The decoded transformation matrix. + public static HeifTrackMatrix Parse(ReadOnlySpan data) + => new( + BinaryPrimitives.ReadInt32BigEndian(data), + BinaryPrimitives.ReadInt32BigEndian(data[4..]), + BinaryPrimitives.ReadInt32BigEndian(data[8..]), + BinaryPrimitives.ReadInt32BigEndian(data[12..]), + BinaryPrimitives.ReadInt32BigEndian(data[16..]), + BinaryPrimitives.ReadInt32BigEndian(data[20..]), + BinaryPrimitives.ReadInt32BigEndian(data[24..]), + BinaryPrimitives.ReadInt32BigEndian(data[28..]), + BinaryPrimitives.ReadInt32BigEndian(data[32..])); } diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs index e92d760aa..0c996d434 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs @@ -184,6 +184,23 @@ public class HeifSequenceParserTests Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); } + [Theory] + [InlineData(true, false)] + [InlineData(false, true)] + public void ParseRejectsNonIdentityMoviePresentationMatrix(bool nonIdentityMovieMatrix, bool nonIdentityTrackMatrix) + { + byte[] data = CreateSequenceFile( + 1024, + nonIdentityMovieMatrix: nonIdentityMovieMatrix, + nonIdentityTrackMatrix: nonIdentityTrackMatrix); + + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2); + stream.Position = 8; + + Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); + } + [Fact] public void ParseMarksHiddenHevcSamples() { @@ -429,7 +446,9 @@ public class HeifSequenceParserTests byte[] av1Configuration = null, int? sampleSize = null, bool allSamplesSync = false, - uint premultipliedByTrackId = 0) + uint premultipliedByTrackId = 0, + bool nonIdentityMovieMatrix = false, + bool nonIdentityTrackMatrix = false) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.UTF8, true); @@ -441,11 +460,17 @@ public class HeifSequenceParserTests WriteUInt32(writer, 0); WriteUInt32(writer, 1000); WriteUInt32(writer, 600); - WriteZeros(writer, 80); + WriteUInt32(writer, 0x00010000); + WriteUInt16(writer, 0x0100); + WriteUInt16(writer, 0); + WriteZeros(writer, 8); + WritePresentationMatrix(writer, nonIdentityMovieMatrix); + WriteZeros(writer, 24); + WriteUInt32(writer, 3); EndBox(writer, movieHeader); long track = BeginBox(writer, Heif4CharCode.Trak); - WriteTrackHeader(writer, width, height); + WriteTrackHeader(writer, width, height, 1, nonIdentityTrackMatrix); if (premultipliedByTrackId != 0) { WriteTrackReference(writer, Heif4CharCode.Prem, premultipliedByTrackId); @@ -625,7 +650,7 @@ public class HeifSequenceParserTests return data; } - private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1) + private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1, bool nonIdentityMatrix = false) { long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd); WriteFullBoxHeader(writer, 0, 3); @@ -635,7 +660,15 @@ public class HeifSequenceParserTests WriteUInt32(writer, 0); WriteUInt32(writer, 600); WriteZeros(writer, 16); - WriteUInt32(writer, 0x00010000); + WritePresentationMatrix(writer, nonIdentityMatrix); + WriteUInt32(writer, (uint)width << 16); + WriteUInt32(writer, (uint)height << 16); + EndBox(writer, trackHeader); + } + + private static void WritePresentationMatrix(BinaryWriter writer, bool nonIdentityMatrix) + { + WriteUInt32(writer, nonIdentityMatrix ? 0x00020000U : 0x00010000U); WriteUInt32(writer, 0); WriteUInt32(writer, 0); WriteUInt32(writer, 0); @@ -644,9 +677,6 @@ public class HeifSequenceParserTests WriteUInt32(writer, 0); WriteUInt32(writer, 0); WriteUInt32(writer, 0x40000000); - WriteUInt32(writer, (uint)width << 16); - WriteUInt32(writer, (uint)height << 16); - EndBox(writer, trackHeader); } private static void WriteTrackReference(BinaryWriter writer, Heif4CharCode referenceType, uint trackId)