Validate HEIF sequence alpha tracks

1 week ago · f2d7bf212c
4 changed files with 201 additions and 13 deletions
--- a/HEIF_IMPLEMENTATION_PLAN.md
+++ b/HEIF_IMPLEMENTATION_PLAN.md
@ -65,7 +65,8 @@ Checkboxes may be marked complete only when the implementation and the verificat
      - [x] Recognize supported `avis`, `hevc`, and `hevx` sequence brands while continuing to reject layered HEVC and JPEG sequence brands.
      - [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state.
      - [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers.
-      - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, exact alpha-track time matching, track-matrix presentation, and independent vectors.
+      - [x] Match auxiliary alpha samples by exact decode duration, visibility, and presentation time, and validate premultiplication track identity.
+      - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, track-matrix presentation, and independent sequence vectors.
    - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames.
  - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model.
  - [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences.
--- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs
+++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs
@ -152,7 +152,18 @@ internal sealed class HeifSequenceParser
            throw new InvalidImageContentException("The selected HEIF picture track could not be parsed.");
        }

-        ValidateAlphaTrack(colorTrack, alphaTrack);
+        try
+        {
+            ValidateAlphaTrack(colorTrack, alphaTrack);
+        }
+        catch (Exception ex) when (this.ShouldIgnoreImageDataSegmentError(ex))
+        {
+            // Alpha is optional image data. IgnoreImageData permits a malformed auxiliary sequence to be omitted while
+            // retaining the independently decodable color presentation.
+            alphaTrack = null;
+            colorTrack.IsPremultiplied = false;
+        }
+
        return new HeifSequence(colorTrack, alphaTrack, movieTimescale);
    }

@ -257,7 +268,7 @@ internal sealed class HeifSequenceParser
            HandlerType = identity.HandlerType,
            TrackDuration = identity.TrackDuration,
            AuxiliaryForTrackId = identity.AuxiliaryForTrackId,
-            IsPremultiplied = identity.PremultipliedByTrackId != 0
+            PremultipliedByTrackId = identity.PremultipliedByTrackId
        };

        if (edit.IsPresent)
@ -2167,6 +2178,16 @@ internal sealed class HeifSequenceParser
    /// <param name="alphaTrack">The optional linked alpha track.</param>
    private static void ValidateAlphaTrack(HeifSequenceTrack colorTrack, HeifSequenceTrack? alphaTrack)
    {
+        if (colorTrack.PremultipliedByTrackId != 0)
+        {
+            if (alphaTrack is null || colorTrack.PremultipliedByTrackId != alphaTrack.Id)
+            {
+                throw new InvalidImageContentException("The color image-sequence track references an unrelated premultiplication track.");
+            }
+
+            colorTrack.IsPremultiplied = true;
+        }
+
        if (alphaTrack is null)
        {
            return;
@ -2179,12 +2200,41 @@ internal sealed class HeifSequenceParser

        for (int i = 0; i < colorTrack.Samples.Length; i++)
        {
-            ulong colorDuration = (ulong)colorTrack.Samples[i].Duration * alphaTrack.MediaTimescale;
-            ulong alphaDuration = (ulong)alphaTrack.Samples[i].Duration * colorTrack.MediaTimescale;
+            HeifSequenceSample colorSample = colorTrack.Samples[i];
+            HeifSequenceSample alphaSample = alphaTrack.Samples[i];
+            ulong colorDuration = (ulong)colorSample.Duration * alphaTrack.MediaTimescale;
+            ulong alphaDuration = (ulong)alphaSample.Duration * colorTrack.MediaTimescale;
            if (colorDuration != alphaDuration)
            {
                throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation durations.");
            }
+
+            if (colorSample.IsHidden != alphaSample.IsHidden)
+            {
+                throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation visibility.");
+            }
+
+            if (!colorSample.IsHidden)
+            {
+                // Cross-multiplication retains exact signed presentation times without floating-point rounding or overflow.
+                Int128 colorCompositionTime = (Int128)colorSample.CompositionTime * alphaTrack.MediaTimescale;
+                Int128 alphaCompositionTime = (Int128)alphaSample.CompositionTime * colorTrack.MediaTimescale;
+                if (colorCompositionTime != alphaCompositionTime)
+                {
+                    throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation times.");
+                }
+            }
+        }
+
+        bool alphaHasPresentationProperties = alphaTrack.CleanAperture is not null || alphaTrack.RotationAngle is not null || alphaTrack.MirrorAxis is not null;
+        if (alphaHasPresentationProperties &&
+            (!Nullable.Equals(colorTrack.CleanAperture, alphaTrack.CleanAperture) ||
+             colorTrack.RotationAngle != alphaTrack.RotationAngle ||
+             colorTrack.MirrorAxis != alphaTrack.MirrorAxis))
+        {
+            // libavif accepts legacy alpha tracks with no transform properties, but requires exact equality when any
+            // alpha transform is declared because composition occurs before the shared color-track presentation step.
+            throw new NotSupportedException("The alpha and color image-sequence tracks use different presentation transforms.");
        }
    }

--- a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs
+++ b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs
@ -165,6 +165,11 @@ internal sealed class HeifSequenceTrack
    /// </summary>
    public bool IsAlpha { get; set; }

+    /// <summary>
+    /// Gets or sets the identifier of the alpha track that premultiplies this color track, or zero when color is unassociated.
+    /// </summary>
+    public uint PremultipliedByTrackId { get; set; }
+
    /// <summary>
    /// Gets or sets a value indicating whether the color track is premultiplied by this auxiliary alpha track.
    /// </summary>
--- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs
@ -176,6 +176,74 @@ public class HeifSequenceParserTests
        Assert.Equal(100, sequence.ColorTrack.Samples[1].CompositionTime);
    }

+    [Theory]
+    [InlineData(false, false)]
+    [InlineData(true, false)]
+    [InlineData(true, true)]
+    public void ParseMatchesAlphaTrackAndPremultiplicationByTrackId(bool colorTransforms, bool alphaTransforms)
+    {
+        byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 2, colorTransforms, alphaTransforms);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2);
+        stream.Position = 8;
+
+        HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data));
+
+        Assert.NotNull(sequence.AlphaTrack);
+        Assert.Equal(2U, sequence.AlphaTrack.Id);
+        Assert.True(sequence.AlphaTrack.IsAlpha);
+        Assert.True(sequence.ColorTrack.IsPremultiplied);
+    }
+
+    [Theory]
+    [InlineData(SegmentIntegrityHandling.Strict)]
+    [InlineData(SegmentIntegrityHandling.IgnoreAncillary)]
+    public void ParseRejectsAlphaTrackWithDifferentDecodeTiming(SegmentIntegrityHandling handling)
+    {
+        byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: handling);
+        stream.Position = 8;
+
+        Assert.Throws<InvalidImageContentException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+    }
+
+    [Fact]
+    public void ParseDropsAlphaTrackWithDifferentDecodeTimingWhenImageDataErrorsAreIgnored()
+    {
+        byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: SegmentIntegrityHandling.IgnoreImageData);
+        stream.Position = 8;
+
+        HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data));
+
+        Assert.Null(sequence.AlphaTrack);
+        Assert.False(sequence.ColorTrack.IsPremultiplied);
+    }
+
+    [Fact]
+    public void ParseRejectsPremultiplicationReferenceToUnrelatedTrack()
+    {
+        byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 3);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2);
+        stream.Position = 8;
+
+        Assert.Throws<InvalidImageContentException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+    }
+
+    [Fact]
+    public void ParseRejectsMismatchedAlphaPresentationTransforms()
+    {
+        byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 0, false, true);
+        using MemoryStream stream = new(data, false);
+        HeifSequenceParser parser = CreateParser(2);
+        stream.Position = 8;
+
+        Assert.Throws<NotSupportedException>(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+    }
+
    [Fact]
    public void ParseRejectsCompositionOffsetsForAv1()
    {
@ -334,7 +402,8 @@ public class HeifSequenceParserTests
        int height = 240,
        byte[] av1Configuration = null,
        int? sampleSize = null,
-        bool allSamplesSync = false)
+        bool allSamplesSync = false,
+        uint premultipliedByTrackId = 0)
    {
        using MemoryStream stream = new();
        using BinaryWriter writer = new(stream, Encoding.UTF8, true);
@ -351,6 +420,11 @@ public class HeifSequenceParserTests

        long track = BeginBox(writer, Heif4CharCode.Trak);
        WriteTrackHeader(writer, width, height);
+        if (premultipliedByTrackId != 0)
+        {
+            WriteTrackReference(writer, Heif4CharCode.Prem, premultipliedByTrackId);
+        }
+
        WriteEditList(writer);
        if (trackMetadata)
        {
@ -396,6 +470,44 @@ public class HeifSequenceParserTests
        return file;
    }

+    private static byte[] CreateSequenceFileWithAlpha(
+        uint chunkOffset,
+        uint alphaTimescale,
+        uint premultipliedByTrackId,
+        bool colorTransforms = false,
+        bool alphaTransforms = false)
+    {
+        byte[] colorFile = CreateSequenceFile(
+            chunkOffset,
+            trackProperties: colorTransforms,
+            premultipliedByTrackId: premultipliedByTrackId);
+
+        int movieLength = (int)BinaryPrimitives.ReadUInt32BigEndian(colorFile);
+        using MemoryStream stream = new();
+        using BinaryWriter writer = new(stream, Encoding.UTF8, true);
+        long track = BeginBox(writer, Heif4CharCode.Trak);
+        WriteTrackHeader(writer, 320, 240, 2);
+        WriteTrackReference(writer, Heif4CharCode.Auxl, 1);
+
+        long media = BeginBox(writer, Heif4CharCode.Mdia);
+        WriteMediaHeader(writer, alphaTimescale);
+        WriteHandler(writer, Heif4CharCode.Auxv);
+
+        long mediaInformation = BeginBox(writer, Heif4CharCode.Minf);
+        WriteDataInformation(writer);
+        WriteSampleTable(writer, chunkOffset, false, false, false, 1, alphaTransforms, false, 320, 240, null, null, false, true);
+        EndBox(writer, mediaInformation);
+        EndBox(writer, media);
+        EndBox(writer, track);
+
+        byte[] alphaTrack = stream.ToArray();
+        byte[] file = new byte[2048];
+        colorFile.AsSpan(0, movieLength).CopyTo(file);
+        alphaTrack.CopyTo(file, movieLength);
+        BinaryPrimitives.WriteUInt32BigEndian(file, (uint)(movieLength + alphaTrack.Length));
+        return file;
+    }
+
    private static byte[] CreateSequenceContainer(bool trackProperties, bool trackMetadata)
    {
        byte[] movie = CreateSequenceFile(
@ -441,13 +553,13 @@ public class HeifSequenceParserTests
        return data;
    }

-    private static void WriteTrackHeader(BinaryWriter writer, int width, int height)
+    private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1)
    {
        long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd);
        WriteFullBoxHeader(writer, 0, 3);
        WriteUInt32(writer, 0);
        WriteUInt32(writer, 0);
-        WriteUInt32(writer, 1);
+        WriteUInt32(writer, trackId);
        WriteUInt32(writer, 0);
        WriteUInt32(writer, 600);
        WriteZeros(writer, 16);
@ -465,6 +577,15 @@ public class HeifSequenceParserTests
        EndBox(writer, trackHeader);
    }

+    private static void WriteTrackReference(BinaryWriter writer, Heif4CharCode referenceType, uint trackId)
+    {
+        long references = BeginBox(writer, Heif4CharCode.Tref);
+        long reference = BeginBox(writer, referenceType);
+        WriteUInt32(writer, trackId);
+        EndBox(writer, reference);
+        EndBox(writer, references);
+    }
+
    private static void WriteEditList(BinaryWriter writer)
    {
        long edit = BeginBox(writer, Heif4CharCode.Edts);
@ -479,13 +600,13 @@ public class HeifSequenceParserTests
        EndBox(writer, edit);
    }

-    private static void WriteMediaHeader(BinaryWriter writer)
+    private static void WriteMediaHeader(BinaryWriter writer, uint timescale = 1000)
    {
        long mediaHeader = BeginBox(writer, Heif4CharCode.Mdhd);
        WriteFullBoxHeader(writer, 0, 0);
        WriteUInt32(writer, 0);
        WriteUInt32(writer, 0);
-        WriteUInt32(writer, 1000);
+        WriteUInt32(writer, timescale);
        WriteUInt32(writer, 200);
        WriteUInt16(writer, 21956);
        WriteUInt16(writer, 0);
@ -599,10 +720,11 @@ public class HeifSequenceParserTests
        int height,
        byte[] av1Configuration,
        int? sampleSize,
-        bool allSamplesSync)
+        bool allSamplesSync,
+        bool alpha = false)
    {
        long sampleTable = BeginBox(writer, Heif4CharCode.Stbl);
-        WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync);
+        WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync, alpha);

        long timing = BeginBox(writer, Heif4CharCode.Stts);
        WriteFullBoxHeader(writer, 0, 0);
@ -708,7 +830,8 @@ public class HeifSequenceParserTests
        int width,
        int height,
        byte[] av1Configuration,
-        bool allSamplesSync)
+        bool allSamplesSync,
+        bool alpha)
    {
        long description = BeginBox(writer, Heif4CharCode.Stsd);
        WriteFullBoxHeader(writer, 0, 0);
@ -738,6 +861,15 @@ public class HeifSequenceParserTests
            EndBox(writer, configuration);
        }

+        if (alpha)
+        {
+            long auxiliaryType = BeginBox(writer, Heif4CharCode.Auxi);
+            WriteFullBoxHeader(writer, 0, 0);
+            writer.Write(Encoding.UTF8.GetBytes(HeifConstants.AlphaAuxiliaryType));
+            writer.Write((byte)0);
+            EndBox(writer, auxiliaryType);
+        }
+
        if (trackProperties)
        {
            WriteTrackImageProperties(writer, invalidRotation, width, height);