diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 019bb5cfb..02cd6bd9e 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -65,7 +65,8 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Recognize supported `avis`, `hevc`, and `hevx` sequence brands while continuing to reject layered HEVC and JPEG sequence brands. - [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state. - [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers. - - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, exact alpha-track time matching, track-matrix presentation, and independent vectors. + - [x] Match auxiliary alpha samples by exact decode duration, visibility, and presentation time, and validate premultiplication track identity. + - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, track-matrix presentation, and independent sequence vectors. - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames. - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model. - [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences. diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs index 2506def28..5f9d94a6a 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs @@ -152,7 +152,18 @@ internal sealed class HeifSequenceParser throw new InvalidImageContentException("The selected HEIF picture track could not be parsed."); } - ValidateAlphaTrack(colorTrack, alphaTrack); + try + { + ValidateAlphaTrack(colorTrack, alphaTrack); + } + catch (Exception ex) when (this.ShouldIgnoreImageDataSegmentError(ex)) + { + // Alpha is optional image data. IgnoreImageData permits a malformed auxiliary sequence to be omitted while + // retaining the independently decodable color presentation. + alphaTrack = null; + colorTrack.IsPremultiplied = false; + } + return new HeifSequence(colorTrack, alphaTrack, movieTimescale); } @@ -257,7 +268,7 @@ internal sealed class HeifSequenceParser HandlerType = identity.HandlerType, TrackDuration = identity.TrackDuration, AuxiliaryForTrackId = identity.AuxiliaryForTrackId, - IsPremultiplied = identity.PremultipliedByTrackId != 0 + PremultipliedByTrackId = identity.PremultipliedByTrackId }; if (edit.IsPresent) @@ -2167,6 +2178,16 @@ internal sealed class HeifSequenceParser /// The optional linked alpha track. private static void ValidateAlphaTrack(HeifSequenceTrack colorTrack, HeifSequenceTrack? alphaTrack) { + if (colorTrack.PremultipliedByTrackId != 0) + { + if (alphaTrack is null || colorTrack.PremultipliedByTrackId != alphaTrack.Id) + { + throw new InvalidImageContentException("The color image-sequence track references an unrelated premultiplication track."); + } + + colorTrack.IsPremultiplied = true; + } + if (alphaTrack is null) { return; @@ -2179,12 +2200,41 @@ internal sealed class HeifSequenceParser for (int i = 0; i < colorTrack.Samples.Length; i++) { - ulong colorDuration = (ulong)colorTrack.Samples[i].Duration * alphaTrack.MediaTimescale; - ulong alphaDuration = (ulong)alphaTrack.Samples[i].Duration * colorTrack.MediaTimescale; + HeifSequenceSample colorSample = colorTrack.Samples[i]; + HeifSequenceSample alphaSample = alphaTrack.Samples[i]; + ulong colorDuration = (ulong)colorSample.Duration * alphaTrack.MediaTimescale; + ulong alphaDuration = (ulong)alphaSample.Duration * colorTrack.MediaTimescale; if (colorDuration != alphaDuration) { throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation durations."); } + + if (colorSample.IsHidden != alphaSample.IsHidden) + { + throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation visibility."); + } + + if (!colorSample.IsHidden) + { + // Cross-multiplication retains exact signed presentation times without floating-point rounding or overflow. + Int128 colorCompositionTime = (Int128)colorSample.CompositionTime * alphaTrack.MediaTimescale; + Int128 alphaCompositionTime = (Int128)alphaSample.CompositionTime * colorTrack.MediaTimescale; + if (colorCompositionTime != alphaCompositionTime) + { + throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation times."); + } + } + } + + bool alphaHasPresentationProperties = alphaTrack.CleanAperture is not null || alphaTrack.RotationAngle is not null || alphaTrack.MirrorAxis is not null; + if (alphaHasPresentationProperties && + (!Nullable.Equals(colorTrack.CleanAperture, alphaTrack.CleanAperture) || + colorTrack.RotationAngle != alphaTrack.RotationAngle || + colorTrack.MirrorAxis != alphaTrack.MirrorAxis)) + { + // libavif accepts legacy alpha tracks with no transform properties, but requires exact equality when any + // alpha transform is declared because composition occurs before the shared color-track presentation step. + throw new NotSupportedException("The alpha and color image-sequence tracks use different presentation transforms."); } } diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs index a8748f493..c2e932b3a 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs @@ -165,6 +165,11 @@ internal sealed class HeifSequenceTrack /// public bool IsAlpha { get; set; } + /// + /// Gets or sets the identifier of the alpha track that premultiplies this color track, or zero when color is unassociated. + /// + public uint PremultipliedByTrackId { get; set; } + /// /// Gets or sets a value indicating whether the color track is premultiplied by this auxiliary alpha track. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs index 435222e1a..4973becf6 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs @@ -176,6 +176,74 @@ public class HeifSequenceParserTests Assert.Equal(100, sequence.ColorTrack.Samples[1].CompositionTime); } + [Theory] + [InlineData(false, false)] + [InlineData(true, false)] + [InlineData(true, true)] + public void ParseMatchesAlphaTrackAndPremultiplicationByTrackId(bool colorTransforms, bool alphaTransforms) + { + byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 2, colorTransforms, alphaTransforms); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2); + stream.Position = 8; + + HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data)); + + Assert.NotNull(sequence.AlphaTrack); + Assert.Equal(2U, sequence.AlphaTrack.Id); + Assert.True(sequence.AlphaTrack.IsAlpha); + Assert.True(sequence.ColorTrack.IsPremultiplied); + } + + [Theory] + [InlineData(SegmentIntegrityHandling.Strict)] + [InlineData(SegmentIntegrityHandling.IgnoreAncillary)] + public void ParseRejectsAlphaTrackWithDifferentDecodeTiming(SegmentIntegrityHandling handling) + { + byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: handling); + stream.Position = 8; + + Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); + } + + [Fact] + public void ParseDropsAlphaTrackWithDifferentDecodeTimingWhenImageDataErrorsAreIgnored() + { + byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: SegmentIntegrityHandling.IgnoreImageData); + stream.Position = 8; + + HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data)); + + Assert.Null(sequence.AlphaTrack); + Assert.False(sequence.ColorTrack.IsPremultiplied); + } + + [Fact] + public void ParseRejectsPremultiplicationReferenceToUnrelatedTrack() + { + byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 3); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2); + stream.Position = 8; + + Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); + } + + [Fact] + public void ParseRejectsMismatchedAlphaPresentationTransforms() + { + byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 0, false, true); + using MemoryStream stream = new(data, false); + HeifSequenceParser parser = CreateParser(2); + stream.Position = 8; + + Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data))); + } + [Fact] public void ParseRejectsCompositionOffsetsForAv1() { @@ -334,7 +402,8 @@ public class HeifSequenceParserTests int height = 240, byte[] av1Configuration = null, int? sampleSize = null, - bool allSamplesSync = false) + bool allSamplesSync = false, + uint premultipliedByTrackId = 0) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.UTF8, true); @@ -351,6 +420,11 @@ public class HeifSequenceParserTests long track = BeginBox(writer, Heif4CharCode.Trak); WriteTrackHeader(writer, width, height); + if (premultipliedByTrackId != 0) + { + WriteTrackReference(writer, Heif4CharCode.Prem, premultipliedByTrackId); + } + WriteEditList(writer); if (trackMetadata) { @@ -396,6 +470,44 @@ public class HeifSequenceParserTests return file; } + private static byte[] CreateSequenceFileWithAlpha( + uint chunkOffset, + uint alphaTimescale, + uint premultipliedByTrackId, + bool colorTransforms = false, + bool alphaTransforms = false) + { + byte[] colorFile = CreateSequenceFile( + chunkOffset, + trackProperties: colorTransforms, + premultipliedByTrackId: premultipliedByTrackId); + + int movieLength = (int)BinaryPrimitives.ReadUInt32BigEndian(colorFile); + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream, Encoding.UTF8, true); + long track = BeginBox(writer, Heif4CharCode.Trak); + WriteTrackHeader(writer, 320, 240, 2); + WriteTrackReference(writer, Heif4CharCode.Auxl, 1); + + long media = BeginBox(writer, Heif4CharCode.Mdia); + WriteMediaHeader(writer, alphaTimescale); + WriteHandler(writer, Heif4CharCode.Auxv); + + long mediaInformation = BeginBox(writer, Heif4CharCode.Minf); + WriteDataInformation(writer); + WriteSampleTable(writer, chunkOffset, false, false, false, 1, alphaTransforms, false, 320, 240, null, null, false, true); + EndBox(writer, mediaInformation); + EndBox(writer, media); + EndBox(writer, track); + + byte[] alphaTrack = stream.ToArray(); + byte[] file = new byte[2048]; + colorFile.AsSpan(0, movieLength).CopyTo(file); + alphaTrack.CopyTo(file, movieLength); + BinaryPrimitives.WriteUInt32BigEndian(file, (uint)(movieLength + alphaTrack.Length)); + return file; + } + private static byte[] CreateSequenceContainer(bool trackProperties, bool trackMetadata) { byte[] movie = CreateSequenceFile( @@ -441,13 +553,13 @@ public class HeifSequenceParserTests return data; } - private static void WriteTrackHeader(BinaryWriter writer, int width, int height) + private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1) { long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd); WriteFullBoxHeader(writer, 0, 3); WriteUInt32(writer, 0); WriteUInt32(writer, 0); - WriteUInt32(writer, 1); + WriteUInt32(writer, trackId); WriteUInt32(writer, 0); WriteUInt32(writer, 600); WriteZeros(writer, 16); @@ -465,6 +577,15 @@ public class HeifSequenceParserTests EndBox(writer, trackHeader); } + private static void WriteTrackReference(BinaryWriter writer, Heif4CharCode referenceType, uint trackId) + { + long references = BeginBox(writer, Heif4CharCode.Tref); + long reference = BeginBox(writer, referenceType); + WriteUInt32(writer, trackId); + EndBox(writer, reference); + EndBox(writer, references); + } + private static void WriteEditList(BinaryWriter writer) { long edit = BeginBox(writer, Heif4CharCode.Edts); @@ -479,13 +600,13 @@ public class HeifSequenceParserTests EndBox(writer, edit); } - private static void WriteMediaHeader(BinaryWriter writer) + private static void WriteMediaHeader(BinaryWriter writer, uint timescale = 1000) { long mediaHeader = BeginBox(writer, Heif4CharCode.Mdhd); WriteFullBoxHeader(writer, 0, 0); WriteUInt32(writer, 0); WriteUInt32(writer, 0); - WriteUInt32(writer, 1000); + WriteUInt32(writer, timescale); WriteUInt32(writer, 200); WriteUInt16(writer, 21956); WriteUInt16(writer, 0); @@ -599,10 +720,11 @@ public class HeifSequenceParserTests int height, byte[] av1Configuration, int? sampleSize, - bool allSamplesSync) + bool allSamplesSync, + bool alpha = false) { long sampleTable = BeginBox(writer, Heif4CharCode.Stbl); - WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync); + WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync, alpha); long timing = BeginBox(writer, Heif4CharCode.Stts); WriteFullBoxHeader(writer, 0, 0); @@ -708,7 +830,8 @@ public class HeifSequenceParserTests int width, int height, byte[] av1Configuration, - bool allSamplesSync) + bool allSamplesSync, + bool alpha) { long description = BeginBox(writer, Heif4CharCode.Stsd); WriteFullBoxHeader(writer, 0, 0); @@ -738,6 +861,15 @@ public class HeifSequenceParserTests EndBox(writer, configuration); } + if (alpha) + { + long auxiliaryType = BeginBox(writer, Heif4CharCode.Auxi); + WriteFullBoxHeader(writer, 0, 0); + writer.Write(Encoding.UTF8.GetBytes(HeifConstants.AlphaAuxiliaryType)); + writer.Write((byte)0); + EndBox(writer, auxiliaryType); + } + if (trackProperties) { WriteTrackImageProperties(writer, invalidRotation, width, height);