diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md
index 019bb5cfb..02cd6bd9e 100644
--- a/HEIF_IMPLEMENTATION_PLAN.md
+++ b/HEIF_IMPLEMENTATION_PLAN.md
@@ -65,7 +65,8 @@ Checkboxes may be marked complete only when the implementation and the verificat
- [x] Recognize supported `avis`, `hevc`, and `hevx` sequence brands while continuing to reject layered HEVC and JPEG sequence brands.
- [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state.
- [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers.
- - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, exact alpha-track time matching, track-matrix presentation, and independent vectors.
+ - [x] Match auxiliary alpha samples by exact decode duration, visibility, and presentation time, and validate premultiplication track identity.
+ - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, track-matrix presentation, and independent sequence vectors.
- [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames.
- [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model.
- [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences.
diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs
index 2506def28..5f9d94a6a 100644
--- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs
+++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs
@@ -152,7 +152,18 @@ internal sealed class HeifSequenceParser
throw new InvalidImageContentException("The selected HEIF picture track could not be parsed.");
}
- ValidateAlphaTrack(colorTrack, alphaTrack);
+ try
+ {
+ ValidateAlphaTrack(colorTrack, alphaTrack);
+ }
+ catch (Exception ex) when (this.ShouldIgnoreImageDataSegmentError(ex))
+ {
+ // Alpha is optional image data. IgnoreImageData permits a malformed auxiliary sequence to be omitted while
+ // retaining the independently decodable color presentation.
+ alphaTrack = null;
+ colorTrack.IsPremultiplied = false;
+ }
+
return new HeifSequence(colorTrack, alphaTrack, movieTimescale);
}
@@ -257,7 +268,7 @@ internal sealed class HeifSequenceParser
HandlerType = identity.HandlerType,
TrackDuration = identity.TrackDuration,
AuxiliaryForTrackId = identity.AuxiliaryForTrackId,
- IsPremultiplied = identity.PremultipliedByTrackId != 0
+ PremultipliedByTrackId = identity.PremultipliedByTrackId
};
if (edit.IsPresent)
@@ -2167,6 +2178,16 @@ internal sealed class HeifSequenceParser
/// The optional linked alpha track.
private static void ValidateAlphaTrack(HeifSequenceTrack colorTrack, HeifSequenceTrack? alphaTrack)
{
+ if (colorTrack.PremultipliedByTrackId != 0)
+ {
+ if (alphaTrack is null || colorTrack.PremultipliedByTrackId != alphaTrack.Id)
+ {
+ throw new InvalidImageContentException("The color image-sequence track references an unrelated premultiplication track.");
+ }
+
+ colorTrack.IsPremultiplied = true;
+ }
+
if (alphaTrack is null)
{
return;
@@ -2179,12 +2200,41 @@ internal sealed class HeifSequenceParser
for (int i = 0; i < colorTrack.Samples.Length; i++)
{
- ulong colorDuration = (ulong)colorTrack.Samples[i].Duration * alphaTrack.MediaTimescale;
- ulong alphaDuration = (ulong)alphaTrack.Samples[i].Duration * colorTrack.MediaTimescale;
+ HeifSequenceSample colorSample = colorTrack.Samples[i];
+ HeifSequenceSample alphaSample = alphaTrack.Samples[i];
+ ulong colorDuration = (ulong)colorSample.Duration * alphaTrack.MediaTimescale;
+ ulong alphaDuration = (ulong)alphaSample.Duration * colorTrack.MediaTimescale;
if (colorDuration != alphaDuration)
{
throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation durations.");
}
+
+ if (colorSample.IsHidden != alphaSample.IsHidden)
+ {
+ throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation visibility.");
+ }
+
+ if (!colorSample.IsHidden)
+ {
+ // Cross-multiplication retains exact signed presentation times without floating-point rounding or overflow.
+ Int128 colorCompositionTime = (Int128)colorSample.CompositionTime * alphaTrack.MediaTimescale;
+ Int128 alphaCompositionTime = (Int128)alphaSample.CompositionTime * colorTrack.MediaTimescale;
+ if (colorCompositionTime != alphaCompositionTime)
+ {
+ throw new InvalidImageContentException("The alpha and color image-sequence samples have different presentation times.");
+ }
+ }
+ }
+
+ bool alphaHasPresentationProperties = alphaTrack.CleanAperture is not null || alphaTrack.RotationAngle is not null || alphaTrack.MirrorAxis is not null;
+ if (alphaHasPresentationProperties &&
+ (!Nullable.Equals(colorTrack.CleanAperture, alphaTrack.CleanAperture) ||
+ colorTrack.RotationAngle != alphaTrack.RotationAngle ||
+ colorTrack.MirrorAxis != alphaTrack.MirrorAxis))
+ {
+ // libavif accepts legacy alpha tracks with no transform properties, but requires exact equality when any
+ // alpha transform is declared because composition occurs before the shared color-track presentation step.
+ throw new NotSupportedException("The alpha and color image-sequence tracks use different presentation transforms.");
}
}
diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs
index a8748f493..c2e932b3a 100644
--- a/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs
+++ b/src/ImageSharp/Formats/Heif/HeifSequenceTrack.cs
@@ -165,6 +165,11 @@ internal sealed class HeifSequenceTrack
///
public bool IsAlpha { get; set; }
+ ///
+ /// Gets or sets the identifier of the alpha track that premultiplies this color track, or zero when color is unassociated.
+ ///
+ public uint PremultipliedByTrackId { get; set; }
+
///
/// Gets or sets a value indicating whether the color track is premultiplied by this auxiliary alpha track.
///
diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs
index 435222e1a..4973becf6 100644
--- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs
@@ -176,6 +176,74 @@ public class HeifSequenceParserTests
Assert.Equal(100, sequence.ColorTrack.Samples[1].CompositionTime);
}
+ [Theory]
+ [InlineData(false, false)]
+ [InlineData(true, false)]
+ [InlineData(true, true)]
+ public void ParseMatchesAlphaTrackAndPremultiplicationByTrackId(bool colorTransforms, bool alphaTransforms)
+ {
+ byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 2, colorTransforms, alphaTransforms);
+ using MemoryStream stream = new(data, false);
+ HeifSequenceParser parser = CreateParser(2);
+ stream.Position = 8;
+
+ HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data));
+
+ Assert.NotNull(sequence.AlphaTrack);
+ Assert.Equal(2U, sequence.AlphaTrack.Id);
+ Assert.True(sequence.AlphaTrack.IsAlpha);
+ Assert.True(sequence.ColorTrack.IsPremultiplied);
+ }
+
+ [Theory]
+ [InlineData(SegmentIntegrityHandling.Strict)]
+ [InlineData(SegmentIntegrityHandling.IgnoreAncillary)]
+ public void ParseRejectsAlphaTrackWithDifferentDecodeTiming(SegmentIntegrityHandling handling)
+ {
+ byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0);
+ using MemoryStream stream = new(data, false);
+ HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: handling);
+ stream.Position = 8;
+
+ Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+ }
+
+ [Fact]
+ public void ParseDropsAlphaTrackWithDifferentDecodeTimingWhenImageDataErrorsAreIgnored()
+ {
+ byte[] data = CreateSequenceFileWithAlpha(1024, 2000, 0);
+ using MemoryStream stream = new(data, false);
+ HeifSequenceParser parser = CreateParser(2, segmentIntegrityHandling: SegmentIntegrityHandling.IgnoreImageData);
+ stream.Position = 8;
+
+ HeifSequence sequence = parser.Parse(stream, GetMoviePayloadLength(data));
+
+ Assert.Null(sequence.AlphaTrack);
+ Assert.False(sequence.ColorTrack.IsPremultiplied);
+ }
+
+ [Fact]
+ public void ParseRejectsPremultiplicationReferenceToUnrelatedTrack()
+ {
+ byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 3);
+ using MemoryStream stream = new(data, false);
+ HeifSequenceParser parser = CreateParser(2);
+ stream.Position = 8;
+
+ Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+ }
+
+ [Fact]
+ public void ParseRejectsMismatchedAlphaPresentationTransforms()
+ {
+ byte[] data = CreateSequenceFileWithAlpha(1024, 1000, 0, false, true);
+ using MemoryStream stream = new(data, false);
+ HeifSequenceParser parser = CreateParser(2);
+ stream.Position = 8;
+
+ Assert.Throws(() => parser.Parse(stream, GetMoviePayloadLength(data)));
+ }
+
[Fact]
public void ParseRejectsCompositionOffsetsForAv1()
{
@@ -334,7 +402,8 @@ public class HeifSequenceParserTests
int height = 240,
byte[] av1Configuration = null,
int? sampleSize = null,
- bool allSamplesSync = false)
+ bool allSamplesSync = false,
+ uint premultipliedByTrackId = 0)
{
using MemoryStream stream = new();
using BinaryWriter writer = new(stream, Encoding.UTF8, true);
@@ -351,6 +420,11 @@ public class HeifSequenceParserTests
long track = BeginBox(writer, Heif4CharCode.Trak);
WriteTrackHeader(writer, width, height);
+ if (premultipliedByTrackId != 0)
+ {
+ WriteTrackReference(writer, Heif4CharCode.Prem, premultipliedByTrackId);
+ }
+
WriteEditList(writer);
if (trackMetadata)
{
@@ -396,6 +470,44 @@ public class HeifSequenceParserTests
return file;
}
+ private static byte[] CreateSequenceFileWithAlpha(
+ uint chunkOffset,
+ uint alphaTimescale,
+ uint premultipliedByTrackId,
+ bool colorTransforms = false,
+ bool alphaTransforms = false)
+ {
+ byte[] colorFile = CreateSequenceFile(
+ chunkOffset,
+ trackProperties: colorTransforms,
+ premultipliedByTrackId: premultipliedByTrackId);
+
+ int movieLength = (int)BinaryPrimitives.ReadUInt32BigEndian(colorFile);
+ using MemoryStream stream = new();
+ using BinaryWriter writer = new(stream, Encoding.UTF8, true);
+ long track = BeginBox(writer, Heif4CharCode.Trak);
+ WriteTrackHeader(writer, 320, 240, 2);
+ WriteTrackReference(writer, Heif4CharCode.Auxl, 1);
+
+ long media = BeginBox(writer, Heif4CharCode.Mdia);
+ WriteMediaHeader(writer, alphaTimescale);
+ WriteHandler(writer, Heif4CharCode.Auxv);
+
+ long mediaInformation = BeginBox(writer, Heif4CharCode.Minf);
+ WriteDataInformation(writer);
+ WriteSampleTable(writer, chunkOffset, false, false, false, 1, alphaTransforms, false, 320, 240, null, null, false, true);
+ EndBox(writer, mediaInformation);
+ EndBox(writer, media);
+ EndBox(writer, track);
+
+ byte[] alphaTrack = stream.ToArray();
+ byte[] file = new byte[2048];
+ colorFile.AsSpan(0, movieLength).CopyTo(file);
+ alphaTrack.CopyTo(file, movieLength);
+ BinaryPrimitives.WriteUInt32BigEndian(file, (uint)(movieLength + alphaTrack.Length));
+ return file;
+ }
+
private static byte[] CreateSequenceContainer(bool trackProperties, bool trackMetadata)
{
byte[] movie = CreateSequenceFile(
@@ -441,13 +553,13 @@ public class HeifSequenceParserTests
return data;
}
- private static void WriteTrackHeader(BinaryWriter writer, int width, int height)
+ private static void WriteTrackHeader(BinaryWriter writer, int width, int height, uint trackId = 1)
{
long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd);
WriteFullBoxHeader(writer, 0, 3);
WriteUInt32(writer, 0);
WriteUInt32(writer, 0);
- WriteUInt32(writer, 1);
+ WriteUInt32(writer, trackId);
WriteUInt32(writer, 0);
WriteUInt32(writer, 600);
WriteZeros(writer, 16);
@@ -465,6 +577,15 @@ public class HeifSequenceParserTests
EndBox(writer, trackHeader);
}
+ private static void WriteTrackReference(BinaryWriter writer, Heif4CharCode referenceType, uint trackId)
+ {
+ long references = BeginBox(writer, Heif4CharCode.Tref);
+ long reference = BeginBox(writer, referenceType);
+ WriteUInt32(writer, trackId);
+ EndBox(writer, reference);
+ EndBox(writer, references);
+ }
+
private static void WriteEditList(BinaryWriter writer)
{
long edit = BeginBox(writer, Heif4CharCode.Edts);
@@ -479,13 +600,13 @@ public class HeifSequenceParserTests
EndBox(writer, edit);
}
- private static void WriteMediaHeader(BinaryWriter writer)
+ private static void WriteMediaHeader(BinaryWriter writer, uint timescale = 1000)
{
long mediaHeader = BeginBox(writer, Heif4CharCode.Mdhd);
WriteFullBoxHeader(writer, 0, 0);
WriteUInt32(writer, 0);
WriteUInt32(writer, 0);
- WriteUInt32(writer, 1000);
+ WriteUInt32(writer, timescale);
WriteUInt32(writer, 200);
WriteUInt16(writer, 21956);
WriteUInt16(writer, 0);
@@ -599,10 +720,11 @@ public class HeifSequenceParserTests
int height,
byte[] av1Configuration,
int? sampleSize,
- bool allSamplesSync)
+ bool allSamplesSync,
+ bool alpha = false)
{
long sampleTable = BeginBox(writer, Heif4CharCode.Stbl);
- WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync);
+ WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync, alpha);
long timing = BeginBox(writer, Heif4CharCode.Stts);
WriteFullBoxHeader(writer, 0, 0);
@@ -708,7 +830,8 @@ public class HeifSequenceParserTests
int width,
int height,
byte[] av1Configuration,
- bool allSamplesSync)
+ bool allSamplesSync,
+ bool alpha)
{
long description = BeginBox(writer, Heif4CharCode.Stsd);
WriteFullBoxHeader(writer, 0, 0);
@@ -738,6 +861,15 @@ public class HeifSequenceParserTests
EndBox(writer, configuration);
}
+ if (alpha)
+ {
+ long auxiliaryType = BeginBox(writer, Heif4CharCode.Auxi);
+ WriteFullBoxHeader(writer, 0, 0);
+ writer.Write(Encoding.UTF8.GetBytes(HeifConstants.AlphaAuxiliaryType));
+ writer.Write((byte)0);
+ EndBox(writer, auxiliaryType);
+ }
+
if (trackProperties)
{
WriteTrackImageProperties(writer, invalidRotation, width, height);