diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 3df12c1808..019bb5cfb7 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -62,6 +62,10 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Parse bounded sequence metadata items. - Selected image tracks now retain implicitly associated Exif and unencoded `application/rdf+xml` XMP items from bounded `iinf`/`infe` and `iloc` declarations, including file-relative and `idat`-relative extents. Item identifiers and extent descriptors use allocator-owned temporary storage; only the exact retained profile payloads survive parsing. `DecoderOptions.SkipMetadata` bypasses metadata parsing, `SegmentIntegrityHandling` follows the shared ancillary/image-data policy, and structural errors that prevent safe track parsing remain fatal. The focused Release sequence-parser suite passes all 15 tests. - [ ] Connect the parsed sequence index to HEIF detection, Identify, frame decode, alpha matching, and frame metadata without changing still-image source selection. + - [x] Recognize supported `avis`, `hevc`, and `hevx` sequence brands while continuing to reject layered HEVC and JPEG sequence brands. + - [x] Identify bounded sequence dimensions, frame count, timing, repetition, codec precision, color, HDR, pixel aspect ratio, Exif, and XMP state. + - [x] Decode all-sync independently decodable AV1 samples into directly adopted ImageSharp frames without cloning complete pixel buffers. + - [ ] Complete reference-dependent AV1 and HEVC sample reconstruction, exact alpha-track time matching, track-matrix presentation, and independent vectors. - [ ] Write the same bounded movie, track, sample-description, location, dependency, timing, repetition, alpha, and metadata syntax from ImageSharp frames. - [ ] Decode frame dependencies, durations, repetition, frame-local auxiliary images, and frame-local metadata into the existing ImageSharp multi-frame model. - [ ] Encode ImageSharp frames, durations, repetition, frame-local auxiliary images, and frame-local metadata as independently decodable HEIC and AVIF image sequences. diff --git a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs index b808095e5f..4fc74ca9bc 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs @@ -4,6 +4,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.PixelFormats.Utils; @@ -72,6 +73,44 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable CicpProfile? containerColorProfile = null, Av1CodecConfiguration? codecConfiguration = null) where TPixel : unmanaged, IPixel + { + ImageFrame frame = this.DecodeFrame(buffer, containerColorProfile, codecConfiguration, out CicpProfile effectiveColorProfile); + ImageMetadata metadata = new() + { + CicpProfile = effectiveColorProfile + }; + + try + { + return new Image(this.configuration, metadata, [frame]); + } + catch + { + // Ownership transfers only after the image constructor accepts the decoded frame. + frame.Dispose(); + throw; + } + } + + /// + /// Decodes an AV1 elementary-stream payload into one independently owned ImageSharp frame. + /// + /// The destination pixel type. + /// The complete AV1 elementary-stream payload. + /// + /// The container color description that supplies unspecified sequence-header color information. + /// + /// + /// The AV1 codec configuration validated against the coded sequence header. + /// + /// Receives the effective CICP description used for conversion. + /// The decoded frame. Ownership transfers to the caller. + public ImageFrame DecodeFrame( + Span buffer, + CicpProfile? containerColorProfile, + Av1CodecConfiguration? codecConfiguration, + out CicpProfile effectiveColorProfile) + where TPixel : unmanaged, IPixel { Av1BitStreamReader reader = new(buffer); this.obuReader.ReadAll(ref reader, buffer.Length, () => this, false); @@ -135,32 +174,31 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable Av1FrameDecoder frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, frameBuffer); frameDecoder.DecodeFrame(); - Image? resultImage = null; + ImageFrame? resultFrame = null; try { - resultImage = new Image( + resultFrame = new ImageFrame( this.configuration, this.FrameHeader.FrameSize.SuperResolutionUpscaledWidth, - this.FrameHeader.FrameSize.FrameHeight, - null); + this.FrameHeader.FrameSize.FrameHeight); - ImageFrame resultFrame = resultImage.Frames.RootFrame; Av1YuvConverter.ConvertToRgb(this.configuration, frameBuffer, resultFrame); // Preserve the effective CICP description used for conversion, including container values that legally // supplied unspecified bitstream fields. This also exposes bitstream-only color metadata to callers. ObuColorConfig effectiveColorConfig = this.SequenceHeader.ColorConfig; - resultImage.Metadata.CicpProfile = new CicpProfile( + effectiveColorProfile = new CicpProfile( (byte)effectiveColorConfig.ColorPrimaries, (byte)effectiveColorConfig.TransferCharacteristics, (byte)effectiveColorConfig.MatrixCoefficients, effectiveColorConfig.ColorRange); - return resultImage; + resultFrame.Metadata.CicpProfile = effectiveColorProfile.DeepClone(); + return resultFrame; } catch { - resultImage?.Dispose(); + resultFrame?.Dispose(); throw; } } diff --git a/src/ImageSharp/Formats/Heif/HeifConstants.cs b/src/ImageSharp/Formats/Heif/HeifConstants.cs index 4efc49b818..5f5ed075da 100644 --- a/src/ImageSharp/Formats/Heif/HeifConstants.cs +++ b/src/ImageSharp/Formats/Heif/HeifConstants.cs @@ -36,17 +36,17 @@ internal static class HeifConstants public static readonly IEnumerable FileExtensions = new[] { "heic", "heif", "hif", "avif" }; /// - /// Determines whether a file-type box describes a supported still-image container. + /// Determines the supported image presentation declared by a file-type box. /// /// /// The file-type box payload, beginning with the major brand and minor version and followed by compatible brands. /// - /// - /// when the major brand is a supported still-image brand, or when an otherwise unknown - /// major brand declares a supported compatible still-image brand; otherwise, . - /// - public static bool IsSupportedFileType(ReadOnlySpan boxContent) + /// Receives the supported still-image or image-sequence presentation. + /// when the payload declares a supported HEIF image presentation. + public static bool TryGetFileType(ReadOnlySpan boxContent, out HeifFileType fileType) { + fileType = HeifFileType.Unsupported; + // Every brand is a four-character code. The payload must contain the major brand and minor version before // any compatible brands, otherwise accepting a partial trailing code could produce a false detection. if (boxContent.Length < 8 || (boxContent.Length & 3) != 0) @@ -55,28 +55,43 @@ internal static class HeifConstants } Heif4CharCode majorBrand = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(boxContent); - if (IsSequenceBrand(majorBrand)) + if (IsUnsupportedSequenceBrand(majorBrand)) { - // Sequence major brands describe timed image sequences, which the still-image decoder cannot expose. + // A sequence major brand controls the presentation even if a still-image compatible brand is present. return false; } + if (IsSupportedSequenceBrand(majorBrand)) + { + fileType = HeifFileType.ImageSequence; + return true; + } + if (IsSupportedStillImageBrand(majorBrand)) { + fileType = HeifFileType.StillImage; return true; } // The minor-version field follows the major brand; compatible brands start at byte eight. + bool hasStillImageBrand = false; for (int offset = 8; offset < boxContent.Length; offset += 4) { Heif4CharCode compatibleBrand = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(boxContent[offset..]); - if (IsSupportedStillImageBrand(compatibleBrand)) + if (IsSupportedSequenceBrand(compatibleBrand)) { + fileType = HeifFileType.ImageSequence; return true; } + + if (IsSupportedStillImageBrand(compatibleBrand)) + { + hasStillImageBrand = true; + } } - return false; + fileType = hasStillImageBrand ? HeifFileType.StillImage : HeifFileType.Unsupported; + return hasStillImageBrand; } /// @@ -100,15 +115,22 @@ internal static class HeifConstants or Heif4CharCode.Jpeg; /// - /// Determines whether identifies a timed image sequence. + /// Determines whether identifies a supported timed image sequence. /// /// The registered file-type brand. - /// when the brand identifies a timed image sequence. - private static bool IsSequenceBrand(Heif4CharCode brand) + /// when the brand identifies a supported timed image sequence. + private static bool IsSupportedSequenceBrand(Heif4CharCode brand) => brand is Heif4CharCode.Hevc or Heif4CharCode.Hevx - or Heif4CharCode.Hevm + or Heif4CharCode.Avis; + + /// + /// Determines whether requires an image-sequence profile outside the implemented scope. + /// + /// The registered file-type brand. + /// when the major brand requires layered HEVC or JPEG sequence support. + private static bool IsUnsupportedSequenceBrand(Heif4CharCode brand) + => brand is Heif4CharCode.Hevm or Heif4CharCode.Hevs - or Heif4CharCode.Avis or Heif4CharCode.Jpgs; } diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 647270f474..d87df90553 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -58,6 +58,11 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// private readonly HeifBoxReader boxReader; + /// + /// The bounded image-sequence parser sharing the configured frame and metadata policy. + /// + private readonly HeifSequenceParser sequenceParser; + /// /// The fixed scratch buffer reused for all item-container box headers in this decode operation. /// @@ -98,6 +103,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore this.configuration = options.Configuration; this.metadata = new ImageMetadata(); this.boxReader = new HeifBoxReader(this.configuration.MemoryAllocator); + this.sequenceParser = new HeifSequenceParser(options); this.boxHeaderScratch = new byte[8]; this.items = []; this.itemLinks = []; @@ -106,11 +112,18 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// protected override Image Decode(BufferedReadStream stream, CancellationToken cancellationToken) { - if (!this.CheckFileTypeBox(stream)) + HeifFileType fileType = this.ReadFileTypeBox(stream); + if (fileType == HeifFileType.Unsupported) { throw new ImageFormatException("Not an HEIF image."); } + if (fileType == HeifFileType.ImageSequence) + { + HeifSequence sequence = this.ParseImageSequence(stream); + return this.DecodeImageSequence(stream, sequence, cancellationToken); + } + this.items.Clear(); this.itemLinks.Clear(); this.itemDataOffset = -1; @@ -146,11 +159,17 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// protected override ImageInfo Identify(BufferedReadStream stream, CancellationToken cancellationToken) { - if (!this.CheckFileTypeBox(stream)) + HeifFileType fileType = this.ReadFileTypeBox(stream); + if (fileType == HeifFileType.Unsupported) { throw new ImageFormatException("Not an HEIF image."); } + if (fileType == HeifFileType.ImageSequence) + { + return this.IdentifyImageSequence(this.ParseImageSequence(stream)); + } + this.items.Clear(); this.itemLinks.Clear(); this.itemDataOffset = -1; @@ -186,26 +205,378 @@ internal sealed class HeifDecoderCore : ImageDecoderCore } /// - /// Reads and validates the leading file-type box against the still-image brands supported by this decoder. + /// Reads and validates the leading file-type box against the image presentations supported by this decoder. /// /// The container stream positioned at its first top-level box. - /// when the complete file-type payload advertises a supported still-image brand. - private bool CheckFileTypeBox(BufferedReadStream stream) + /// The declared supported image presentation, or . + private HeifFileType ReadFileTypeBox(BufferedReadStream stream) { long boxLength = HeifBoxReader.ReadHeader(stream, stream.Length, this.boxHeaderScratch, out Heif4CharCode boxType, true); if (boxType != Heif4CharCode.Ftyp) { - return false; + return HeifFileType.Unsupported; } if (boxLength < 8 || boxLength > int.MaxValue || (boxLength & 3) != 0) { - return false; + return HeifFileType.Unsupported; } using IMemoryOwner boxMemory = this.boxReader.ReadPayload(stream, boxLength); Span boxBuffer = boxMemory.GetSpan(); - return HeifConstants.IsSupportedFileType(boxBuffer); + return HeifConstants.TryGetFileType(boxBuffer, out HeifFileType fileType) ? fileType : HeifFileType.Unsupported; + } + + /// + /// Locates and parses the single movie box of a supported HEIC or AVIF image sequence. + /// + /// The complete container stream positioned after its file-type box. + /// The bounded selected image-sequence model. + private HeifSequence ParseImageSequence(BufferedReadStream stream) + { + HeifSequence? sequence = null; + while (stream.Position < stream.Length) + { + long boxLength = HeifBoxReader.ReadHeader(stream, stream.Length, this.boxHeaderScratch, out Heif4CharCode boxType, true); + if (boxType == Heif4CharCode.Moov) + { + if (sequence is not null) + { + throw new InvalidImageContentException("The HEIF image sequence contains more than one movie box."); + } + + sequence = this.sequenceParser.Parse(stream, boxLength); + } + else + { + // Sequence samples use absolute file offsets, so unrelated top-level payloads never need buffering. + HeifBoxReader.Skip(stream, boxLength); + } + } + + return sequence ?? throw new InvalidImageContentException("The HEIF image sequence contains no movie box."); + } + + /// + /// Creates image and frame metadata from a parsed HEIC or AVIF image sequence without decoding its samples. + /// + /// The parsed selected image sequence. + /// The identified dimensions and bounded visible-frame metadata. + private ImageInfo IdentifyImageSequence(HeifSequence sequence) + { + HeifSequenceTrack colorTrack = sequence.ColorTrack; + this.UpdateSequenceMetadata(this.metadata, sequence); + ImageFrameMetadata[] frameMetadata = CreateSequenceFrameMetadata(colorTrack); + this.Dimensions = GetSequencePresentationExtent(colorTrack); + return new ImageInfo(this.Dimensions, this.metadata, frameMetadata); + } + + /// + /// Decodes the retained visible samples of a HEIC or AVIF image sequence into one multi-frame image. + /// + /// The destination pixel format. + /// The complete seekable HEIF stream. + /// The parsed selected image sequence. + /// The token used to cancel work between coded samples. + /// The decoded multi-frame image. + private Image DecodeImageSequence( + BufferedReadStream stream, + HeifSequence sequence, + CancellationToken cancellationToken) + where TPixel : unmanaged, IPixel + { + HeifSequenceTrack colorTrack = sequence.ColorTrack; + this.UpdateSequenceMetadata(this.metadata, sequence); + ImageFrame[] colorFrames = this.DecodeVisibleSequenceFrames( + stream, + colorTrack, + cancellationToken, + out int[] sampleIndices); + + Image? image = null; + try + { + // Each codec frame owns its pixel buffer. The multi-frame image adopts those buffers directly instead + // of cloning a complete decoded frame on every append. + image = new Image(this.configuration, this.metadata, colorFrames); + HeifSequenceTrack? alphaTrack = sequence.AlphaTrack; + if (alphaTrack is not null) + { + for (int frameIndex = 0; frameIndex < colorFrames.Length; frameIndex++) + { + cancellationToken.ThrowIfCancellationRequested(); + ImageFrame? alphaFrame = null; + HeifSequenceSample alphaSample = alphaTrack.Samples[sampleIndices[frameIndex]]; + this.ExecuteImageDataSegmentAction( + () => alphaFrame = this.DecodeSequenceFrame(stream, alphaTrack, alphaSample)); + + if (alphaFrame is null) + { + continue; + } + + if (alphaFrame.Size == colorFrames[frameIndex].Size) + { + using (alphaFrame) + { + this.ApplyAlpha(colorFrames[frameIndex], alphaFrame, colorTrack.IsPremultiplied); + } + } + else + { + // Auxiliary planes may use a lower resolution. Adopt the decoded frame into a temporary + // image so the established box resampler can resize it without another source-frame clone. + using Image alphaImage = new(this.configuration, new ImageMetadata(), [alphaFrame]); + alphaImage.Mutate( + context => context.Resize(colorFrames[frameIndex].Width, colorFrames[frameIndex].Height, KnownResamplers.Box)); + + this.ApplyAlpha(colorFrames[frameIndex], alphaImage.Frames.RootFrame, colorTrack.IsPremultiplied); + } + } + } + + ApplyPresentationTransforms( + image, + colorTrack.CleanAperture, + colorTrack.RotationAngle, + colorTrack.MirrorAxis); + + if (!this.Options.SkipMetadata) + { + image.Metadata.CicpProfile ??= image.Frames.RootFrame.Metadata.CicpProfile?.DeepClone(); + _ = this.TryConvertIccProfile(image); + } + else + { + foreach (ImageFrame frame in image.Frames) + { + frame.Metadata.CicpProfile = null; + } + } + + this.Dimensions = image.Size; + return image; + } + catch + { + if (image is not null) + { + image.Dispose(); + } + else + { + // Ownership transfers to Image only after its constructor validates every decoded frame. + foreach (ImageFrame frame in colorFrames) + { + frame.Dispose(); + } + } + + throw; + } + } + + /// + /// Decodes visible samples while preserving their source indices for frame-aligned alpha lookup. + /// + /// The destination pixel format. + /// The complete seekable HEIF stream. + /// The selected coded-image track. + /// The token used to cancel work between coded samples. + /// Receives the decode-order sample index for each returned visible frame. + /// The exact array of successfully decoded visible frames. + private ImageFrame[] DecodeVisibleSequenceFrames( + BufferedReadStream stream, + HeifSequenceTrack track, + CancellationToken cancellationToken, + out int[] sampleIndices) + where TPixel : unmanaged, IPixel + { + int visibleFrameCount = 0; + foreach (HeifSequenceSample sample in track.Samples) + { + visibleFrameCount += sample.IsHidden ? 0 : 1; + } + + ImageFrame[] frames = new ImageFrame[visibleFrameCount]; + sampleIndices = new int[visibleFrameCount]; + int decodedFrameCount = 0; + for (int sampleIndex = 0; sampleIndex < track.Samples.Length; sampleIndex++) + { + HeifSequenceSample sample = track.Samples[sampleIndex]; + if (sample.IsHidden) + { + continue; + } + + cancellationToken.ThrowIfCancellationRequested(); + ImageFrame? frame = null; + this.ExecuteImageDataSegmentAction(() => frame = this.DecodeSequenceFrame(stream, track, sample)); + if (frame is null) + { + continue; + } + + frame.Metadata.GetHeifMetadata().FrameDelay = new Rational(sample.Duration, track.MediaTimescale); + frames[decodedFrameCount] = frame; + sampleIndices[decodedFrameCount] = sampleIndex; + decodedFrameCount++; + } + + if (decodedFrameCount == 0) + { + throw new InvalidImageContentException("The HEIF image sequence contains no decodable visible samples."); + } + + if (decodedFrameCount != frames.Length) + { + // Compaction occurs only in IgnoreImageData mode after a recoverable coded-sample failure. + Array.Resize(ref frames, decodedFrameCount); + Array.Resize(ref sampleIndices, decodedFrameCount); + } + + return frames; + } + + /// + /// Reads and decodes one bounded coded sample without retaining its encoded byte buffer. + /// + /// The destination pixel format. + /// The complete seekable HEIF stream. + /// The track supplying the codec configuration and color description. + /// The validated sample range. + /// The independently owned decoded frame. + private ImageFrame DecodeSequenceFrame( + BufferedReadStream stream, + HeifSequenceTrack track, + HeifSequenceSample sample) + where TPixel : unmanaged, IPixel + { + if (track.CodecType != Heif4CharCode.Av01) + { + throw new ImageFormatException($"No decoder is available for image-sequence sample type '{track.CodecType}'."); + } + + Av1CodecConfiguration codecConfiguration = track.Av1CodecConfiguration + ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); + + using IMemoryOwner sampleOwner = this.configuration.MemoryAllocator.Allocate(sample.Length); + Span sampleData = sampleOwner.GetSpan()[..sample.Length]; + stream.Position = sample.Offset; + HeifBoxReader.ReadExactly(stream, sampleData, "The HEIF image-sequence sample is truncated."); + + codecConfiguration.ValidateItemData( + sampleData, + track.ContentLightLevel, + track.MasteringDisplayColorVolume, + out _, + out _); + + using Av1Decoder decoder = new(this.configuration); + ImageFrame frame = decoder.DecodeFrame( + sampleData, + track.CicpProfile, + codecConfiguration, + out _); + + if (frame.Width != track.CodedWidth || frame.Height != track.CodedHeight) + { + frame.Dispose(); + throw new InvalidImageContentException("The decoded image-sequence sample dimensions do not match its visual sample entry."); + } + + return frame; + } + + /// + /// Updates image-level metadata from the selected color and optional alpha sequence tracks. + /// + /// The image metadata receiving the sequence description. + /// The parsed selected image sequence. + private void UpdateSequenceMetadata(ImageMetadata metadata, HeifSequence sequence) + { + HeifSequenceTrack colorTrack = sequence.ColorTrack; + HeifMetadata heifMetadata = metadata.GetHeifMetadata(); + heifMetadata.RepeatCount = colorTrack.RepeatCount; + heifMetadata.AnimateRootFrame = true; + heifMetadata.HasAlpha = sequence.AlphaTrack is not null; + switch (colorTrack.CodecType) + { + case Heif4CharCode.Av01: + Av1CodecConfiguration av1Configuration = colorTrack.Av1CodecConfiguration + ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); + + heifMetadata.CompressionMethod = HeifCompressionMethod.Av1; + heifMetadata.BitDepth = av1Configuration.BitDepth; + heifMetadata.IsMonochrome = av1Configuration.IsMonochrome; + break; + case Heif4CharCode.Hvc1: + HevcCodecConfiguration hevcConfiguration = colorTrack.HevcCodecConfiguration + ?? throw new InvalidImageContentException("The HEVC image-sequence track has no codec configuration."); + + heifMetadata.CompressionMethod = HeifCompressionMethod.Hevc; + heifMetadata.BitDepth = hevcConfiguration.BitDepth; + heifMetadata.IsMonochrome = hevcConfiguration.IsMonochrome; + break; + default: + throw new InvalidImageContentException($"The image-sequence sample entry '{colorTrack.CodecType}' is not supported."); + } + + if (this.Options.SkipMetadata) + { + return; + } + + metadata.IccProfile = colorTrack.IccProfile?.DeepClone(); + metadata.CicpProfile = colorTrack.CicpProfile?.DeepClone(); + heifMetadata.ContentLightLevel = colorTrack.ContentLightLevel; + heifMetadata.MasteringDisplayColorVolume = colorTrack.MasteringDisplayColorVolume; + heifMetadata.ContentColorVolume = colorTrack.ContentColorVolume; + heifMetadata.AmbientViewingEnvironment = colorTrack.AmbientViewingEnvironment; + heifMetadata.ReferenceViewingEnvironment = colorTrack.ReferenceViewingEnvironment; + heifMetadata.NominalDiffuseWhite = colorTrack.NominalDiffuseWhite; + ApplyPixelAspectRatioMetadata(metadata, colorTrack.PixelAspectRatio, colorTrack.RotationAngle); + + HeifSequenceMetadata? trackMetadata = colorTrack.Metadata; + if (trackMetadata?.ExifData is not null) + { + this.ExecuteAncillarySegmentAction(() => ApplyExifProfile(metadata, trackMetadata.ExifData)); + } + + if (trackMetadata?.XmpData is not null) + { + this.ExecuteAncillarySegmentAction(() => metadata.XmpProfile = new XmpProfile(trackMetadata.XmpData)); + } + } + + /// + /// Creates one HEIF frame-metadata entry for each visible retained sequence sample. + /// + /// The selected color track supplying sample durations. + /// The exact visible-frame metadata array in presentation order. + private static ImageFrameMetadata[] CreateSequenceFrameMetadata(HeifSequenceTrack track) + { + int visibleFrameCount = 0; + foreach (HeifSequenceSample sample in track.Samples) + { + visibleFrameCount += sample.IsHidden ? 0 : 1; + } + + ImageFrameMetadata[] result = new ImageFrameMetadata[visibleFrameCount]; + int frameIndex = 0; + foreach (HeifSequenceSample sample in track.Samples) + { + if (sample.IsHidden) + { + continue; + } + + ImageFrameMetadata frameMetadata = new(); + frameMetadata.GetHeifMetadata().FrameDelay = new Rational(sample.Duration, track.MediaTimescale); + result[frameIndex++] = frameMetadata; + } + + return result; } /// @@ -1569,9 +1940,28 @@ internal sealed class HeifDecoderCore : ImageDecoderCore return; } + ApplyPixelAspectRatioMetadata(metadata, pixelAspectRatio, imageItem.RotationAngle); + } + + /// + /// Applies registered pixel spacing to ImageSharp's aspect-ratio resolution metadata. + /// + /// The image metadata receiving the aspect ratio. + /// The optional registered horizontal and vertical spacing. + /// The optional counter-clockwise quarter-turn count. + private static void ApplyPixelAspectRatioMetadata( + ImageMetadata metadata, + HeifPixelAspectRatio? pixelAspectRatio, + byte? rotationAngle) + { + if (pixelAspectRatio is null) + { + return; + } + // ImageMetadata expresses pixel width:height as vertical-density:horizontal-density. A quarter-turn exchanges // the displayed pixel axes, so it also exchanges which spacing value supplies each density. - bool swapsAxes = imageItem.RotationAngle is 1 or 3; + bool swapsAxes = rotationAngle is 1 or 3; metadata.HorizontalResolution = swapsAxes ? pixelAspectRatio.HorizontalSpacing : pixelAspectRatio.VerticalSpacing; @@ -1610,52 +2000,59 @@ internal sealed class HeifDecoderCore : ImageDecoderCore byte[] itemData = itemMemory.GetSpan().ToArray(); if (metadataItem.Type == Heif4CharCode.Exif) { - this.ExecuteAncillarySegmentAction(() => - { - if (itemData.Length < 8) - { - throw new InvalidImageContentException("The HEIF Exif item is truncated."); - } - - uint declaredTiffHeaderOffset = BinaryPrimitives.ReadUInt32BigEndian(itemData); - Span exifData = itemData.AsSpan(4); - int actualTiffHeaderOffset = -1; + this.ExecuteAncillarySegmentAction(() => ApplyExifProfile(metadata, itemData)); + } + else if (metadataItem.Type == Heif4CharCode.Mime && + string.Equals(metadataItem.ContentType, "application/rdf+xml", StringComparison.Ordinal)) + { + this.ExecuteAncillarySegmentAction(() => metadata.XmpProfile = new XmpProfile(itemData)); + } + } + } - // Annex A stores the offset to the first TIFF byte-order marker. Match libavif by finding the - // first valid TIFF signature and requiring the declared offset to identify that same header. - for (int i = 0; i <= exifData.Length - 4; i++) - { - bool isBigEndianTiff = exifData[i] == (byte)'M' && - exifData[i + 1] == (byte)'M' && - exifData[i + 2] == 0 && - exifData[i + 3] == 42; + /// + /// Validates the HEIF Exif TIFF-header offset and applies the contained TIFF payload. + /// + /// The image metadata receiving the Exif profile. + /// The complete HEIF Exif item including its four-byte offset field. + private static void ApplyExifProfile(ImageMetadata metadata, byte[] itemData) + { + if (itemData.Length < 8) + { + throw new InvalidImageContentException("The HEIF Exif item is truncated."); + } - bool isLittleEndianTiff = exifData[i] == (byte)'I' && - exifData[i + 1] == (byte)'I' && - exifData[i + 2] == 42 && - exifData[i + 3] == 0; + uint declaredTiffHeaderOffset = BinaryPrimitives.ReadUInt32BigEndian(itemData); + Span exifData = itemData.AsSpan(4); + int actualTiffHeaderOffset = -1; - if (isBigEndianTiff || isLittleEndianTiff) - { - actualTiffHeaderOffset = i; - break; - } - } + // Annex A stores the offset to the first TIFF byte-order marker. Match libavif by finding the first valid + // TIFF signature and requiring the declared offset to identify that same header. + for (int i = 0; i <= exifData.Length - 4; i++) + { + bool isBigEndianTiff = exifData[i] == (byte)'M' && + exifData[i + 1] == (byte)'M' && + exifData[i + 2] == 0 && + exifData[i + 3] == 42; - if (actualTiffHeaderOffset < 0 || declaredTiffHeaderOffset != (uint)actualTiffHeaderOffset) - { - throw new InvalidImageContentException("The HEIF Exif item has an invalid TIFF-header offset."); - } + bool isLittleEndianTiff = exifData[i] == (byte)'I' && + exifData[i + 1] == (byte)'I' && + exifData[i + 2] == 42 && + exifData[i + 3] == 0; - metadata.ExifProfile = new ExifProfile(exifData[actualTiffHeaderOffset..].ToArray()); - }); - } - else if (metadataItem.Type == Heif4CharCode.Mime && - string.Equals(metadataItem.ContentType, "application/rdf+xml", StringComparison.Ordinal)) + if (isBigEndianTiff || isLittleEndianTiff) { - this.ExecuteAncillarySegmentAction(() => metadata.XmpProfile = new XmpProfile(itemData)); + actualTiffHeaderOffset = i; + break; } } + + if (actualTiffHeaderOffset < 0 || declaredTiffHeaderOffset != (uint)actualTiffHeaderOffset) + { + throw new InvalidImageContentException("The HEIF Exif item has an invalid TIFF-header offset."); + } + + metadata.ExifProfile = new ExifProfile(exifData[actualTiffHeaderOffset..].ToArray()); } /// @@ -1721,6 +2118,20 @@ internal sealed class HeifDecoderCore : ImageDecoderCore : extent; } + /// + /// Gets the dimensions of a sequence sample after its clean-aperture and rotation properties are applied. + /// + /// The selected color track whose samples share the presentation properties. + /// The displayed frame dimensions. + private static Size GetSequencePresentationExtent(HeifSequenceTrack track) + { + Size codedExtent = new(track.CodedWidth, track.CodedHeight); + Size extent = track.CleanAperture is not null ? track.CleanAperture.Value.ToRectangle(codedExtent).Size : codedExtent; + return track.RotationAngle is not null && (track.RotationAngle.Value & 1) != 0 + ? new Size(extent.Height, extent.Width) + : extent; + } + /// /// Applies the clean-aperture, rotation, and mirror properties associated with an image item. /// @@ -1729,20 +2140,36 @@ internal sealed class HeifDecoderCore : ImageDecoderCore /// The item carrying the presentation properties. private static void ApplyPresentationTransforms(Image image, HeifItem item) where TPixel : unmanaged, IPixel + => ApplyPresentationTransforms(image, item.CleanAperture, item.RotationAngle, item.MirrorAxis); + + /// + /// Applies shared clean-aperture, rotation, and mirror properties to every frame of an image presentation. + /// + /// The image pixel format. + /// The decoded image presentation. + /// The optional clean-aperture crop. + /// The optional counter-clockwise quarter-turn count. + /// The optional horizontal or vertical mirror axis. + private static void ApplyPresentationTransforms( + Image image, + HeifCleanAperture? cleanAperture, + byte? rotationAngle, + byte? mirrorAxis) + where TPixel : unmanaged, IPixel { - if (item.CleanAperture is not null) + if (cleanAperture is not null) { - Rectangle cropRectangle = item.CleanAperture.Value.ToRectangle(image.Size); + Rectangle cropRectangle = cleanAperture.Value.ToRectangle(image.Size); if (cropRectangle != image.Bounds) { image.Mutate(context => context.Crop(cropRectangle)); } } - if (item.RotationAngle is not null) + if (rotationAngle is not null) { // HEIF angles count quarter turns counter-clockwise, while ImageSharp's optimized rotate modes are clockwise. - RotateMode rotation = item.RotationAngle.Value switch + RotateMode rotation = rotationAngle.Value switch { 1 => RotateMode.Rotate270, 2 => RotateMode.Rotate180, @@ -1756,10 +2183,10 @@ internal sealed class HeifDecoderCore : ImageDecoderCore } } - if (item.MirrorAxis is not null) + if (mirrorAxis is not null) { // Axis zero reflects top-to-bottom around the horizontal axis; axis one reflects left-to-right. - FlipMode flip = item.MirrorAxis.Value == 0 ? FlipMode.Vertical : FlipMode.Horizontal; + FlipMode flip = mirrorAxis.Value == 0 ? FlipMode.Vertical : FlipMode.Horizontal; image.Mutate(context => context.Flip(flip)); } } @@ -1855,17 +2282,28 @@ internal sealed class HeifDecoderCore : ImageDecoderCore alphaImage.Mutate(context => context.Resize(image.Width, image.Height, KnownResamplers.Box)); } - using IMemoryOwner rowOwner = this.configuration.MemoryAllocator.Allocate(image.Width); - Span rgbaRow = rowOwner.GetSpan()[..image.Width]; + this.ApplyAlpha(image.Frames.RootFrame, alphaImage.Frames.RootFrame, premultiplied); + } + + /// + /// Composes one same-sized auxiliary alpha frame into a decoded color frame. + /// + /// The destination color pixel format. + /// The decoded color frame receiving alpha values. + /// The decoded same-sized 16-bit alpha frame. + /// Whether the encoded color samples are premultiplied by alpha. + private void ApplyAlpha(ImageFrame colorFrame, ImageFrame alphaFrame, bool premultiplied) + where TPixel : unmanaged, IPixel + { + using IMemoryOwner rowOwner = this.configuration.MemoryAllocator.Allocate(colorFrame.Width); + Span rgbaRow = rowOwner.GetSpan()[..colorFrame.Width]; PixelOperations pixelOperations = PixelOperations.Instance; - ImageFrame colorFrame = image.Frames.RootFrame; - ImageFrame alphaFrame = alphaImage.Frames.RootFrame; - for (int y = 0; y < image.Height; y++) + for (int y = 0; y < colorFrame.Height; y++) { Span colorRow = colorFrame.PixelBuffer.DangerousGetRowSpan(y); Span alphaRow = alphaFrame.PixelBuffer.DangerousGetRowSpan(y); pixelOperations.ToRgba64(this.configuration, colorRow, rgbaRow); - for (int x = 0; x < image.Width; x++) + for (int x = 0; x < colorFrame.Width; x++) { Rgba64 pixel = rgbaRow[x]; pixel.A = alphaRow[x].PackedValue; diff --git a/src/ImageSharp/Formats/Heif/HeifFileType.cs b/src/ImageSharp/Formats/Heif/HeifFileType.cs new file mode 100644 index 0000000000..86c1cb9b1a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/HeifFileType.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif; + +/// +/// Identifies the supported image presentation declared by a HEIF file-type box. +/// +internal enum HeifFileType +{ + /// + /// The file-type box does not declare a supported HEIF image presentation. + /// + Unsupported, + + /// + /// The container presents a primary image item. + /// + StillImage, + + /// + /// The container presents a timed HEIC or AVIF image sequence. + /// + ImageSequence +} diff --git a/src/ImageSharp/Formats/Heif/HeifImageFormatDetector.cs b/src/ImageSharp/Formats/Heif/HeifImageFormatDetector.cs index dffa45ee19..a8b07402fb 100644 --- a/src/ImageSharp/Formats/Heif/HeifImageFormatDetector.cs +++ b/src/ImageSharp/Formats/Heif/HeifImageFormatDetector.cs @@ -22,10 +22,10 @@ public sealed class HeifImageFormatDetector : IImageFormatDetector } /// - /// Determines whether the available header begins with a supported still-image HEIF file-type box. + /// Determines whether the available header begins with a supported HEIF image file-type box. /// /// The fixed-size header prefix supplied by format detection. - /// when the prefix declares a supported still-image brand; otherwise, . + /// when the prefix declares a supported image or image-sequence brand. private static bool IsSupportedFileFormat(ReadOnlySpan header) { // Detection is intentionally limited to files beginning with ftyp. Other valid top-level boxes can precede @@ -45,6 +45,6 @@ public sealed class HeifImageFormatDetector : IImageFormatDetector // sufficient for detection; the decoder validates the complete box before reading the rest of the container. int availableContentLength = (int)Math.Min(boxSize - 8, (uint)header.Length - 8); availableContentLength &= ~3; - return HeifConstants.IsSupportedFileType(header.Slice(8, availableContentLength)); + return HeifConstants.TryGetFileType(header.Slice(8, availableContentLength), out _); } } diff --git a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs index 6d19166a2e..2506def285 100644 --- a/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs +++ b/src/ImageSharp/Formats/Heif/HeifSequenceParser.cs @@ -2209,7 +2209,7 @@ internal sealed class HeifSequenceParser /// The exception raised while parsing the ancillary segment. /// when decoding may continue without the segment. private bool ShouldIgnoreAncillarySegmentError(Exception exception) - => this.options.SegmentIntegrityHandling is not SegmentIntegrityHandling.Strict && IsRecoverableSegmentError(exception); + => this.options.SegmentIntegrityHandling is not SegmentIntegrityHandling.Strict && ImageDecoderCore.IsRecoverableSegmentError(exception); /// /// Determines whether a recoverable image-data-segment error should be ignored by the configured decoder policy. @@ -2217,19 +2217,7 @@ internal sealed class HeifSequenceParser /// The exception raised while parsing the image-data segment. /// when decoding may continue without the segment. private bool ShouldIgnoreImageDataSegmentError(Exception exception) - => this.options.SegmentIntegrityHandling is SegmentIntegrityHandling.IgnoreImageData && IsRecoverableSegmentError(exception); - - /// - /// Identifies the recoverable exception types handled by . - /// - /// The exception raised while parsing a segment. - /// when the exception represents a recoverable segment error. - private static bool IsRecoverableSegmentError(Exception exception) - => exception is ImageFormatException - or InvalidIccProfileException - or InvalidImageContentException - or InvalidOperationException - or NotSupportedException; + => this.options.SegmentIntegrityHandling is SegmentIntegrityHandling.IgnoreImageData && ImageDecoderCore.IsRecoverableSegmentError(exception); /// /// Records one unique child box while retaining only its stream range. diff --git a/src/ImageSharp/Formats/ImageDecoderCore.cs b/src/ImageSharp/Formats/ImageDecoderCore.cs index 3f36b8931f..a3338f12d6 100644 --- a/src/ImageSharp/Formats/ImageDecoderCore.cs +++ b/src/ImageSharp/Formats/ImageDecoderCore.cs @@ -49,12 +49,7 @@ internal abstract class ImageDecoderCore { action(); } - catch (Exception ex) when (ex - is ImageFormatException - or InvalidIccProfileException - or InvalidImageContentException - or InvalidOperationException - or NotSupportedException) + catch (Exception ex) when (IsRecoverableSegmentError(ex)) { // Intentionally ignored in non-strict segment integrity modes. } @@ -76,17 +71,24 @@ internal abstract class ImageDecoderCore { action(); } - catch (Exception ex) when (ex - is ImageFormatException - or InvalidIccProfileException - or InvalidImageContentException - or InvalidOperationException - or NotSupportedException) + catch (Exception ex) when (IsRecoverableSegmentError(ex)) { // Intentionally ignored when image data integrity handling is set to IgnoreImageData. } } + /// + /// Determines whether an exception represents a recoverable image segment error. + /// + /// The exception raised while processing an image segment. + /// when a segment integrity policy may ignore the exception. + public static bool IsRecoverableSegmentError(Exception exception) + => exception is ImageFormatException + or InvalidIccProfileException + or InvalidImageContentException + or InvalidOperationException + or NotSupportedException; + /// /// Throws unless the decoder is running in a non-strict segment integrity mode. /// Use this only from within when local control flow diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs index ceaa292c3c..0762aebcc6 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs @@ -125,11 +125,24 @@ public class HeifDecoderTests [Theory] [InlineData(Heif4CharCode.Hevc)] [InlineData(Heif4CharCode.Hevx)] + [InlineData(Heif4CharCode.Avis)] + public void DetectorRecognizesSupportedSequenceMajorBrand(Heif4CharCode brand) + { + byte[] data = CreateEncodedContainer(); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(8), (uint)brand); + HeifImageFormatDetector detector = new(); + + bool detected = detector.TryDetectFormat(data.AsSpan(0, detector.HeaderSize), out IImageFormat format); + + Assert.True(detected); + Assert.Same(HeifFormat.Instance, format); + } + + [Theory] [InlineData(Heif4CharCode.Hevm)] [InlineData(Heif4CharCode.Hevs)] - [InlineData(Heif4CharCode.Avis)] [InlineData(Heif4CharCode.Jpgs)] - public void DetectorRejectsSequenceMajorBrand(Heif4CharCode brand) + public void DetectorRejectsUnsupportedSequenceMajorBrand(Heif4CharCode brand) { byte[] data = CreateEncodedContainer(); BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(8), (uint)brand); diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs index 844bcaa58b..435222e1aa 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifSequenceParserTests.cs @@ -5,6 +5,7 @@ using System.Buffers.Binary; using System.Text; using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats.Heif; +using SixLabors.ImageSharp.PixelFormats; namespace SixLabors.ImageSharp.Tests.Formats.Heif; @@ -19,6 +20,76 @@ public class HeifSequenceParserTests private static ReadOnlySpan TrackXmpData => ""u8; + [Fact] + public void IdentifyReturnsBoundedSequenceAndFrameMetadata() + { + byte[] data = CreateSequenceContainer(trackProperties: true, trackMetadata: true); + + ImageInfo info = Image.Identify(data); + HeifMetadata metadata = info.Metadata.GetHeifMetadata(); + + Assert.Equal(new Size(240, 320), info.Size); + Assert.Equal(2, info.FrameCount); + Assert.Equal(HeifCompressionMethod.Av1, metadata.CompressionMethod); + Assert.Equal(HeifBitDepth.Bit8, metadata.BitDepth); + Assert.Equal(3, metadata.RepeatCount); + Assert.False(metadata.HasAlpha); + Assert.NotNull(info.Metadata.CicpProfile); + Assert.NotNull(info.Metadata.ExifProfile); + Assert.NotNull(info.Metadata.XmpProfile); + Assert.NotNull(metadata.ContentLightLevel); + Assert.Equal(4D, info.Metadata.HorizontalResolution); + Assert.Equal(3D, info.Metadata.VerticalResolution); + Assert.All( + info.FrameMetadataCollection, + frame => Assert.Equal(new Rational(1, 10), frame.GetHeifMetadata().FrameDelay)); + } + + [Fact] + public void IdentifySkipsAncillarySequenceMetadataWithoutDroppingImageMetadata() + { + byte[] data = CreateSequenceContainer(trackProperties: true, trackMetadata: true); + using MemoryStream stream = new(data, false); + DecoderOptions options = new() { SkipMetadata = true }; + + ImageInfo info = Image.Identify(options, stream); + HeifMetadata metadata = info.Metadata.GetHeifMetadata(); + + Assert.Equal(HeifCompressionMethod.Av1, metadata.CompressionMethod); + Assert.Equal(HeifBitDepth.Bit8, metadata.BitDepth); + Assert.Equal(3, metadata.RepeatCount); + Assert.Null(info.Metadata.CicpProfile); + Assert.Null(info.Metadata.IccProfile); + Assert.Null(info.Metadata.ExifProfile); + Assert.Null(info.Metadata.XmpProfile); + Assert.Null(metadata.ContentLightLevel); + Assert.All( + info.FrameMetadataCollection, + frame => Assert.Equal(new Rational(1, 10), frame.GetHeifMetadata().FrameDelay)); + } + + [Fact] + public void DecodeAdoptsIndependentAv1SamplesAsImageFrames() + { + byte[] source = TestFile.Create(TestImages.Heif.Orange4x4).Bytes; + byte[] data = CreateDecodableAv1SequenceContainer(source.AsSpan(0x10E, 0x1D), source.AsSpan(0xC7, 4)); + + using Image expected = Image.Load(source); + using Image actual = Image.Load(data); + + Assert.Equal(new Size(4, 4), actual.Size); + Assert.Equal(2, actual.Frames.Count); + foreach (ImageFrame frame in actual.Frames) + { + Assert.Equal(new Rational(1, 10), frame.Metadata.GetHeifMetadata().FrameDelay); + + for (int y = 0; y < frame.Height; y++) + { + Assert.True(frame.PixelBuffer.DangerousGetRowSpan(y).SequenceEqual(expected.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y))); + } + } + } + [Fact] public void ParseResolvesLibavifShapedSampleTable() { @@ -258,7 +329,12 @@ public class HeifSequenceParserTests bool trackMetadata = false, bool metadataInItemData = false, bool invalidTrackMetadata = false, - bool invalidRotation = false) + bool invalidRotation = false, + int width = 320, + int height = 240, + byte[] av1Configuration = null, + int? sampleSize = null, + bool allSamplesSync = false) { using MemoryStream stream = new(); using BinaryWriter writer = new(stream, Encoding.UTF8, true); @@ -274,7 +350,7 @@ public class HeifSequenceParserTests EndBox(writer, movieHeader); long track = BeginBox(writer, Heif4CharCode.Trak); - WriteTrackHeader(writer); + WriteTrackHeader(writer, width, height); WriteEditList(writer); if (trackMetadata) { @@ -295,7 +371,12 @@ public class HeifSequenceParserTests directReferences, directReferenceSampleId, trackProperties, - invalidRotation); + invalidRotation, + width, + height, + av1Configuration, + sampleSize, + allSamplesSync); EndBox(writer, mediaInformation); EndBox(writer, media); @@ -315,7 +396,52 @@ public class HeifSequenceParserTests return file; } - private static void WriteTrackHeader(BinaryWriter writer) + private static byte[] CreateSequenceContainer(bool trackProperties, bool trackMetadata) + { + byte[] movie = CreateSequenceFile( + 1024, + trackProperties: trackProperties, + trackMetadata: trackMetadata, + metadataInItemData: true); + + byte[] data = new byte[movie.Length + 24]; + BinaryPrimitives.WriteUInt32BigEndian(data, 24); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(4), (uint)Heif4CharCode.Ftyp); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(8), (uint)Heif4CharCode.Avis); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(12), 0); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(16), (uint)Heif4CharCode.Avif); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(20), (uint)Heif4CharCode.Mif1); + movie.CopyTo(data, 24); + return data; + } + + private static byte[] CreateDecodableAv1SequenceContainer(ReadOnlySpan sample, ReadOnlySpan configuration) + { + const int fileTypeLength = 24; + const int movieStorageLength = 2048; + uint chunkOffset = fileTypeLength + movieStorageLength; + byte[] movie = CreateSequenceFile( + chunkOffset, + width: 4, + height: 4, + av1Configuration: configuration.ToArray(), + sampleSize: sample.Length, + allSamplesSync: true); + + byte[] data = new byte[chunkOffset + (sample.Length * 2)]; + BinaryPrimitives.WriteUInt32BigEndian(data, fileTypeLength); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(4), (uint)Heif4CharCode.Ftyp); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(8), (uint)Heif4CharCode.Avis); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(12), 0); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(16), (uint)Heif4CharCode.Avif); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(20), (uint)Heif4CharCode.Mif1); + movie.CopyTo(data, fileTypeLength); + sample.CopyTo(data.AsSpan((int)chunkOffset)); + sample.CopyTo(data.AsSpan((int)chunkOffset + sample.Length)); + return data; + } + + private static void WriteTrackHeader(BinaryWriter writer, int width, int height) { long trackHeader = BeginBox(writer, Heif4CharCode.Tkhd); WriteFullBoxHeader(writer, 0, 3); @@ -334,8 +460,8 @@ public class HeifSequenceParserTests WriteUInt32(writer, 0); WriteUInt32(writer, 0); WriteUInt32(writer, 0x40000000); - WriteUInt32(writer, 320U << 16); - WriteUInt32(writer, 240U << 16); + WriteUInt32(writer, (uint)width << 16); + WriteUInt32(writer, (uint)height << 16); EndBox(writer, trackHeader); } @@ -468,10 +594,15 @@ public class HeifSequenceParserTests bool directReferences, uint directReferenceSampleId, bool trackProperties, - bool invalidRotation) + bool invalidRotation, + int width, + int height, + byte[] av1Configuration, + int? sampleSize, + bool allSamplesSync) { long sampleTable = BeginBox(writer, Heif4CharCode.Stbl); - WriteSampleDescription(writer, hevc, trackProperties, invalidRotation); + WriteSampleDescription(writer, hevc, trackProperties, invalidRotation, width, height, av1Configuration, allSamplesSync); long timing = BeginBox(writer, Heif4CharCode.Stts); WriteFullBoxHeader(writer, 0, 0); @@ -492,8 +623,8 @@ public class HeifSequenceParserTests WriteFullBoxHeader(writer, 0, 0); WriteUInt32(writer, 0); WriteUInt32(writer, 2); - WriteUInt32(writer, 10); - WriteUInt32(writer, 12); + WriteUInt32(writer, (uint)(sampleSize ?? 10)); + WriteUInt32(writer, (uint)(sampleSize ?? 12)); EndBox(writer, sampleSizes); long chunkOffsets = BeginBox(writer, Heif4CharCode.Stco); @@ -504,8 +635,13 @@ public class HeifSequenceParserTests long syncSamples = BeginBox(writer, Heif4CharCode.Stss); WriteFullBoxHeader(writer, 0, 0); + WriteUInt32(writer, allSamplesSync ? 2U : 1U); WriteUInt32(writer, 1); - WriteUInt32(writer, 1); + if (allSamplesSync) + { + WriteUInt32(writer, 2); + } + EndBox(writer, syncSamples); if (compositionOffsets) @@ -564,7 +700,15 @@ public class HeifSequenceParserTests EndBox(writer, sampleMap); } - private static void WriteSampleDescription(BinaryWriter writer, bool hevc, bool trackProperties, bool invalidRotation) + private static void WriteSampleDescription( + BinaryWriter writer, + bool hevc, + bool trackProperties, + bool invalidRotation, + int width, + int height, + byte[] av1Configuration, + bool allSamplesSync) { long description = BeginBox(writer, Heif4CharCode.Stsd); WriteFullBoxHeader(writer, 0, 0); @@ -573,8 +717,8 @@ public class HeifSequenceParserTests WriteZeros(writer, 6); WriteUInt16(writer, 1); WriteZeros(writer, 16); - WriteUInt16(writer, 320); - WriteUInt16(writer, 240); + WriteUInt16(writer, (ushort)width); + WriteUInt16(writer, (ushort)height); WriteUInt32(writer, 0x00480000); WriteUInt32(writer, 0x00480000); WriteUInt32(writer, 0); @@ -590,24 +734,24 @@ public class HeifSequenceParserTests else { long configuration = BeginBox(writer, Heif4CharCode.Av1C); - writer.Write(new byte[] { 0x81, 0, 0, 0 }); + writer.Write(av1Configuration ?? [0x81, 0, 0, 0]); EndBox(writer, configuration); } if (trackProperties) { - WriteTrackImageProperties(writer, invalidRotation); + WriteTrackImageProperties(writer, invalidRotation, width, height); } long codingConstraints = BeginBox(writer, Heif4CharCode.Ccst); WriteFullBoxHeader(writer, 0, 0); - WriteUInt32(writer, 0x7C000000); + WriteUInt32(writer, allSamplesSync ? 0xFC000000U : 0x7C000000U); EndBox(writer, codingConstraints); EndBox(writer, sampleEntry); EndBox(writer, description); } - private static void WriteTrackImageProperties(BinaryWriter writer, bool invalidRotation) + private static void WriteTrackImageProperties(BinaryWriter writer, bool invalidRotation, int width, int height) { long color = BeginBox(writer, Heif4CharCode.Colr); WriteUInt32(writer, (uint)Heif4CharCode.Nclx); @@ -623,9 +767,9 @@ public class HeifSequenceParserTests EndBox(writer, pixelAspectRatio); long cleanAperture = BeginBox(writer, Heif4CharCode.Clap); - WriteUInt32(writer, 320); + WriteUInt32(writer, (uint)width); WriteUInt32(writer, 1); - WriteUInt32(writer, 240); + WriteUInt32(writer, (uint)height); WriteUInt32(writer, 1); WriteUInt32(writer, 0); WriteUInt32(writer, 1);