Browse Source

Implement bounded AV1 image sequences

pull/2633/head
James Jackson-South 4 weeks ago
parent
commit
5ad0ca8756
  1. 7
      HEIF_IMPLEMENTATION_PLAN.md
  2. 888
      src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs
  3. 136
      src/ImageSharp/Formats/Heif/HeifEncoderCore.cs
  4. 86
      tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs

7
HEIF_IMPLEMENTATION_PLAN.md

@ -27,14 +27,14 @@ Reference checkout evidence on 2026-08-31:
## Current source reconciliation ## Current source reconciliation
Reconciled with the worktree on 2026-09-03. Reconciled with the worktree on 2026-09-04.
- [~] The bounded container reader, still-image path, sequence parser, AV1 decoder, color pipeline, presentation pipeline, and broad AV1 test suite exist locally. - [~] The bounded container reader, still-image path, sequence parser, AV1 decoder, color pipeline, presentation pipeline, and broad AV1 test suite exist locally.
- [x] The inter-frame decoder has verified checkpoints through inter deblocking decisions and - [x] The inter-frame decoder has verified checkpoints through inter deblocking decisions and
reference/mode deltas. reference/mode deltas.
- [~] Loop filtering, CDEF, super-resolution, restoration, film grain, layered presentation, alpha composition, and color conversion exist locally. Shared-source cleanup changed the current tree, so final production-path verification is open. - [~] Loop filtering, CDEF, super-resolution, restoration, film grain, layered presentation, alpha composition, and color conversion exist locally. Shared-source cleanup changed the current tree, so final production-path verification is open.
- [~] AV1 writer primitives, forward transforms, symbol encoding, and tile-writing source are connected to the public encoder for bounded still-image AVIF color and optional auxiliary alpha output. - [~] AV1 writer primitives, forward transforms, symbol encoding, and tile-writing source are connected to the public encoder for bounded still-image and all-intra sequence AVIF color with optional auxiliary alpha output.
- [~] The public AV1 encoder supports explicit single-image requests. Lossless output, bounded sequences, grids, orientation handling, and default format registration remain open. - [~] The public AV1 encoder supports explicit single-image and bounded all-intra sequence requests. Lossless output is implemented; inter-frame sequence search, grids, orientation handling, and default format registration remain open.
- [x] Patented codec production code, registrations, tests, benchmarks, fixtures, reference outputs, and notices were manually deleted and committed by `78a74d448`. - [x] Patented codec production code, registrations, tests, benchmarks, fixtures, reference outputs, and notices were manually deleted and committed by `78a74d448`.
- [x] Remaining task-created HM, HEVC, libheif, GPAC, Nokia, FFmpeg, Pillow HEIF, libavif-build, and libjpeg-build directories were traced to their creation commands in the recovered Codex session history and deleted on 2026-08-31. The user-provided repositories and all libaom-only source, build, and reference data were left untouched. - [x] Remaining task-created HM, HEVC, libheif, GPAC, Nokia, FFmpeg, Pillow HEIF, libavif-build, and libjpeg-build directories were traced to their creation commands in the recovered Codex session history and deleted on 2026-08-31. The user-provided repositories and all libaom-only source, build, and reference data were left untouched.
- [x] The PNG metadata-suppression fix and three HEIF/AV1 diagnostic-save call-site corrections passed the exact 34 net11.0 ARM CI cases and were committed with the single-reference checkpoint as `54bb6cbe59bd113058854a3ee31448cf61f462ca`. They are infrastructure evidence, not decoder or encoder completion evidence. - [x] The PNG metadata-suppression fix and three HEIF/AV1 diagnostic-save call-site corrections passed the exact 34 net11.0 ARM CI cases and were committed with the single-reference checkpoint as `54bb6cbe59bd113058854a3ee31448cf61f462ca`. They are infrastructure evidence, not decoder or encoder completion evidence.
@ -887,6 +887,7 @@ Encoder verification contract:
- [x] Still-image AVIF metadata preservation now writes an unrestricted ICC `colr/prof` property before the independent `colr/nclx` property, Exif and XMP as separate `mdat` items, and one `cdsc` relationship from each metadata item to the primary color item. Exif stores the exact big-endian TIFF-header offset required by the HEIF item syntax; XMP uses the `mime` item type and `application/rdf+xml` content type. Existing ICC and XMP storage is read synchronously and copied once into final encoder storage rather than cloned into an intermediate array. `SkipMetadata` suppresses all three profile types while retaining the CICP values required to describe the encoded planes. The same option now reaches legacy JPEG payloads, whose encoder no longer writes application profiles or comments when metadata is disabled. - [x] Still-image AVIF metadata preservation now writes an unrestricted ICC `colr/prof` property before the independent `colr/nclx` property, Exif and XMP as separate `mdat` items, and one `cdsc` relationship from each metadata item to the primary color item. Exif stores the exact big-endian TIFF-header offset required by the HEIF item syntax; XMP uses the `mime` item type and `application/rdf+xml` content type. Existing ICC and XMP storage is read synchronously and copied once into final encoder storage rather than cloned into an intermediate array. `SkipMetadata` suppresses all three profile types while retaining the CICP values required to describe the encoded planes. The same option now reaches legacy JPEG payloads, whose encoder no longer writes application profiles or comments when metadata is disabled.
- [x] Exact container tests verify every emitted item declaration, name, MIME content type, `cdsc` relationship, Exif offset and payload, XMP payload, ICC/CICP property order, compact association byte, propertyless metadata exclusion, decoded profile value, and both `SkipMetadata` branches. The final HEIF encoder set passes 44 of 44 and the complete JPEG encoder set passes 257 of 257 through direct foreground net11 Release VSTest. The complete non-HEVC HEIF namespace passes 9,282 of 9,282 with no failure, crash, or detached test host, and current official libaom accepts all 47 current generated AV1 payloads. - [x] Exact container tests verify every emitted item declaration, name, MIME content type, `cdsc` relationship, Exif offset and payload, XMP payload, ICC/CICP property order, compact association byte, propertyless metadata exclusion, decoded profile value, and both `SkipMetadata` branches. The final HEIF encoder set passes 44 of 44 and the complete JPEG encoder set passes 257 of 257 through direct foreground net11 Release VSTest. The complete non-HEVC HEIF namespace passes 9,282 of 9,282 with no failure, crash, or detached test host, and current official libaom accepts all 47 current generated AV1 payloads.
- [x] A code-wide production HEIF/AV1 stack-storage audit, excluding HEVC, removed every block-sized, variable-length, or repeatedly nested scratch buffer. Spatial luma and chroma, filter-intra, chroma-from-luma, luma and chroma palette selection, and K-means iteration now use typed views over 642 signed-integer elements, about 2.51 KiB, at the start of the existing 3.125 KiB IBC region. Those searches are sequential for one block, so the block-workspace owner does not grow and no rent, copy, or additional lifetime is introduced. CDEF directions, variances, and its 64-entry block list now append 1 KiB to the existing bounded operation owner instead of occupying hidden inline or explicit stack arrays. No remaining `stackalloc` depends on block dimensions, sample count, or runtime length; the largest remaining individual span is 128 bytes, and the remaining sites are fixed syntax, SIMD-lane, filter-tap, plane-metadata, or small candidate storage. The exact-owner test now proves the mode, palette, and IBC views share one allocation. Roslynk reports zero compiler errors and no diagnostics in the changed files, the Release test-project build completes with the established 1,992 warnings and zero errors, 81 of 81 focused cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,282 of 9,282 through one foreground net11 VSTest run. - [x] A code-wide production HEIF/AV1 stack-storage audit, excluding HEVC, removed every block-sized, variable-length, or repeatedly nested scratch buffer. Spatial luma and chroma, filter-intra, chroma-from-luma, luma and chroma palette selection, and K-means iteration now use typed views over 642 signed-integer elements, about 2.51 KiB, at the start of the existing 3.125 KiB IBC region. Those searches are sequential for one block, so the block-workspace owner does not grow and no rent, copy, or additional lifetime is introduced. CDEF directions, variances, and its 64-entry block list now append 1 KiB to the existing bounded operation owner instead of occupying hidden inline or explicit stack arrays. No remaining `stackalloc` depends on block dimensions, sample count, or runtime length; the largest remaining individual span is 128 bytes, and the remaining sites are fixed syntax, SIMD-lane, filter-tap, plane-metadata, or small candidate storage. The exact-owner test now proves the mode, palette, and IBC views share one allocation. Roslynk reports zero compiler errors and no diagnostics in the changed files, the Release test-project build completes with the established 1,992 warnings and zero errors, 81 of 81 focused cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,282 of 9,282 through one foreground net11 VSTest run.
- [~] Bounded public image-sequence output now emits an `avis` movie with version-one movie, track, and media headers; AV1 visual sample entries; exact run-length-compressed timing; per-sample sizes; 64-bit chunk offsets; and an explicit sync-sample table. Color and optional auxiliary alpha use independently configured AV1 tracks linked by `auxl`, while every current sample is an independently decodable all-intra picture. Frame payloads are written once into contiguous allocator-backed chunks per track. One compact managed table retains only offset, length, and duration for both tracks, and the bounded `moov` owner is patched once after its final size is known, so prefixed and non-seekable destinations require neither seeking nor a file-sized copy. The media timescale uses the exact representable least common multiple of frame-delay denominators and a documented microsecond fallback; zero delays become the smallest legal positive duration. Public lossless color-and-alpha round trips preserve all frames, distinct 24, 25, and 30 fps delays, and finite or infinite repetition, while a separate case proves prefixed non-seekable output. The net11 Release build completes with the established 1,005 warnings and zero errors, all 48 HEIF encoder cases pass, and the complete non-HEVC HEIF/AV1 namespace passes 9,317 of 9,317 through foreground VSTest. Current official libaom `main` at `d565eec60f084421fa34fc0534b760c6452b6a6c` accepts all 66 raw AV1 payloads regenerated by that suite. Inter-picture reference search, sequence Exif/XMP, grids, and orientation remain open.
- [~] Write the correct AVIF file type, item information, locations, references, properties, AV1 configuration, dimensions, color, alpha, metadata, and media data. - [~] Write the correct AVIF file type, item information, locations, references, properties, AV1 configuration, dimensions, color, alpha, metadata, and media data.
- [~] Support single images, alpha auxiliary images, grids, multiple extents, and bounded image sequences in the final public scope. - [~] Support single images, alpha auxiliary images, grids, multiple extents, and bounded image sequences in the final public scope.
- [x] Preserve ICC, Exif, and XMP according to encoder options. - [x] Preserve ICC, Exif, and XMP according to encoder options.

888
src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs

@ -0,0 +1,888 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using System.Buffers.Binary;
using System.Text;
using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.IO;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata.Profiles.Cicp;
using SixLabors.ImageSharp.Metadata.Profiles.Icc;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp.Formats.Heif;
internal sealed partial class HeifEncoderCore
{
private const uint DefaultSequenceTimescale = 1000;
private const uint FallbackSequenceTimescale = 1000000;
private const uint UnityFixed16Point16 = 1U << 16;
private const uint UnityFixed2Point30 = 1U << 30;
private const ushort UnityFixed8Point8 = 1 << 8;
private const ushort PackedUndeterminedLanguage = 0x55C4;
private Av1EncodingSettings ResolveAv1Encoding<TPixel>(Image<TPixel> image)
where TPixel : unmanaged, IPixel<TPixel>
{
HeifMetadata metadata = image.Metadata.GetHeifMetadata();
HeifBitDepth bitDepth = this.encoder.BitDepth ?? metadata.BitDepth;
Av1BitDepth av1BitDepth = bitDepth switch
{
HeifBitDepth.Bit8 => Av1BitDepth.EightBit,
HeifBitDepth.Bit10 => Av1BitDepth.TenBit,
HeifBitDepth.Bit12 => Av1BitDepth.TwelveBit,
_ => throw new NotSupportedException($"HEIF bit depth '{bitDepth}' is not supported.")
};
HeifChromaSubsampling defaultChromaSubsampling = this.encoder.Lossless
? HeifChromaSubsampling.Yuv444
: HeifChromaSubsampling.Yuv420;
HeifChromaSubsampling chromaSubsampling = this.encoder.ChromaSubsampling ??
(metadata.IsMonochrome ? HeifChromaSubsampling.Monochrome : defaultChromaSubsampling);
(bool isMonochrome, bool subsamplingX, bool subsamplingY) = chromaSubsampling switch
{
HeifChromaSubsampling.Monochrome => (true, true, true),
HeifChromaSubsampling.Yuv420 => (false, true, true),
HeifChromaSubsampling.Yuv422 => (false, true, false),
HeifChromaSubsampling.Yuv444 => (false, false, false),
_ => throw new NotSupportedException($"HEIF chroma sampling '{chromaSubsampling}' is not supported.")
};
CicpProfile? sourceColorProfile = image.Metadata.CicpProfile;
CicpProfile colorProfile;
if (sourceColorProfile is null)
{
colorProfile = new CicpProfile(2, 2, 6, false);
}
else
{
bool identityMatrix = sourceColorProfile.MatrixCoefficients == CicpMatrixCoefficients.Identity;
bool legalIdentityMatrix = !isMonochrome
&& chromaSubsampling == HeifChromaSubsampling.Yuv444
&& sourceColorProfile.ColorPrimaries == CicpColorPrimaries.ItuRBt709_6
&& sourceColorProfile.TransferCharacteristics == CicpTransferCharacteristics.Iec61966_2_1;
if (sourceColorProfile.MatrixCoefficients == CicpMatrixCoefficients.Unspecified
|| (identityMatrix && !legalIdentityMatrix))
{
// The converter uses BT.601 for unspecified or incompatible identity signaling, so record that actual matrix.
colorProfile = new CicpProfile(
(byte)sourceColorProfile.ColorPrimaries,
(byte)sourceColorProfile.TransferCharacteristics,
(byte)CicpMatrixCoefficients.ItuRBt601_7_525,
sourceColorProfile.FullRange);
}
else if (identityMatrix && !sourceColorProfile.FullRange)
{
colorProfile = new CicpProfile(
(byte)sourceColorProfile.ColorPrimaries,
(byte)sourceColorProfile.TransferCharacteristics,
(byte)sourceColorProfile.MatrixCoefficients,
true);
}
else
{
colorProfile = sourceColorProfile;
}
}
ObuColorConfig colorConfig = new()
{
IsColorDescriptionPresent = true,
IsMonochrome = isMonochrome,
ColorPrimaries = (ObuColorPrimaries)colorProfile.ColorPrimaries,
TransferCharacteristics = (ObuTransferCharacteristics)colorProfile.TransferCharacteristics,
MatrixCoefficients = (ObuMatrixCoefficients)colorProfile.MatrixCoefficients,
ColorRange = colorProfile.FullRange,
SubSamplingX = subsamplingX,
SubSamplingY = subsamplingY,
ChromaSamplePosition = ObuChromoSamplePosition.Unknown,
BitDepth = av1BitDepth
};
ObuColorConfig alphaConfig = new()
{
IsMonochrome = true,
ColorRange = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = av1BitDepth
};
int quality = this.encoder.Quality ?? 75;
int colorQIndex = this.encoder.Lossless ? 0 : GetAv1QuantizerIndex(quality);
int alphaQuality = this.encoder.AlphaQuality ?? quality;
int alphaQIndex = this.encoder.Lossless ? 0 : GetAv1QuantizerIndex(alphaQuality);
bool hasAlpha = TPixel.GetPixelTypeInfo().AlphaRepresentation != PixelAlphaRepresentation.None;
return new Av1EncodingSettings(
bitDepth,
chromaSubsampling,
colorProfile,
colorConfig,
alphaConfig,
colorQIndex,
alphaQIndex,
hasAlpha);
}
private HeifSequenceEncoding CompressAv1Sequence<TPixel>(
Image<TPixel> image,
ChunkedMemoryStream stream,
Av1EncodingSettings settings,
CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel>
{
if (image.Width > ushort.MaxValue || image.Height > ushort.MaxValue)
{
throw new NotSupportedException("AV1 image-sequence dimensions cannot exceed 65535 pixels.");
}
int frameCount = image.Frames.Count;
uint timescale = GetSequenceTimescale(image);
int sampleCount = checked(frameCount * (settings.HasAlpha ? 2 : 1));
// The container needs only offset, length, and duration after each frame is streamed. Color and alpha
// share one compact table, with each track occupying one contiguous slice for its complete operation lifetime.
HeifSequenceSampleInfo[] samples = new HeifSequenceSampleInfo[sampleCount];
Span<HeifSequenceSampleInfo> colorSamples = samples.AsSpan(0, frameCount);
ImageFrame<TPixel> rootFrame = image.Frames.RootFrame;
uint duration = GetSequenceSampleDuration(rootFrame.Metadata.GetHeifMetadata().FrameDelay, timescale);
cancellationToken.ThrowIfCancellationRequested();
long colorOffset = stream.Length;
ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode(
this.configuration,
rootFrame,
stream,
settings.ColorConfig,
settings.ColorQIndex,
this.encoder.Effort);
colorSamples[0] = new HeifSequenceSampleInfo(
colorOffset,
checked((int)(stream.Length - colorOffset)),
duration);
for (int frameIndex = 1; frameIndex < frameCount; frameIndex++)
{
cancellationToken.ThrowIfCancellationRequested();
ImageFrame<TPixel> frame = image.Frames[frameIndex];
duration = GetSequenceSampleDuration(frame.Metadata.GetHeifMetadata().FrameDelay, timescale);
colorOffset = stream.Length;
_ = Av1FrameEncoder.Encode(
this.configuration,
frame,
stream,
settings.ColorConfig,
settings.ColorQIndex,
this.encoder.Effort);
colorSamples[frameIndex] = new HeifSequenceSampleInfo(
colorOffset,
checked((int)(stream.Length - colorOffset)),
duration);
}
HeifSequenceTrackEncoding colorTrack = new(
new Av1CodecConfiguration(colorHeader),
samples,
0,
frameCount,
false);
HeifSequenceTrackEncoding? alphaTrack = null;
if (settings.HasAlpha)
{
Span<HeifSequenceSampleInfo> alphaSamples = samples.AsSpan(frameCount, frameCount);
cancellationToken.ThrowIfCancellationRequested();
long alphaOffset = stream.Length;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha(
this.configuration,
rootFrame,
stream,
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
alphaSamples[0] = new HeifSequenceSampleInfo(
alphaOffset,
checked((int)(stream.Length - alphaOffset)),
colorSamples[0].Duration);
for (int frameIndex = 1; frameIndex < frameCount; frameIndex++)
{
cancellationToken.ThrowIfCancellationRequested();
alphaOffset = stream.Length;
_ = Av1FrameEncoder.EncodeAlpha(
this.configuration,
image.Frames[frameIndex],
stream,
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
alphaSamples[frameIndex] = new HeifSequenceSampleInfo(
alphaOffset,
checked((int)(stream.Length - alphaOffset)),
colorSamples[frameIndex].Duration);
}
alphaTrack = new HeifSequenceTrackEncoding(
new Av1CodecConfiguration(alphaHeader),
samples,
frameCount,
frameCount,
true);
}
return new HeifSequenceEncoding(
image.Width,
image.Height,
image.Metadata.GetHeifMetadata().RepeatCount,
timescale,
colorTrack,
alphaTrack,
settings.ColorProfile,
this.encoder.SkipMetadata ? null : image.Metadata.IccProfile);
}
private int WriteSequenceFileTypeBox(Stream stream)
{
Span<byte> buffer = stackalloc byte[32];
int bytesWritten = WriteBoxHeader(buffer, Heif4CharCode.Ftyp);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], 0);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avif);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Msf1);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Iso8);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avio);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer, (uint)bytesWritten);
stream.Write(buffer[..bytesWritten]);
return bytesWritten;
}
private void WriteSequenceMovieBox(HeifSequenceEncoding sequence, int fileTypeLength, Stream stream)
{
// Chunk offsets point past the completed movie box, so retain only this bounded metadata box and patch
// its two offsets once its size is known. The encoded frame payload remains in allocator-backed chunks.
using AutoExpandingMemory<byte> memory = new(this.configuration, 0x1000);
int offset = 0;
int movieStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Moov);
ulong mediaDuration = GetSequenceMediaDuration(sequence.ColorTrack.Samples);
ulong trackDuration = sequence.RepeatCount == 0
? ulong.MaxValue
: checked(mediaDuration * sequence.RepeatCount);
bool hasAlpha = sequence.AlphaTrack.HasValue;
WriteSequenceMovieHeader(
memory,
ref offset,
sequence.Timescale,
trackDuration,
hasAlpha ? 3U : 2U);
int colorChunkOffsetPosition = WriteSequenceTrack(
memory,
ref offset,
sequence,
sequence.ColorTrack,
1,
mediaDuration,
trackDuration);
int alphaChunkOffsetPosition = -1;
long alphaPayloadOffset = 0;
if (hasAlpha)
{
HeifSequenceTrackEncoding alphaTrack = sequence.AlphaTrack.GetValueOrDefault();
alphaPayloadOffset = alphaTrack.Samples[0].Offset;
alphaChunkOffsetPosition = WriteSequenceTrack(
memory,
ref offset,
sequence,
alphaTrack,
2,
mediaDuration,
trackDuration);
}
EndSequenceBox(memory, movieStart, offset);
ulong mediaDataOffset = checked((ulong)fileTypeLength + (uint)offset + 8U);
Span<byte> movie = memory.GetSpan(offset);
BinaryPrimitives.WriteUInt64BigEndian(
movie[colorChunkOffsetPosition..],
checked(mediaDataOffset + (ulong)sequence.ColorTrack.Samples[0].Offset));
if (alphaChunkOffsetPosition >= 0)
{
BinaryPrimitives.WriteUInt64BigEndian(
movie[alphaChunkOffsetPosition..],
checked(mediaDataOffset + (ulong)alphaPayloadOffset));
}
stream.Write(movie);
}
private static void WriteSequenceMovieHeader(
AutoExpandingMemory<byte> memory,
ref int offset,
uint timescale,
ulong duration,
uint nextTrackId)
{
int movieHeaderStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Mvhd);
WriteSequenceFullBoxHeader(memory, ref offset, 1, 0);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, timescale);
WriteSequenceUInt64(memory, ref offset, duration);
WriteSequenceUInt32(memory, ref offset, UnityFixed16Point16);
WriteSequenceUInt16(memory, ref offset, UnityFixed8Point8);
WriteSequenceUInt16(memory, ref offset, 0);
WriteSequenceZeros(memory, ref offset, 2 * sizeof(uint));
WriteSequenceIdentityMatrix(memory, ref offset);
WriteSequenceZeros(memory, ref offset, 6 * sizeof(uint));
WriteSequenceUInt32(memory, ref offset, nextTrackId);
EndSequenceBox(memory, movieHeaderStart, offset);
}
private static int WriteSequenceTrack(
AutoExpandingMemory<byte> memory,
ref int offset,
HeifSequenceEncoding sequence,
HeifSequenceTrackEncoding track,
uint trackId,
ulong mediaDuration,
ulong trackDuration)
{
int trackStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Trak);
WriteSequenceTrackHeader(
memory,
ref offset,
sequence.Width,
sequence.Height,
trackId,
trackDuration);
if (track.IsAlpha)
{
WriteSequenceTrackReference(memory, ref offset, Heif4CharCode.Auxl, 1);
}
if (sequence.RepeatCount != 1)
{
WriteSequenceEditList(memory, ref offset, mediaDuration);
}
int mediaStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Mdia);
WriteSequenceMediaHeader(memory, ref offset, sequence.Timescale, mediaDuration);
WriteSequenceHandler(memory, ref offset, track.IsAlpha ? Heif4CharCode.Auxv : Heif4CharCode.Pict);
int mediaInformationStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Minf);
WriteSequenceDataInformation(memory, ref offset);
int chunkOffsetPosition = WriteSequenceSampleTable(memory, ref offset, sequence, track);
EndSequenceBox(memory, mediaInformationStart, offset);
EndSequenceBox(memory, mediaStart, offset);
EndSequenceBox(memory, trackStart, offset);
return chunkOffsetPosition;
}
private static void WriteSequenceTrackHeader(
AutoExpandingMemory<byte> memory,
ref int offset,
int width,
int height,
uint trackId,
ulong duration)
{
int trackHeaderStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Tkhd);
WriteSequenceFullBoxHeader(memory, ref offset, 1, 1);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, trackId);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt64(memory, ref offset, duration);
WriteSequenceZeros(memory, ref offset, (2 * sizeof(uint)) + (4 * sizeof(ushort)));
WriteSequenceIdentityMatrix(memory, ref offset);
WriteSequenceUInt32(memory, ref offset, checked((uint)width << 16));
WriteSequenceUInt32(memory, ref offset, checked((uint)height << 16));
EndSequenceBox(memory, trackHeaderStart, offset);
}
private static void WriteSequenceTrackReference(
AutoExpandingMemory<byte> memory,
ref int offset,
Heif4CharCode referenceType,
uint referencedTrackId)
{
int referencesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Tref);
int referenceStart = BeginSequenceBox(memory, ref offset, referenceType);
WriteSequenceUInt32(memory, ref offset, referencedTrackId);
EndSequenceBox(memory, referenceStart, offset);
EndSequenceBox(memory, referencesStart, offset);
}
private static void WriteSequenceEditList(
AutoExpandingMemory<byte> memory,
ref int offset,
ulong mediaDuration)
{
int editStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Edts);
int editListStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Elst);
WriteSequenceFullBoxHeader(memory, ref offset, 1, 1);
WriteSequenceUInt32(memory, ref offset, 1);
WriteSequenceUInt64(memory, ref offset, mediaDuration);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt16(memory, ref offset, 1);
WriteSequenceUInt16(memory, ref offset, 0);
EndSequenceBox(memory, editListStart, offset);
EndSequenceBox(memory, editStart, offset);
}
private static void WriteSequenceMediaHeader(
AutoExpandingMemory<byte> memory,
ref int offset,
uint timescale,
ulong mediaDuration)
{
int mediaHeaderStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Mdhd);
WriteSequenceFullBoxHeader(memory, ref offset, 1, 0);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt64(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, timescale);
WriteSequenceUInt64(memory, ref offset, mediaDuration);
WriteSequenceUInt16(memory, ref offset, PackedUndeterminedLanguage);
WriteSequenceUInt16(memory, ref offset, 0);
EndSequenceBox(memory, mediaHeaderStart, offset);
}
private static void WriteSequenceHandler(
AutoExpandingMemory<byte> memory,
ref int offset,
Heif4CharCode handlerType)
{
int handlerStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Hdlr);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, (uint)handlerType);
WriteSequenceZeros(memory, ref offset, 12);
memory.GetSpan(offset++, 1)[0] = 0;
EndSequenceBox(memory, handlerStart, offset);
}
private static void WriteSequenceDataInformation(AutoExpandingMemory<byte> memory, ref int offset)
{
int dataInformationStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Dinf);
int dataReferenceStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Dref);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 1);
int locationStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Url);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 1);
EndSequenceBox(memory, locationStart, offset);
EndSequenceBox(memory, dataReferenceStart, offset);
EndSequenceBox(memory, dataInformationStart, offset);
}
private static int WriteSequenceSampleTable(
AutoExpandingMemory<byte> memory,
ref int offset,
HeifSequenceEncoding sequence,
HeifSequenceTrackEncoding track)
{
int sampleTableStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stbl);
WriteSequenceSampleDescription(memory, ref offset, sequence, track);
WriteSequenceSampleTiming(memory, ref offset, track.Samples);
// Payloads are emitted contiguously per track, so one chunk maps directly to every sample in that track.
int sampleToChunkStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stsc);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 1);
WriteSequenceUInt32(memory, ref offset, 1);
WriteSequenceUInt32(memory, ref offset, checked((uint)track.Samples.Length));
WriteSequenceUInt32(memory, ref offset, 1);
EndSequenceBox(memory, sampleToChunkStart, offset);
int sampleSizesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stsz);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, checked((uint)track.Samples.Length));
foreach (HeifSequenceSampleInfo sample in track.Samples)
{
WriteSequenceUInt32(memory, ref offset, checked((uint)sample.Length));
}
EndSequenceBox(memory, sampleSizesStart, offset);
int chunkOffsetsStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Co64);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 1);
int chunkOffsetPosition = offset;
WriteSequenceUInt64(memory, ref offset, 0);
EndSequenceBox(memory, chunkOffsetsStart, offset);
// The current bounded sequence encoder emits independent all-intra pictures; every sample is seekable.
int syncSamplesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stss);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, checked((uint)track.Samples.Length));
for (uint sampleIndex = 1; sampleIndex <= track.Samples.Length; sampleIndex++)
{
WriteSequenceUInt32(memory, ref offset, sampleIndex);
}
EndSequenceBox(memory, syncSamplesStart, offset);
EndSequenceBox(memory, sampleTableStart, offset);
return chunkOffsetPosition;
}
private static void WriteSequenceSampleDescription(
AutoExpandingMemory<byte> memory,
ref int offset,
HeifSequenceEncoding sequence,
HeifSequenceTrackEncoding track)
{
int descriptionStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stsd);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, 1);
int sampleEntryStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Av01);
WriteSequenceZeros(memory, ref offset, 6);
WriteSequenceUInt16(memory, ref offset, 1);
WriteSequenceZeros(memory, ref offset, (2 * sizeof(ushort)) + (3 * sizeof(uint)));
WriteSequenceUInt16(memory, ref offset, checked((ushort)sequence.Width));
WriteSequenceUInt16(memory, ref offset, checked((ushort)sequence.Height));
WriteSequenceUInt32(memory, ref offset, 72U << 16);
WriteSequenceUInt32(memory, ref offset, 72U << 16);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt16(memory, ref offset, 1);
WriteSequenceZeros(memory, ref offset, 32);
WriteSequenceUInt16(memory, ref offset, 24);
WriteSequenceUInt16(memory, ref offset, ushort.MaxValue);
int configurationStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Av1C);
track.Configuration.WriteFixedHeader(memory.GetSpan(offset, Av1CodecConfiguration.FixedHeaderSize));
offset += Av1CodecConfiguration.FixedHeaderSize;
EndSequenceBox(memory, configurationStart, offset);
if (track.IsAlpha)
{
int auxiliaryTypeStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Auxi);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
int auxiliaryTypeLength = Encoding.UTF8.GetByteCount(HeifConstants.AlphaAuxiliaryType);
Span<byte> auxiliaryType = memory.GetSpan(offset, auxiliaryTypeLength + 1);
offset += Encoding.UTF8.GetBytes(HeifConstants.AlphaAuxiliaryType, auxiliaryType);
memory.GetSpan(offset++, 1)[0] = 0;
EndSequenceBox(memory, auxiliaryTypeStart, offset);
}
else
{
if (sequence.IccProfile is not null)
{
offset += WriteIccColorInformationPropertyBox(memory, offset, sequence.IccProfile);
}
offset += WriteColorInformationPropertyBox(memory, offset, sequence.ColorProfile);
}
int codingConstraintsStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Ccst);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
// Every emitted sequence sample is independently decodable, while intra prediction remains available inside
// each picture. No inter-picture reference slot is therefore advertised.
WriteSequenceUInt32(memory, ref offset, 0xC0000000);
EndSequenceBox(memory, codingConstraintsStart, offset);
EndSequenceBox(memory, sampleEntryStart, offset);
EndSequenceBox(memory, descriptionStart, offset);
}
private static void WriteSequenceSampleTiming(
AutoExpandingMemory<byte> memory,
ref int offset,
ReadOnlySpan<HeifSequenceSampleInfo> samples)
{
// The time-to-sample table stores runs, not one entry per frame. Preserve exact resolved durations while
// combining only adjacent frames whose delays are equal.
int runCount = 1;
for (int sampleIndex = 1; sampleIndex < samples.Length; sampleIndex++)
{
runCount += samples[sampleIndex].Duration == samples[sampleIndex - 1].Duration ? 0 : 1;
}
int timingStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stts);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, checked((uint)runCount));
uint runDuration = samples[0].Duration;
uint runLength = 1;
for (int sampleIndex = 1; sampleIndex <= samples.Length; sampleIndex++)
{
if (sampleIndex < samples.Length && samples[sampleIndex].Duration == runDuration)
{
runLength++;
continue;
}
WriteSequenceUInt32(memory, ref offset, runLength);
WriteSequenceUInt32(memory, ref offset, runDuration);
if (sampleIndex < samples.Length)
{
runDuration = samples[sampleIndex].Duration;
runLength = 1;
}
}
EndSequenceBox(memory, timingStart, offset);
}
private static uint GetSequenceSampleDuration(Rational delay, uint timescale)
{
if (delay.Numerator == 0)
{
return 1;
}
ulong scaledDuration = ((ulong)delay.Numerator * timescale) + (delay.Denominator / 2U);
return checked((uint)Math.Max(1UL, scaledDuration / delay.Denominator));
}
private static uint GetSequenceTimescale<TPixel>(Image<TPixel> image)
where TPixel : unmanaged, IPixel<TPixel>
{
uint timescale = DefaultSequenceTimescale;
foreach (ImageFrame<TPixel> frame in image.Frames)
{
Rational delay = frame.Metadata.GetHeifMetadata().FrameDelay;
if (delay.Numerator == 0)
{
continue;
}
uint commonDivisor = GetGreatestCommonDivisor(timescale, delay.Denominator);
ulong commonTimescale = ((ulong)timescale / commonDivisor) * delay.Denominator;
if (commonTimescale > uint.MaxValue)
{
// A media timescale is a 32-bit field. Microsecond fallback retains bounded timing precision when
// the exact least common multiple of caller-provided rational delays cannot be represented.
return FallbackSequenceTimescale;
}
timescale = (uint)commonTimescale;
}
return timescale;
}
private static uint GetGreatestCommonDivisor(uint left, uint right)
{
while (right != 0)
{
uint remainder = left % right;
left = right;
right = remainder;
}
return left;
}
private static ulong GetSequenceMediaDuration(ReadOnlySpan<HeifSequenceSampleInfo> samples)
{
ulong duration = 0;
foreach (HeifSequenceSampleInfo sample in samples)
{
duration = checked(duration + sample.Duration);
}
return duration;
}
private static int BeginSequenceBox(
AutoExpandingMemory<byte> memory,
ref int offset,
Heif4CharCode type)
{
// Reserve the size field now and patch it at the matching EndSequenceBox call after nested boxes expand.
int start = offset;
offset += WriteBoxHeader(memory.GetSpan(offset, 8), type);
return start;
}
private static void EndSequenceBox(AutoExpandingMemory<byte> memory, int start, int offset)
=> BinaryPrimitives.WriteUInt32BigEndian(
memory.GetSpan(start, sizeof(uint)),
checked((uint)(offset - start)));
private static void WriteSequenceFullBoxHeader(
AutoExpandingMemory<byte> memory,
ref int offset,
byte version,
uint flags)
{
Span<byte> destination = memory.GetSpan(offset, sizeof(uint));
BinaryPrimitives.WriteUInt32BigEndian(destination, flags);
destination[0] = version;
offset += sizeof(uint);
}
private static void WriteSequenceIdentityMatrix(AutoExpandingMemory<byte> memory, ref int offset)
{
WriteSequenceUInt32(memory, ref offset, UnityFixed16Point16);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, UnityFixed16Point16);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, 0);
WriteSequenceUInt32(memory, ref offset, UnityFixed2Point30);
}
private static void WriteSequenceZeros(AutoExpandingMemory<byte> memory, ref int offset, int length)
{
memory.GetSpan(offset, length).Clear();
offset += length;
}
private static void WriteSequenceUInt16(AutoExpandingMemory<byte> memory, ref int offset, ushort value)
{
BinaryPrimitives.WriteUInt16BigEndian(memory.GetSpan(offset, sizeof(ushort)), value);
offset += sizeof(ushort);
}
private static void WriteSequenceUInt32(AutoExpandingMemory<byte> memory, ref int offset, uint value)
{
BinaryPrimitives.WriteUInt32BigEndian(memory.GetSpan(offset, sizeof(uint)), value);
offset += sizeof(uint);
}
private static void WriteSequenceUInt64(AutoExpandingMemory<byte> memory, ref int offset, ulong value)
{
BinaryPrimitives.WriteUInt64BigEndian(memory.GetSpan(offset, sizeof(ulong)), value);
offset += sizeof(ulong);
}
private readonly struct Av1EncodingSettings
{
public Av1EncodingSettings(
HeifBitDepth bitDepth,
HeifChromaSubsampling chromaSubsampling,
CicpProfile colorProfile,
ObuColorConfig colorConfig,
ObuColorConfig alphaConfig,
int colorQIndex,
int alphaQIndex,
bool hasAlpha)
{
this.BitDepth = bitDepth;
this.ChromaSubsampling = chromaSubsampling;
this.ColorProfile = colorProfile;
this.ColorConfig = colorConfig;
this.AlphaConfig = alphaConfig;
this.ColorQIndex = colorQIndex;
this.AlphaQIndex = alphaQIndex;
this.HasAlpha = hasAlpha;
}
public HeifBitDepth BitDepth { get; }
public HeifChromaSubsampling ChromaSubsampling { get; }
public CicpProfile ColorProfile { get; }
public ObuColorConfig ColorConfig { get; }
public ObuColorConfig AlphaConfig { get; }
public int ColorQIndex { get; }
public int AlphaQIndex { get; }
public bool HasAlpha { get; }
}
private readonly struct HeifSequenceSampleInfo
{
public HeifSequenceSampleInfo(long offset, int length, uint duration)
{
this.Offset = offset;
this.Length = length;
this.Duration = duration;
}
public long Offset { get; }
public int Length { get; }
public uint Duration { get; }
}
private readonly struct HeifSequenceEncoding
{
public HeifSequenceEncoding(
int width,
int height,
ushort repeatCount,
uint timescale,
HeifSequenceTrackEncoding colorTrack,
HeifSequenceTrackEncoding? alphaTrack,
CicpProfile colorProfile,
IccProfile? iccProfile)
{
this.Width = width;
this.Height = height;
this.RepeatCount = repeatCount;
this.Timescale = timescale;
this.ColorTrack = colorTrack;
this.AlphaTrack = alphaTrack;
this.ColorProfile = colorProfile;
this.IccProfile = iccProfile;
}
public int Width { get; }
public int Height { get; }
public ushort RepeatCount { get; }
public uint Timescale { get; }
public HeifSequenceTrackEncoding ColorTrack { get; }
public HeifSequenceTrackEncoding? AlphaTrack { get; }
public CicpProfile ColorProfile { get; }
public IccProfile? IccProfile { get; }
}
private readonly struct HeifSequenceTrackEncoding
{
private readonly HeifSequenceSampleInfo[] samples;
private readonly int sampleOffset;
private readonly int sampleCount;
public HeifSequenceTrackEncoding(
Av1CodecConfiguration configuration,
HeifSequenceSampleInfo[] samples,
int sampleOffset,
int sampleCount,
bool isAlpha)
{
this.Configuration = configuration;
this.samples = samples;
this.sampleOffset = sampleOffset;
this.sampleCount = sampleCount;
this.IsAlpha = isAlpha;
}
public Av1CodecConfiguration Configuration { get; }
public ReadOnlySpan<HeifSequenceSampleInfo> Samples
=> this.samples.AsSpan(this.sampleOffset, this.sampleCount);
public bool IsAlpha { get; }
}
}

136
src/ImageSharp/Formats/Heif/HeifEncoderCore.cs

@ -18,7 +18,7 @@ namespace SixLabors.ImageSharp.Formats.Heif;
/// <summary> /// <summary>
/// Image encoder for writing an image to a stream as a HEIF image. /// Image encoder for writing an image to a stream as a HEIF image.
/// </summary> /// </summary>
internal sealed class HeifEncoderCore internal sealed partial class HeifEncoderCore
{ {
/// <summary> /// <summary>
/// The global configuration. /// The global configuration.
@ -54,9 +54,25 @@ internal sealed class HeifEncoderCore
Guard.NotNull(image, nameof(image)); Guard.NotNull(image, nameof(image));
Guard.NotNull(stream, nameof(stream)); Guard.NotNull(stream, nameof(stream));
using ChunkedMemoryStream compressedPixels = new(this.configuration.MemoryAllocator);
if (this.encoder.CompressionMethod == HeifCompressionMethod.Av1 && image.Frames.Count > 1)
{
Av1EncodingSettings settings = this.ResolveAv1Encoding(image);
HeifSequenceEncoding sequence = this.CompressAv1Sequence(
image,
compressedPixels,
settings,
cancellationToken);
int fileTypeLength = this.WriteSequenceFileTypeBox(stream);
this.WriteSequenceMovieBox(sequence, fileTypeLength, stream);
this.WriteMediaDataBox(compressedPixels, stream);
stream.Flush();
return;
}
List<HeifItem> items = new(); List<HeifItem> items = new();
List<HeifItemLink> links = new(); List<HeifItemLink> links = new();
using ChunkedMemoryStream compressedPixels = new(this.configuration.MemoryAllocator);
switch (this.encoder.CompressionMethod) switch (this.encoder.CompressionMethod)
{ {
case HeifCompressionMethod.LegacyJpeg: case HeifCompressionMethod.LegacyJpeg:
@ -786,138 +802,42 @@ internal sealed class HeifEncoderCore
CancellationToken cancellationToken) CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
if (image.Frames.Count != 1) Av1EncodingSettings settings = this.ResolveAv1Encoding(image);
{
throw new NotSupportedException("AV1 image-sequence encoding is not implemented.");
}
HeifMetadata metadata = image.Metadata.GetHeifMetadata();
HeifBitDepth bitDepth = this.encoder.BitDepth ?? metadata.BitDepth;
Av1BitDepth av1BitDepth = bitDepth switch
{
HeifBitDepth.Bit8 => Av1BitDepth.EightBit,
HeifBitDepth.Bit10 => Av1BitDepth.TenBit,
HeifBitDepth.Bit12 => Av1BitDepth.TwelveBit,
_ => throw new NotSupportedException($"HEIF bit depth '{bitDepth}' is not supported.")
};
HeifChromaSubsampling defaultChromaSubsampling = this.encoder.Lossless
? HeifChromaSubsampling.Yuv444
: HeifChromaSubsampling.Yuv420;
HeifChromaSubsampling chromaSubsampling = this.encoder.ChromaSubsampling ??
(metadata.IsMonochrome ? HeifChromaSubsampling.Monochrome : defaultChromaSubsampling);
(bool isMonochrome, bool subsamplingX, bool subsamplingY) = chromaSubsampling switch
{
HeifChromaSubsampling.Monochrome => (true, true, true),
HeifChromaSubsampling.Yuv420 => (false, true, true),
HeifChromaSubsampling.Yuv422 => (false, true, false),
HeifChromaSubsampling.Yuv444 => (false, false, false),
_ => throw new NotSupportedException($"HEIF chroma sampling '{chromaSubsampling}' is not supported.")
};
CicpProfile? sourceColorProfile = image.Metadata.CicpProfile;
CicpProfile colorProfile;
if (sourceColorProfile is null)
{
colorProfile = new CicpProfile(2, 2, 6, false);
}
else
{
bool identityMatrix = sourceColorProfile.MatrixCoefficients == CicpMatrixCoefficients.Identity;
bool legalIdentityMatrix = !isMonochrome
&& chromaSubsampling == HeifChromaSubsampling.Yuv444
&& sourceColorProfile.ColorPrimaries == CicpColorPrimaries.ItuRBt709_6
&& sourceColorProfile.TransferCharacteristics == CicpTransferCharacteristics.Iec61966_2_1;
if (sourceColorProfile.MatrixCoefficients == CicpMatrixCoefficients.Unspecified
|| (identityMatrix && !legalIdentityMatrix))
{
// The converter uses BT.601 for unspecified or incompatible identity signaling, so record that actual matrix.
colorProfile = new CicpProfile(
(byte)sourceColorProfile.ColorPrimaries,
(byte)sourceColorProfile.TransferCharacteristics,
(byte)CicpMatrixCoefficients.ItuRBt601_7_525,
sourceColorProfile.FullRange);
}
else if (identityMatrix && !sourceColorProfile.FullRange)
{
colorProfile = new CicpProfile(
(byte)sourceColorProfile.ColorPrimaries,
(byte)sourceColorProfile.TransferCharacteristics,
(byte)sourceColorProfile.MatrixCoefficients,
true);
}
else
{
colorProfile = sourceColorProfile;
}
}
ObuColorConfig colorConfig = new()
{
IsColorDescriptionPresent = true,
IsMonochrome = isMonochrome,
ColorPrimaries = (ObuColorPrimaries)colorProfile.ColorPrimaries,
TransferCharacteristics = (ObuTransferCharacteristics)colorProfile.TransferCharacteristics,
MatrixCoefficients = (ObuMatrixCoefficients)colorProfile.MatrixCoefficients,
ColorRange = colorProfile.FullRange,
SubSamplingX = subsamplingX,
SubSamplingY = subsamplingY,
ChromaSamplePosition = ObuChromoSamplePosition.Unknown,
BitDepth = av1BitDepth
};
int quality = this.encoder.Quality ?? 75;
int qIndex = this.encoder.Lossless ? 0 : GetAv1QuantizerIndex(quality);
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode( ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode(
this.configuration, this.configuration,
image.Frames.RootFrame, image.Frames.RootFrame,
stream, stream,
colorConfig, settings.ColorConfig,
qIndex, settings.ColorQIndex,
this.encoder.Effort); this.encoder.Effort);
long colorLength = stream.Length; long colorLength = stream.Length;
byte channelBitDepth = (byte)bitDepth; byte channelBitDepth = (byte)settings.BitDepth;
HeifItem colorItem = new(Heif4CharCode.Av01, 1) HeifItem colorItem = new(Heif4CharCode.Av01, 1)
{ {
ChannelCount = isMonochrome ? 1 : 3, ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3,
UniformChannelBitDepth = channelBitDepth, UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth * (isMonochrome ? 1 : 3), BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3),
Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader), Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader),
IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile, IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile,
CicpProfile = colorProfile CicpProfile = settings.ColorProfile
}; };
colorItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, 0L, colorLength)); colorItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, 0L, colorLength));
colorItem.SetExtent(image.Size); colorItem.SetExtent(image.Size);
items.Add(colorItem); items.Add(colorItem);
bool hasAlpha = TPixel.GetPixelTypeInfo().AlphaRepresentation != PixelAlphaRepresentation.None; if (settings.HasAlpha)
if (hasAlpha)
{ {
ObuColorConfig alphaConfig = new()
{
IsMonochrome = true,
ColorRange = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = av1BitDepth
};
int alphaQuality = this.encoder.AlphaQuality ?? quality;
int alphaQIndex = this.encoder.Lossless ? 0 : GetAv1QuantizerIndex(alphaQuality);
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
long alphaOffset = stream.Length; long alphaOffset = stream.Length;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha( ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha(
this.configuration, this.configuration,
image.Frames.RootFrame, image.Frames.RootFrame,
stream, stream,
alphaConfig, settings.AlphaConfig,
alphaQIndex, settings.AlphaQIndex,
this.encoder.Effort); this.encoder.Effort);
long alphaLength = stream.Length - alphaOffset; long alphaLength = stream.Length - alphaOffset;

86
tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs

@ -313,16 +313,90 @@ public class HeifEncoderTests
private static ushort ExpandToUShort(int sample, int maximum) private static ushort ExpandToUShort(int sample, int maximum)
=> (ushort)(((sample * (long)ushort.MaxValue) + (maximum / 2)) / maximum); => (ushort)(((sample * (long)ushort.MaxValue) + (maximum / 2)) / maximum);
[Fact] [Theory]
public void Av1RejectsImageSequenceBeforeWritingOutput() [InlineData((ushort)0)]
[InlineData((ushort)3)]
public void Av1LosslessImageSequencePreservesFramesTimingAndAlpha(ushort repeatCount)
{ {
using Image<Rgb24> image = new(1, 1); const int width = 8;
const int height = 8;
const int frameCount = 3;
using Image<Rgba32> image = new(width, height);
image.Frames.AddFrame(image.Frames.RootFrame);
image.Frames.AddFrame(image.Frames.RootFrame); image.Frames.AddFrame(image.Frames.RootFrame);
for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
{
ImageFrame<Rgba32> frame = image.Frames[frameIndex];
frame.Metadata.GetHeifMetadata().FrameDelay = frameIndex switch
{
0 => new Rational(1, 24),
1 => new Rational(1, 25),
_ => new Rational(1, 30)
};
for (int row = 0; row < height; row++)
{
Span<Rgba32> pixels = frame.PixelBuffer.DangerousGetRowSpan(row);
for (int column = 0; column < width; column++)
{
pixels[column] = new Rgba32(
(byte)((frameIndex * 53) + (column * 19) + row),
(byte)((frameIndex * 31) + (row * 23) + column),
(byte)((frameIndex * 71) + (column * 7) + (row * 13)),
(byte)((frameIndex * 47) + (column * 17) + (row * 11)));
}
}
}
image.Metadata.CicpProfile = new CicpProfile(1, 13, 0, true);
image.Metadata.GetHeifMetadata().RepeatCount = repeatCount;
using MemoryStream stream = new(); using MemoryStream stream = new();
HeifEncoder encoder = new() { CompressionMethod = HeifCompressionMethod.Av1 }; HeifEncoder encoder = new()
{
CompressionMethod = HeifCompressionMethod.Av1,
Lossless = true,
Effort = 0
};
Assert.Throws<NotSupportedException>(() => image.Save(stream, encoder)); image.Save(stream, encoder);
Assert.Equal(0, stream.Length); byte[] file = stream.ToArray();
Assert.Equal((uint)Heif4CharCode.Avis, BinaryPrimitives.ReadUInt32BigEndian(file.AsSpan(8)));
stream.Position = 0;
using Image<Rgba32> decoded = Image.Load<Rgba32>(stream);
Assert.Equal(frameCount, decoded.Frames.Count);
Assert.Equal(repeatCount, decoded.Metadata.GetHeifMetadata().RepeatCount);
Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded));
for (int frameIndex = 0; frameIndex < frameCount; frameIndex++)
{
Assert.Equal(
image.Frames[frameIndex].Metadata.GetHeifMetadata().FrameDelay,
decoded.Frames[frameIndex].Metadata.GetHeifMetadata().FrameDelay);
}
}
[Fact]
public void Av1ImageSequenceWritesToPrefixedNonSeekableStream()
{
using Image<Rgb24> image = new(8, 8);
image.Frames.AddFrame(image.Frames.RootFrame);
image.Frames.RootFrame.Metadata.GetHeifMetadata().FrameDelay = new Rational(1, 10);
image.Frames[1].Metadata.GetHeifMetadata().FrameDelay = new Rational(1, 20);
using MemoryStream storage = new();
storage.Write([1, 2, 3, 4]);
long fileStart = storage.Position;
using NonSeekableStream destination = new(storage);
HeifEncoder encoder = new()
{
CompressionMethod = HeifCompressionMethod.Av1,
Effort = 0
};
image.Save(destination, encoder);
storage.Position = fileStart;
using Image<Rgb24> decoded = Image.Load<Rgb24>(storage);
Assert.Equal(image.Size, decoded.Size);
Assert.Equal(image.Frames.Count, decoded.Frames.Count);
} }
[Theory] [Theory]

Loading…
Cancel
Save