From d401064cadb695bc75497533d242b4354d09cb57 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 24 Aug 2026 19:08:21 +1000 Subject: [PATCH] Handle HEIF container extensions --- HEIF_IMPLEMENTATION_PLAN.md | 2 +- .../Formats/Heif/HeifDecoderCore.cs | 17 ++- .../Formats/Heif/HeifDecoderTests.cs | 127 ++++++++++++++++++ 3 files changed, 142 insertions(+), 4 deletions(-) diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 443eeb6891..11327e4d5c 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -62,7 +62,7 @@ This snapshot pins or classifies the available references and failures; it does | `Av1FrameInfo`, `Av1TileReader`, and `Av1BlockDecoder` transform/coefficient storage | AV1 section 5.11.39 coefficient syntax and section 7.11.2 reconstruction | libaom `av1/decoder/decodetxb.c` and `av1/decoder/decoder.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Preserve separate luma and chroma transform coefficients at monotonically advancing per-plane offsets within each superblock so reconstruction consumes the same transform-block order produced by tile parsing. | | `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. | | `Av1Inverse2dTransformer` and `Av1InverseTransformerFactory` | AV1 section 7.11.2 inverse transform and reconstruction | libaom `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, and `av1/common/idct.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar transform oracle for coefficient-row traversal, intermediate layout, stage ranges, clipping, and high-bit-depth sample addition. The managed 16-bit overload is also used as a parity oracle for the byte overload. | -| `HeifDecoderCore` and `HeifEncoderCore` item-property containers and associations | ISO/IEC 14496-12 section 8.11.14 item properties and `ipma` syntax | libavif `src/read.c` and `src/write.c` at `092276ce89098ead06db80975173191e5fee1826` | Preserve the position of every property in `ipco`, associate properties by item ID, and read or write the essential bit plus one-based 7-bit or 15-bit property index according to the full-box flags. Independent HEIC, HIF, and AVIF fixtures provide the reader oracle; container-level identification of encoded output guards the writer independently of pixel roundtripping. | +| `HeifDecoderCore` box extension handling and `HeifDecoderCore`/`HeifEncoderCore` item-property associations | ISO/IEC 14496-12 box extensibility and section 8.11.14 item properties and `ipma` syntax | libavif `src/read.c` and `src/write.c` at `092276ce89098ead06db80975173191e5fee1826` | Skip unrecognized top-level and metadata child boxes, preserve the position of every property in `ipco`, reject an unrecognized property only when its item association marks it essential, associate properties by item ID, and read or write the essential bit plus one-based 7-bit or 15-bit property index according to the full-box flags. Independent HEIC, HIF, and AVIF fixtures provide the reader oracle; container-level identification of encoded output guards the writer independently of pixel roundtripping. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 642811bc1a..7f881520a8 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -19,6 +19,8 @@ namespace SixLabors.ImageSharp.Formats.Heif; /// internal sealed class HeifDecoderCore : ImageDecoderCore { + private static readonly object UnknownProperty = new(); + /// /// The general configuration. /// @@ -81,7 +83,8 @@ internal sealed class HeifDecoderCore : ImageDecoderCore stream.Skip((int)(stream.Length - stream.Position)); break; default: - throw new ImageFormatException($"Unknown box type of '{PrettyPrint(boxType)}'"); + SkipBox(stream, boxLength); + break; } } @@ -245,7 +248,8 @@ internal sealed class HeifDecoderCore : ImageDecoderCore SkipBox(stream, length); break; default: - throw new ImageFormatException($"Unknown metadata box type of '{PrettyPrint(boxType)}'"); + SkipBox(stream, length); + break; } } } @@ -484,7 +488,9 @@ internal sealed class HeifDecoderCore : ImageDecoderCore properties.Add(new KeyValuePair(itemType, new object())); break; default: - throw new ImageFormatException($"Unknown item type in property box of '{PrettyPrint(itemType)}'"); + // Unknown properties still occupy an ipco index and become an error only when marked essential. + properties.Add(new KeyValuePair(itemType, UnknownProperty)); + break; } } } @@ -546,6 +552,11 @@ internal sealed class HeifDecoderCore : ImageDecoderCore } KeyValuePair prop = properties[(int)propertyIndex]; + if (essential && ReferenceEquals(prop.Value, UnknownProperty)) + { + throw new InvalidImageContentException($"Item {itemId} associates unknown essential property '{PrettyPrint(prop.Key)}'."); + } + switch (prop.Key) { case Heif4CharCode.Ispe: diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs index 1675801bc3..0aca10f07e 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Buffers.Binary; using SixLabors.ImageSharp.Formats.Heif; using SixLabors.ImageSharp.PixelFormats; @@ -10,6 +11,8 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif; [ValidateDisposedMemoryAllocations] public class HeifDecoderTests { + private const uint UnknownBoxType = 0x74657374U; + [Theory] [InlineData(TestImages.Heif.Image1, HeifCompressionMethod.Hevc, 3992, 2992)] [InlineData(TestImages.Heif.Sample640x427, HeifCompressionMethod.Hevc, 640, 428)] @@ -42,4 +45,128 @@ public class HeifDecoderTests image.CompareToReferenceOutput(provider); Assert.Equal(HeifCompressionMethod.LegacyJpeg, heicMetadata.CompressionMethod); } + + [Fact] + public void DecodeIgnoresUnknownTopLevelBox() + { + byte[] data = CreateEncodedContainer(); + data = InsertBytes(data, data.Length, CreateUnknownBox()); + + using Image image = Image.Load(data); + + Assert.Equal(new Size(2, 3), image.Size); + } + + [Fact] + public void IdentifyIgnoresUnknownMetadataBox() + { + byte[] data = CreateEncodedContainer(); + int metaOffset = FindBoxOffset(data, Heif4CharCode.Meta, 0, data.Length); + int metaSize = (int)BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(metaOffset)); + data = InsertBytes(data, metaOffset + metaSize, CreateUnknownBox()); + IncrementBoxSize(data, metaOffset, 8); + + ImageInfo imageInfo = Image.Identify(data); + + Assert.Equal(new Size(2, 3), imageInfo.Size); + } + + [Fact] + public void IdentifyIgnoresUnknownNonEssentialProperty() + { + byte[] data = CreateContainerWithUnknownProperty(false); + + ImageInfo imageInfo = Image.Identify(data); + + Assert.Equal(new Size(2, 3), imageInfo.Size); + } + + [Fact] + public void IdentifyRejectsUnknownEssentialProperty() + { + byte[] data = CreateContainerWithUnknownProperty(true); + + InvalidImageContentException exception = Assert.Throws(() => Image.Identify(data)); + + Assert.Contains("essential", exception.Message, StringComparison.OrdinalIgnoreCase); + } + + private static byte[] CreateEncodedContainer() + { + using Image image = new(2, 3); + using MemoryStream stream = new(); + image.Save(stream, new HeifEncoder()); + return stream.ToArray(); + } + + private static byte[] CreateContainerWithUnknownProperty(bool essential) + { + byte[] data = CreateEncodedContainer(); + int metaOffset = FindBoxOffset(data, Heif4CharCode.Meta, 0, data.Length); + int metaSize = (int)BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(metaOffset)); + int iprpOffset = FindBoxOffset(data, Heif4CharCode.Iprp, metaOffset + 12, metaSize - 12); + int iprpSize = (int)BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(iprpOffset)); + int ipcoOffset = FindBoxOffset(data, Heif4CharCode.Ipco, iprpOffset + 8, iprpSize - 8); + int ipcoSize = (int)BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(ipcoOffset)); + int ipmaOffset = FindBoxOffset(data, Heif4CharCode.Ipma, iprpOffset + 8, iprpSize - 8); + + // Insert the property before ipma so its one-based index is 2 and all parent box sizes remain explicit. + data = InsertBytes(data, ipcoOffset + ipcoSize, CreateUnknownBox()); + IncrementBoxSize(data, metaOffset, 8); + IncrementBoxSize(data, iprpOffset, 8); + IncrementBoxSize(data, ipcoOffset, 8); + ipmaOffset += 8; + + // The generated container has one item with one property association; append the unknown property to that entry. + int associationCountOffset = ipmaOffset + 18; + data[associationCountOffset]++; + int associationOffset = ipmaOffset + (int)BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(ipmaOffset)); + byte association = (byte)(2 | (essential ? 0x80 : 0)); + data = InsertBytes(data, associationOffset, new byte[] { association }); + IncrementBoxSize(data, metaOffset, 1); + IncrementBoxSize(data, iprpOffset, 1); + IncrementBoxSize(data, ipmaOffset, 1); + return data; + } + + private static byte[] CreateUnknownBox() + { + byte[] box = new byte[8]; + BinaryPrimitives.WriteUInt32BigEndian(box, (uint)box.Length); + BinaryPrimitives.WriteUInt32BigEndian(box.AsSpan(4), UnknownBoxType); + return box; + } + + private static int FindBoxOffset(ReadOnlySpan data, Heif4CharCode type, int offset, int length) + { + int endOffset = offset + length; + while (offset < endOffset) + { + int boxSize = (int)BinaryPrimitives.ReadUInt32BigEndian(data[offset..]); + Heif4CharCode boxType = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(data[(offset + 4)..]); + if (boxType == type) + { + return offset; + } + + offset += boxSize; + } + + return -1; + } + + private static byte[] InsertBytes(byte[] data, int offset, ReadOnlySpan inserted) + { + byte[] result = new byte[data.Length + inserted.Length]; + data.AsSpan(0, offset).CopyTo(result); + inserted.CopyTo(result.AsSpan(offset)); + data.AsSpan(offset).CopyTo(result.AsSpan(offset + inserted.Length)); + return result; + } + + private static void IncrementBoxSize(byte[] data, int offset, int increment) + { + uint size = BinaryPrimitives.ReadUInt32BigEndian(data.AsSpan(offset)); + BinaryPrimitives.WriteUInt32BigEndian(data.AsSpan(offset), size + (uint)increment); + } }