From fcd16c862c437e3249c7e693012c717512985163 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sun, 30 Aug 2026 07:59:58 +1000 Subject: [PATCH] Complete HEVC supplemental presentation handling --- .../Formats/Heif/HeifDecoderCore.cs | 9 +- .../Heif/Hevc/HevcImageItemBitstream.cs | 21 + .../HevcSupplementalEnhancementInformation.cs | 349 +++++++++++++++ .../Formats/Heif/HevcHeifItemDecoder.cs | 148 ++++++- .../Heif/Hevc/HevcPictureDecoderTests.cs | 407 ++++++++++++++++++ 5 files changed, 926 insertions(+), 8 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Hevc/HevcSupplementalEnhancementInformation.cs diff --git a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs index 18daea6ce..3e01a22c2 100644 --- a/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs +++ b/src/ImageSharp/Formats/Heif/HeifDecoderCore.cs @@ -2352,8 +2352,15 @@ internal sealed class HeifDecoderCore : ImageDecoderCore this.ApplyAssociatedMetadata(image.Metadata, rootItem, buffers); } + if (itemDecoder is HevcHeifItemDecoder hevcItemDecoder) + { + // The codec orientation describes the complete cropped picture. Item scaling and alpha composition + // must finish first so rotation neither resizes back to ispe nor leaves the auxiliary plane unrotated. + hevcItemDecoder.ApplySupplementalPresentation(image); + } + // MIAF defines crop, rotation, and mirror as presentation operations in that order. Applying the - // implemented transforms after alpha composition keeps the auxiliary plane in the same coordinate space. + // container transforms after codec presentation keeps every composed plane in the same coordinate space. ApplyPresentationTransforms(image, itemToDecode); if (!this.Options.SkipMetadata) diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcImageItemBitstream.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcImageItemBitstream.cs index 0d3a0185e..03ac96fb0 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcImageItemBitstream.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcImageItemBitstream.cs @@ -21,6 +21,7 @@ internal sealed class HevcImageItemBitstream { List nalUnits = []; List sliceSegments = []; + HevcSupplementalEnhancementInformation supplementalEnhancementInformation = new(); int offset = 0; while (offset < data.Length) { @@ -77,6 +78,20 @@ internal sealed class HevcImageItemBitstream { throw new InvalidImageContentException("The HEVC image item contains an end-of-sequence NAL unit."); } + + if (nalUnit.Header.NalUnitType == 39) + { + // Prefix SEI belongs to the following VCL NAL unit. Once this bounded item has started its only + // picture, another prefix unit would describe a second access unit that the item is not allowed to carry. + if (sliceSegments.Count != 0) + { + throw new InvalidImageContentException("The HEVC image item contains prefix SEI after its first coded slice."); + } + + // Prefix SEI messages are associated with this item's only access unit. Parse the observable still-image + // state in NAL and message order without retaining generic video persistence or timing state. + supplementalEnhancementInformation.ReadPrefixNalUnit(nalUnit.Rbsp.Span); + } } if (sliceSegments.Count == 0) @@ -86,6 +101,7 @@ internal sealed class HevcImageItemBitstream this.NalUnits = nalUnits; this.SliceSegments = sliceSegments; + this.SupplementalEnhancementInformation = supplementalEnhancementInformation; } /// @@ -98,6 +114,11 @@ internal sealed class HevcImageItemBitstream /// public IReadOnlyList SliceSegments { get; } + /// + /// Gets the presentation and exposed metadata decoded from prefix SEI NAL units. + /// + public HevcSupplementalEnhancementInformation SupplementalEnhancementInformation { get; } + /// /// Reads an unsigned one-through-four-byte NAL-unit length without assuming four-byte item framing. /// diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcSupplementalEnhancementInformation.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcSupplementalEnhancementInformation.cs new file mode 100644 index 000000000..7bb89218e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcSupplementalEnhancementInformation.cs @@ -0,0 +1,349 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.ColorProfiles; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Reads the presentation and exposed metadata carried by prefix SEI NAL units for one bounded still picture. +/// +internal sealed class HevcSupplementalEnhancementInformation +{ + private const int DisplayOrientationPayloadType = 47; + private const int MasteringDisplayColorVolumePayloadType = 137; + private const int NoDisplayPayloadType = 135; + private const int ContentLightLevelPayloadType = 144; + private const int AlternativeTransferCharacteristicsPayloadType = 147; + private const int AmbientViewingEnvironmentPayloadType = 148; + private const int ContentColorVolumePayloadType = 149; + + /// + /// Gets a value indicating whether the selected still picture is marked as unavailable for display. + /// + public bool NoDisplay { get; private set; } + + /// + /// Gets a value indicating whether an active display-orientation message is present. + /// + public bool HasDisplayOrientation { get; private set; } + + /// + /// Gets a value indicating whether the cropped decoded picture is flipped horizontally before rotation. + /// + public bool HorizontalFlip { get; private set; } + + /// + /// Gets a value indicating whether the cropped decoded picture is flipped vertically before rotation. + /// + public bool VerticalFlip { get; private set; } + + /// + /// Gets the unsigned fraction of one complete anticlockwise turn applied after flipping. + /// + public ushort AnticlockwiseRotation { get; private set; } + + /// + /// Gets the preferred CICP transfer-characteristics code, when signaled. + /// + public byte? PreferredTransferCharacteristics { get; private set; } + + /// + /// Gets the content light-level description, when signaled. + /// + public HeifContentLightLevel? ContentLightLevel { get; private set; } + + /// + /// Gets the mastering-display color volume, when signaled. + /// + public HeifMasteringDisplayColorVolume? MasteringDisplayColorVolume { get; private set; } + + /// + /// Gets the content color volume, when signaled and not cancelled. + /// + public HeifContentColorVolume? ContentColorVolume { get; private set; } + + /// + /// Gets the ambient viewing environment, when signaled. + /// + public HeifAmbientViewingEnvironment? AmbientViewingEnvironment { get; private set; } + + /// + /// Reads every byte-aligned message from one prefix SEI RBSP in bitstream order. + /// + /// The decoded NAL payload, including its RBSP trailing byte. + public void ReadPrefixNalUnit(ReadOnlySpan rbsp) + { + if (rbsp.IsEmpty) + { + throw new InvalidImageContentException("The HEVC prefix SEI NAL unit is missing RBSP trailing bits."); + } + + int offset = 0; + while (rbsp.Length - offset > 1) + { + int payloadType = ReadExtendedValue(rbsp, ref offset, "payload type"); + int payloadSize = ReadExtendedValue(rbsp, ref offset, "payload size"); + if (payloadSize > rbsp.Length - offset) + { + throw new InvalidImageContentException("The HEVC prefix SEI message payload is truncated."); + } + + ReadOnlySpan payload = rbsp.Slice(offset, payloadSize); + offset += payloadSize; + switch (payloadType) + { + case DisplayOrientationPayloadType: + this.ReadDisplayOrientation(payload); + break; + case NoDisplayPayloadType: + this.ReadNoDisplay(payload); + break; + case MasteringDisplayColorVolumePayloadType: + this.ReadMasteringDisplayColorVolume(payload); + break; + case ContentLightLevelPayloadType: + this.ReadContentLightLevel(payload); + break; + case AlternativeTransferCharacteristicsPayloadType: + this.ReadAlternativeTransferCharacteristics(payload); + break; + case AmbientViewingEnvironmentPayloadType: + this.ReadAmbientViewingEnvironment(payload); + break; + case ContentColorVolumePayloadType: + this.ReadContentColorVolume(payload); + break; + } + } + + if (offset != rbsp.Length - 1 || rbsp[offset] != 0x80) + { + throw new InvalidImageContentException("The HEVC prefix SEI NAL unit has invalid RBSP trailing bits."); + } + } + + /// + /// Reads the legacy HEVC display-orientation payload retained by pinned HM. + /// + private void ReadDisplayOrientation(ReadOnlySpan payload) + { + HevcBitReader reader = new(payload); + bool cancel = reader.ReadFlag(); + if (cancel) + { + this.HasDisplayOrientation = false; + this.HorizontalFlip = false; + this.VerticalFlip = false; + this.AnticlockwiseRotation = 0; + ValidatePayloadExtension(ref reader, "display orientation"); + return; + } + + this.HorizontalFlip = reader.ReadFlag(); + this.VerticalFlip = reader.ReadFlag(); + this.AnticlockwiseRotation = (ushort)reader.ReadBits(16); + _ = reader.ReadFlag(); + ValidatePayloadExtension(ref reader, "display orientation"); + this.HasDisplayOrientation = true; + } + + /// + /// Records that the selected picture is not intended for display. + /// + private void ReadNoDisplay(ReadOnlySpan payload) + { + // Pinned HM writes no syntax bits for this message, producing a zero-byte payload. A nonempty payload can + // contain only the generic reserved extension and payload-alignment marker handled by the shared validator. + ValidateByteAlignedPayloadExtension(payload, "no-display"); + this.NoDisplay = true; + } + + /// + /// Reads mastering-display metadata in its HEVC fixed-point representation. + /// + private void ReadMasteringDisplayColorVolume(ReadOnlySpan payload) + { + const int syntaxLength = 24; + if (payload.Length < syntaxLength) + { + throw new InvalidImageContentException("The HEVC mastering-display color-volume SEI payload is truncated."); + } + + this.MasteringDisplayColorVolume = HeifPropertyParser.ParseMasteringDisplayColorVolume(payload[..syntaxLength]); + ValidateByteAlignedPayloadExtension(payload[syntaxLength..], "mastering-display color-volume"); + } + + /// + /// Reads content light-level metadata in its HEVC fixed-width representation. + /// + private void ReadContentLightLevel(ReadOnlySpan payload) + { + const int syntaxLength = 4; + if (payload.Length < syntaxLength) + { + throw new InvalidImageContentException("The HEVC content light-level SEI payload is truncated."); + } + + this.ContentLightLevel = HeifPropertyParser.ParseContentLightLevel(payload[..syntaxLength]); + ValidateByteAlignedPayloadExtension(payload[syntaxLength..], "content light-level"); + } + + /// + /// Reads the preferred transfer function applied when the container does not provide one. + /// + private void ReadAlternativeTransferCharacteristics(ReadOnlySpan payload) + { + if (payload.IsEmpty) + { + throw new InvalidImageContentException("The HEVC alternative-transfer-characteristics SEI payload is truncated."); + } + + this.PreferredTransferCharacteristics = payload[0]; + ValidateByteAlignedPayloadExtension(payload[1..], "alternative transfer characteristics"); + } + + /// + /// Reads the nominal ambient viewing environment. + /// + private void ReadAmbientViewingEnvironment(ReadOnlySpan payload) + { + const int syntaxLength = 8; + if (payload.Length < syntaxLength) + { + throw new InvalidImageContentException("The HEVC ambient-viewing-environment SEI payload is truncated."); + } + + this.AmbientViewingEnvironment = HeifPropertyParser.ParseAmbientViewingEnvironment(payload[..syntaxLength]); + ValidateByteAlignedPayloadExtension(payload[syntaxLength..], "ambient viewing environment"); + } + + /// + /// Reads the bit-packed content color-volume syntax and applies cancellation in message order. + /// + private void ReadContentColorVolume(ReadOnlySpan payload) + { + HevcBitReader reader = new(payload); + bool cancel = reader.ReadFlag(); + if (cancel) + { + this.ContentColorVolume = null; + ValidatePayloadExtension(ref reader, "content color-volume"); + return; + } + + _ = reader.ReadFlag(); + bool primariesPresent = reader.ReadFlag(); + bool minimumLuminancePresent = reader.ReadFlag(); + bool maximumLuminancePresent = reader.ReadFlag(); + bool averageLuminancePresent = reader.ReadFlag(); + RgbPrimariesChromaticityCoordinates? primaries = null; + if (primariesPresent) + { + int greenX = unchecked((int)reader.ReadBits(32)); + int greenY = unchecked((int)reader.ReadBits(32)); + int blueX = unchecked((int)reader.ReadBits(32)); + int blueY = unchecked((int)reader.ReadBits(32)); + int redX = unchecked((int)reader.ReadBits(32)); + int redY = unchecked((int)reader.ReadBits(32)); + const int maximumChromaticityValue = 5_000_000; + if (greenX is < -maximumChromaticityValue or > maximumChromaticityValue + || greenY is < -maximumChromaticityValue or > maximumChromaticityValue + || blueX is < -maximumChromaticityValue or > maximumChromaticityValue + || blueY is < -maximumChromaticityValue or > maximumChromaticityValue + || redX is < -maximumChromaticityValue or > maximumChromaticityValue + || redY is < -maximumChromaticityValue or > maximumChromaticityValue) + { + throw new InvalidImageContentException("The HEVC content color-volume SEI payload has an out-of-range primary coordinate."); + } + + const float chromaticityScale = 1F / 50000F; + + // H.274 stores signed primary coordinates in G, B, R order. Reorder them once at the codec boundary so + // the retained value has the same observable RGB coordinate contract as the equivalent item property. + primaries = new RgbPrimariesChromaticityCoordinates( + new CieXyChromaticityCoordinates(redX * chromaticityScale, redY * chromaticityScale), + new CieXyChromaticityCoordinates(greenX * chromaticityScale, greenY * chromaticityScale), + new CieXyChromaticityCoordinates(blueX * chromaticityScale, blueY * chromaticityScale)); + } + + uint? minimumLuminance = minimumLuminancePresent ? reader.ReadBits(32) : null; + uint? maximumLuminance = maximumLuminancePresent ? reader.ReadBits(32) : null; + uint? averageLuminance = averageLuminancePresent ? reader.ReadBits(32) : null; + if ((minimumLuminance is not null && averageLuminance is not null && minimumLuminance.Value > averageLuminance.Value) + || (averageLuminance is not null && maximumLuminance is not null && averageLuminance.Value > maximumLuminance.Value) + || (minimumLuminance is not null && maximumLuminance is not null && minimumLuminance.Value > maximumLuminance.Value)) + { + throw new InvalidImageContentException("The HEVC content color-volume SEI luminance values are not in ascending order."); + } + + ValidatePayloadExtension(ref reader, "content color-volume"); + const double luminanceScale = 1D / 10000000D; + + this.ContentColorVolume = new HeifContentColorVolume( + primaries, + minimumLuminance * luminanceScale, + maximumLuminance * luminanceScale, + averageLuminance * luminanceScale); + } + + /// + /// Reads an extended SEI payload type or size whose continuation bytes are all 255. + /// + private static int ReadExtendedValue(ReadOnlySpan data, ref int offset, string valueName) + { + int value = 0; + while (true) + { + if ((uint)offset >= (uint)data.Length) + { + throw new InvalidImageContentException($"The HEVC prefix SEI {valueName} is truncated."); + } + + int current = data[offset++]; + if (value > int.MaxValue - current) + { + throw new InvalidImageContentException($"The HEVC prefix SEI {valueName} is too large."); + } + + value += current; + if (current != byte.MaxValue) + { + return value; + } + } + } + + /// + /// Validates an optional extension following fixed byte-aligned SEI syntax. + /// + private static void ValidateByteAlignedPayloadExtension(ReadOnlySpan extension, string payloadName) + { + if (extension.IsEmpty) + { + return; + } + + HevcBitReader reader = new(extension); + ValidatePayloadExtension(ref reader, payloadName); + } + + /// + /// Validates reserved payload-extension data followed by its final one bit and zero padding. + /// + private static void ValidatePayloadExtension(ref HevcBitReader reader, string payloadName) + { + bool foundMarker = false; + while (reader.BitsRemaining > 0) + { + foundMarker |= reader.ReadFlag(); + } + + // The final set bit is payload_bit_equal_to_one; any preceding bits are the reserved extension data that + // pinned HM deliberately skips. An all-zero remainder has no marker and is therefore not a complete payload. + if (!foundMarker) + { + throw new InvalidImageContentException($"The HEVC {payloadName} SEI payload has invalid trailing bits."); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs b/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs index 903ea7c9c..96df7c6ba 100644 --- a/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs +++ b/src/ImageSharp/Formats/Heif/HevcHeifItemDecoder.cs @@ -7,6 +7,7 @@ using SixLabors.ImageSharp.Formats.Heif.Hevc.Color; using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.PixelFormats; +using SixLabors.ImageSharp.Processing; namespace SixLabors.ImageSharp.Formats.Heif; @@ -17,6 +18,8 @@ namespace SixLabors.ImageSharp.Formats.Heif; internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IHeifAlphaItemDecoder where TPixel : unmanaged, IPixel { + private HevcSupplementalEnhancementInformation? supplementalEnhancementInformation; + /// /// Gets the HEVC-coded image item type. /// @@ -43,6 +46,7 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH CicpProfile? colorProfile, CancellationToken cancellationToken) { + this.supplementalEnhancementInformation = null; using HevcPictureDecoder decoder = DecodePicture( options, item, @@ -52,9 +56,19 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH out HevcCodecConfiguration codecConfiguration, out HevcSequenceParameterSet sequenceParameterSet, out CicpProfile effectiveColorProfile, - out HevcChromaSampleLocation chromaSampleLocation); + out HevcChromaSampleLocation chromaSampleLocation, + out HevcSupplementalEnhancementInformation supplementalEnhancementInformation); + + if (supplementalEnhancementInformation.NoDisplay) + { + throw new InvalidImageContentException($"HEVC image item {item.Id} is marked as unavailable for display."); + } + + ValidateSupplementalMetadata(item, supplementalEnhancementInformation); + this.supplementalEnhancementInformation = supplementalEnhancementInformation; ImageFrame? frame = null; + Image? image = null; try { frame = new ImageFrame(options.Configuration, sequenceParameterSet.DisplayWidth, sequenceParameterSet.DisplayHeight); @@ -76,16 +90,75 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH heifMetadata.CompressionMethod = this.CompressionMethod; heifMetadata.BitDepth = codecConfiguration.BitDepth; heifMetadata.IsMonochrome = codecConfiguration.IsMonochrome; - return new Image(options.Configuration, metadata, [frame]); + heifMetadata.ContentLightLevel = supplementalEnhancementInformation.ContentLightLevel; + heifMetadata.MasteringDisplayColorVolume = supplementalEnhancementInformation.MasteringDisplayColorVolume; + heifMetadata.ContentColorVolume = supplementalEnhancementInformation.ContentColorVolume; + heifMetadata.AmbientViewingEnvironment = supplementalEnhancementInformation.AmbientViewingEnvironment; + + image = new Image(options.Configuration, metadata, [frame]); + frame = null; + return image; } catch { - // Ownership transfers only after the image constructor accepts the completely converted frame. + // Before the image constructor succeeds the frame remains locally owned. Afterwards the image owns it and + // every processor-created replacement buffer, so unwind exactly one of those two ownership states. + image?.Dispose(); frame?.Dispose(); throw; } } + /// + /// Applies the active HEVC display-orientation message to the complete presented image. + /// + /// The decoded image after item scaling and auxiliary-alpha composition. + public void ApplySupplementalPresentation(Image image) + { + HevcSupplementalEnhancementInformation supplementalEnhancementInformation + = this.supplementalEnhancementInformation!; + + if (!supplementalEnhancementInformation.HasDisplayOrientation) + { + return; + } + + image.Mutate(context => + { + // H.265 applies both flips to the cropped decoded picture before its anticlockwise rotation. + // ImageSharp's positive rotation is clockwise, so quarter turns use the exact optimized modes and + // all other coded angles use the equivalent positive clockwise angle. + if (supplementalEnhancementInformation.HorizontalFlip) + { + context.Flip(FlipMode.Horizontal); + } + + if (supplementalEnhancementInformation.VerticalFlip) + { + context.Flip(FlipMode.Vertical); + } + + ushort rotation = supplementalEnhancementInformation.AnticlockwiseRotation; + switch (rotation) + { + case 0: + break; + case 16384: + context.Rotate(RotateMode.Rotate270); + break; + case 32768: + context.Rotate(RotateMode.Rotate180); + break; + case 49152: + context.Rotate(RotateMode.Rotate90); + break; + default: + context.Rotate(360F - ((360F * rotation) / 65536F)); + break; + } + }); + } + /// public void DecodeAlphaItemData( DecoderOptions options, @@ -106,7 +179,8 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH out _, out HevcSequenceParameterSet sequenceParameterSet, out CicpProfile effectiveColorProfile, - out HevcChromaSampleLocation chromaSampleLocation); + out HevcChromaSampleLocation chromaSampleLocation, + out _); Rectangle sourceRectangle = new( sequenceParameterSet.ConformanceWindowLeftOffset, @@ -143,6 +217,7 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH /// Receives the sequence parameters describing the visible picture. /// Receives the effective CICP description used for presentation. /// Receives the progressive-frame chroma sample location. + /// Receives the bounded presentation and metadata SEI state. /// The decoder owning the reconstructed native picture. Ownership transfers to the caller. private static HevcPictureDecoder DecodePicture( DecoderOptions options, @@ -153,7 +228,8 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH out HevcCodecConfiguration codecConfiguration, out HevcSequenceParameterSet sequenceParameterSet, out CicpProfile effectiveColorProfile, - out HevcChromaSampleLocation chromaSampleLocation) + out HevcChromaSampleLocation chromaSampleLocation, + out HevcSupplementalEnhancementInformation supplementalEnhancementInformation) { cancellationToken.ThrowIfCancellationRequested(); codecConfiguration = item.HevcCodecConfiguration @@ -165,12 +241,23 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH } HevcImageItemBitstream bitstream = new(data, codecConfiguration); + supplementalEnhancementInformation = bitstream.SupplementalEnhancementInformation; HevcPictureParameterSet pictureParameterSet = bitstream.SliceSegments[0].PictureParameterSet; sequenceParameterSet = pictureParameterSet.SequenceParameterSet; HevcVideoUsabilityInformation? vui = sequenceParameterSet.VideoUsabilityInformation; + byte transferCharacteristics = vui?.ColorDescriptionPresent == true + ? vui.TransferCharacteristics + : (byte)CicpTransferCharacteristics.Unspecified; + + byte? preferredTransferCharacteristics = supplementalEnhancementInformation.PreferredTransferCharacteristics; + if (colorProfile is null && preferredTransferCharacteristics is not null) + { + transferCharacteristics = preferredTransferCharacteristics.Value; + } // ISO BMFF color information takes precedence when both the container and HEVC VUI describe the image. - // Otherwise, retain the VUI values used by conversion so bitstream-only color information reaches metadata. + // Otherwise, retain the VUI values and the SEI-preferred transfer function used by conversion so bitstream-only + // color information reaches metadata. effectiveColorProfile = colorProfile is not null ? new CicpProfile( (byte)colorProfile.ColorPrimaries, @@ -179,7 +266,7 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH colorProfile.FullRange) : new CicpProfile( vui?.ColorDescriptionPresent == true ? vui.ColorPrimaries : (byte)CicpColorPrimaries.Unspecified, - vui?.ColorDescriptionPresent == true ? vui.TransferCharacteristics : (byte)CicpTransferCharacteristics.Unspecified, + transferCharacteristics, vui?.ColorDescriptionPresent == true ? vui.MatrixCoefficients : (byte)CicpMatrixCoefficients.Unspecified, vui?.VideoSignalTypePresent == true && vui.FullRange); @@ -200,4 +287,51 @@ internal sealed class HevcHeifItemDecoder : IHeifItemDecoder, IH throw; } } + + /// + /// Validates equivalent codec and item-property HDR metadata before either representation is exposed. + /// + private static void ValidateSupplementalMetadata( + HeifItem item, + HevcSupplementalEnhancementInformation supplementalEnhancementInformation) + { + HeifContentLightLevel? supplementalContentLightLevel = supplementalEnhancementInformation.ContentLightLevel; + HeifContentLightLevel? itemContentLightLevel = item.ContentLightLevel; + if (supplementalContentLightLevel is not null + && itemContentLightLevel is not null + && (supplementalContentLightLevel.Value.MaximumContentLightLevel != itemContentLightLevel.Value.MaximumContentLightLevel + || supplementalContentLightLevel.Value.MaximumPictureAverageLightLevel + != itemContentLightLevel.Value.MaximumPictureAverageLightLevel)) + { + throw new InvalidImageContentException($"HEVC image item {item.Id} has conflicting content light-level metadata."); + } + + HeifMasteringDisplayColorVolume? supplementalMasteringDisplayColorVolume + = supplementalEnhancementInformation.MasteringDisplayColorVolume; + + if (supplementalMasteringDisplayColorVolume is not null + && item.MasteringDisplayColorVolume is not null + && supplementalMasteringDisplayColorVolume.Value != item.MasteringDisplayColorVolume.Value) + { + throw new InvalidImageContentException($"HEVC image item {item.Id} has conflicting mastering-display metadata."); + } + + HeifContentColorVolume? supplementalContentColorVolume = supplementalEnhancementInformation.ContentColorVolume; + if (supplementalContentColorVolume is not null + && item.ContentColorVolume is not null + && supplementalContentColorVolume.Value != item.ContentColorVolume.Value) + { + throw new InvalidImageContentException($"HEVC image item {item.Id} has conflicting content color-volume metadata."); + } + + HeifAmbientViewingEnvironment? supplementalAmbientViewingEnvironment + = supplementalEnhancementInformation.AmbientViewingEnvironment; + + if (supplementalAmbientViewingEnvironment is not null + && item.AmbientViewingEnvironment is not null + && supplementalAmbientViewingEnvironment.Value != item.AmbientViewingEnvironment.Value) + { + throw new InvalidImageContentException($"HEVC image item {item.Id} has conflicting ambient-viewing metadata."); + } + } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs index 68bf44ced..9ea2832da 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs @@ -3,8 +3,13 @@ using System.Buffers.Binary; using System.Security.Cryptography; +using SixLabors.ImageSharp.Advanced; +using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.Formats.Heif; using SixLabors.ImageSharp.Formats.Heif.Hevc; using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.Metadata.Profiles.Cicp; +using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Tests.Memory; using SixLabors.ImageSharp.Tests.TestUtilities; @@ -22,6 +27,37 @@ public class HevcPictureDecoderTests private const HwIntrinsics LoopFilterConfigurations = HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; + /// + /// Gets a complete prefix SEI RBSP carrying one unknown message followed by every supported metadata message. + /// The values use the exact field widths and G, B, R ordering read by pinned HM. + /// + private static ReadOnlySpan SupplementalMetadataRbsp => + [ + 0xFF, 0x2D, 0x01, 0x7A, + 0x89, 0x18, + 0x3A, 0x98, 0x75, 0x30, + 0x1D, 0x4C, 0x13, 0x88, + 0x7D, 0x00, 0x3E, 0x80, + 0x3D, 0x13, 0x40, 0x42, + 0x02, 0x03, 0x04, 0x05, + 0x01, 0x02, 0x03, 0x04, + 0x90, 0x04, 0x03, 0xE8, 0x01, 0x90, + 0x93, 0x01, 0x10, + 0x94, 0x08, 0x00, 0x0F, 0x42, 0x40, 0x3D, 0x13, 0x40, 0x42, + 0x95, 0x0D, 0x1C, 0x00, 0x3D, 0x09, 0x00, 0x01, 0x31, 0x2D, 0x00, 0x00, 0xB7, 0x1B, 0x02, + 0x80 + ]; + + /// + /// Gets a display-orientation prefix SEI RBSP that flips horizontally and then rotates a quarter turn + /// anticlockwise. + /// + private static ReadOnlySpan DisplayOrientationRbsp => + [ + 0x2F, 0x03, 0x48, 0x00, 0x08, + 0x80 + ]; + /// /// Identifies residual-tool signaling that an official independently decoded picture must exercise. /// @@ -649,6 +685,140 @@ public class HevcPictureDecoderTests Assert.Equal(9, decodedBoundary - DecodedHeaderLength); } + /// + /// Verifies every supported prefix SEI payload against the field ordering and fixed-point units read by pinned HM. + /// + [Fact] + public void SupplementalEnhancementInformationReadsPinnedHmSyntax() + { + HevcSupplementalEnhancementInformation supplementalEnhancementInformation = new(); + supplementalEnhancementInformation.ReadPrefixNalUnit(SupplementalMetadataRbsp); + supplementalEnhancementInformation.ReadPrefixNalUnit(DisplayOrientationRbsp); + + Assert.True(supplementalEnhancementInformation.HasDisplayOrientation); + Assert.True(supplementalEnhancementInformation.HorizontalFlip); + Assert.False(supplementalEnhancementInformation.VerticalFlip); + Assert.Equal((ushort)16384, supplementalEnhancementInformation.AnticlockwiseRotation); + Assert.Equal((byte)CicpTransferCharacteristics.SmpteSt2084, supplementalEnhancementInformation.PreferredTransferCharacteristics); + + HeifContentLightLevel contentLightLevel = supplementalEnhancementInformation.ContentLightLevel.Value; + Assert.Equal((ushort)1000, contentLightLevel.MaximumContentLightLevel); + Assert.Equal((ushort)400, contentLightLevel.MaximumPictureAverageLightLevel); + + HeifMasteringDisplayColorVolume masteringDisplayColorVolume + = supplementalEnhancementInformation.MasteringDisplayColorVolume.Value; + + Assert.Equal(0.64F, masteringDisplayColorVolume.Primaries.R.X, 5); + Assert.Equal(0.32F, masteringDisplayColorVolume.Primaries.R.Y, 5); + Assert.Equal(3375.2069D, masteringDisplayColorVolume.MaximumLuminance, 4); + Assert.Equal(1690.906D, masteringDisplayColorVolume.MinimumLuminance, 4); + + HeifAmbientViewingEnvironment ambientViewingEnvironment + = supplementalEnhancementInformation.AmbientViewingEnvironment.Value; + + Assert.Equal(100D, ambientViewingEnvironment.Illuminance); + Assert.Equal(0.3127F, ambientViewingEnvironment.AmbientLight.X, 5); + Assert.Equal(0.329F, ambientViewingEnvironment.AmbientLight.Y, 5); + + HeifContentColorVolume contentColorVolume = supplementalEnhancementInformation.ContentColorVolume.Value; + Assert.Null(contentColorVolume.Primaries); + Assert.Equal(0.1D, contentColorVolume.MinimumLuminance.Value, 5); + Assert.Equal(0.5D, contentColorVolume.MaximumLuminance.Value, 5); + Assert.Equal(0.3D, contentColorVolume.AverageLuminance.Value, 5); + + ReadOnlySpan cancellationRbsp = + [ + 0x2F, 0x01, 0xC0, + 0x95, 0x01, 0xC0, + 0x80 + ]; + + supplementalEnhancementInformation.ReadPrefixNalUnit(cancellationRbsp); + + Assert.False(supplementalEnhancementInformation.HasDisplayOrientation); + Assert.Null(supplementalEnhancementInformation.ContentColorVolume); + Assert.NotNull(supplementalEnhancementInformation.ContentLightLevel); + } + + /// + /// Verifies malformed SEI framing and payload trailing bits fail at the bounded NAL boundary. + /// + [Fact] + public void SupplementalEnhancementInformationRejectsMalformedPayloads() + { + byte[] emptyRbsp = []; + byte[] truncatedHeader = [0xFF, 0x80]; + byte[] truncatedPayload = [0x90, 0x04, 0x03, 0xE8, 0x80]; + byte[] missingPayloadMarker = [0x2F, 0x03, 0x48, 0x00, 0x00, 0x80]; + + Assert.Throws( + () => new HevcSupplementalEnhancementInformation().ReadPrefixNalUnit(emptyRbsp)); + + Assert.Throws( + () => new HevcSupplementalEnhancementInformation().ReadPrefixNalUnit(truncatedHeader)); + + Assert.Throws( + () => new HevcSupplementalEnhancementInformation().ReadPrefixNalUnit(truncatedPayload)); + + Assert.Throws( + () => new HevcSupplementalEnhancementInformation().ReadPrefixNalUnit(missingPayloadMarker)); + } + + /// + /// Verifies HEVC item metadata and complete HEIF presentation under normal and scalar execution while every + /// constrained allocator group is returned exactly once. + /// + [Fact] + public void DecodeSupplementalPresentationAndMetadataAcrossIntrinsicWidths() + => FeatureTestRunner.RunWithHwIntrinsicsFeature( + ValidateSupplementalPresentationAndMetadata, + HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic); + + /// + /// Verifies no-display and conflicting item-property metadata are rejected by the complete item decoder. + /// + [Fact] + public void DecodeRejectsNonDisplayAndConflictingSupplementalMetadata() + { + byte[] annexB = TestFile.Create(TestImages.Heif.IntraPredictionB).Bytes; + ConvertAnnexBStillPicture(annexB, 8, 1, out byte[] configurationData, out byte[] itemData); + HevcCodecConfiguration codecConfiguration = new(configurationData); + HeifItem item = new(Heif4CharCode.Hvc1, 1) { HevcCodecConfiguration = codecConfiguration }; + HevcHeifItemDecoder itemDecoder = new(); + DecoderOptions options = new(); + ReadOnlySpan noDisplayRbsp = [0x87, 0x00, 0x80]; + + byte[] noDisplayItemData = PrependPrefixSeiNalUnit(itemData, noDisplayRbsp); + Assert.Throws(() => + { + using Image image = itemDecoder.DecodeItemData( + options, + item, + noDisplayItemData, + null, + TestContext.Current.CancellationToken); + }); + + item.ContentLightLevel = new HeifContentLightLevel(1, 2); + byte[] supplementalItemData = PrependPrefixSeiNalUnit(itemData, SupplementalMetadataRbsp); + Assert.Throws(() => + { + using Image image = itemDecoder.DecodeItemData( + options, + item, + supplementalItemData, + null, + TestContext.Current.CancellationToken); + }); + + int prefixNalLength = supplementalItemData.Length - itemData.Length; + byte[] postVclPrefixSeiItemData = new byte[supplementalItemData.Length]; + itemData.CopyTo(postVclPrefixSeiItemData, 0); + supplementalItemData.AsSpan(0, prefixNalLength).CopyTo(postVclPrefixSeiItemData.AsSpan(itemData.Length)); + Assert.Throws( + () => new HevcImageItemBitstream(postVclPrefixSeiItemData, codecConfiguration)); + } + /// /// Verifies all reconstructed samples from a real HEIC grid tile against the HM reference decoder. /// @@ -875,6 +1045,243 @@ public class HevcPictureDecoderTests /// The displayed component extent. private static int GetDisplaySize(int lumaSize, int subsampling) => (lumaSize + (1 << subsampling) - 1) >> subsampling; + /// + /// Verifies HEVC item metadata and the complete public HEIF presentation path in the active intrinsic + /// configuration. + /// + private static void ValidateSupplementalPresentationAndMetadata() + { + TestMemoryAllocator allocator = new() { BufferCapacityInBytes = 8_192 }; + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = Configuration.Default.Clone(); + configuration.MemoryAllocator = allocator; + DecoderOptions options = new() { Configuration = configuration }; + + // FeatureTestRunner executes this method outside the originating xUnit context when it disables + // intrinsics, so the remote process cannot obtain the test's cancellation token. + CancellationToken cancellationToken = CancellationToken.None; + byte[] annexB = TestFile.Create(TestImages.Heif.IntraPredictionB).Bytes; + ConvertAnnexBStillPicture(annexB, 8, 1, out byte[] configurationData, out byte[] itemData); + HevcCodecConfiguration codecConfiguration = new(configurationData); + HeifItem item = new(Heif4CharCode.Hvc1, 1) { HevcCodecConfiguration = codecConfiguration }; + byte[] supplementalItemData = PrependPrefixSeiNalUnit(itemData, SupplementalMetadataRbsp); + HevcHeifItemDecoder itemDecoder = new(); + using (Image metadataImage = itemDecoder.DecodeItemData( + options, + item, + supplementalItemData, + null, + cancellationToken)) + { + Assert.Equal(CicpTransferCharacteristics.SmpteSt2084, metadataImage.Metadata.CicpProfile.TransferCharacteristics); + HeifMetadata metadata = metadataImage.Metadata.GetHeifMetadata(); + Assert.Equal((ushort)1000, metadata.ContentLightLevel.Value.MaximumContentLightLevel); + Assert.NotNull(metadata.MasteringDisplayColorVolume); + Assert.NotNull(metadata.ContentColorVolume); + Assert.NotNull(metadata.AmbientViewingEnvironment); + } + + byte[] source = [.. TestFile.Create(TestImages.Heif.Image4).Bytes]; + byte[] orientedContainer = InsertPrimaryItemPrefixSeiNalUnit(source, DisplayOrientationRbsp); + string referencePath = Path.Combine( + TestEnvironment.ReferenceOutputDirectoryFullPath, + "HeifDecoderTests", + "DecodeHevcStillImage_Rgba32_image4.png"); + + using (Image baseline = Image.Load(referencePath)) + using (Image actual = Image.Load(options, orientedContainer)) + { + Assert.Equal(baseline.Height, actual.Width); + Assert.Equal(baseline.Width, actual.Height); + ImageFrame baselineFrame = baseline.Frames.RootFrame; + ImageFrame actualFrame = actual.Frames.RootFrame; + + // A horizontal flip followed by the signaled anticlockwise quarter turn is an exact transpose. Compare + // every RGBA sample directly so both color and auxiliary alpha must share the production transform. + for (int y = 0; y < actual.Height; y++) + { + ReadOnlySpan actualRow = actualFrame.DangerousGetPixelRowMemory(y).Span; + for (int x = 0; x < actual.Width; x++) + { + Assert.Equal(baselineFrame.DangerousGetPixelRowMemory(x).Span[y], actualRow[x]); + } + } + } + + Assert.NotEmpty(allocator.AllocationLog); + AssertBalancedAllocations(allocator); + } + + /// + /// Inserts one prefix SEI NAL unit at the start of the real fixture's primary file-relative extent. + /// + private static byte[] InsertPrimaryItemPrefixSeiNalUnit(byte[] container, ReadOnlySpan rbsp) + { + int metaOffset = FindBoxOffset(container, Heif4CharCode.Meta, 0, container.Length); + Assert.True(metaOffset >= 0); + int metaSize = (int)BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(metaOffset)); + int primaryItemOffset = FindBoxOffset(container, Heif4CharCode.Pitm, metaOffset + 12, metaSize - 12); + int itemLocationOffset = FindBoxOffset(container, Heif4CharCode.Iloc, metaOffset + 12, metaSize - 12); + int mediaDataOffset = FindBoxOffset(container, Heif4CharCode.Mdat, 0, container.Length); + Assert.True(primaryItemOffset >= 0); + Assert.True(itemLocationOffset >= 0); + Assert.True(mediaDataOffset >= 0); + + Assert.Equal(0, container[primaryItemOffset + 8]); + ushort primaryItemId = BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(primaryItemOffset + 12)); + + int position = itemLocationOffset + 8; + Assert.Equal(0, container[position]); + position += 4; + byte fieldSizes = container[position++]; + byte baseOffsetSizes = container[position++]; + Assert.Equal(4, fieldSizes >> 4); + Assert.Equal(4, fieldSizes & 15); + Assert.Equal(0, baseOffsetSizes >> 4); + ushort itemCount = BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(position)); + position += 2; + Assert.True(itemCount > 0); + + ushort firstItemId = BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(position)); + position += 2; + Assert.Equal(primaryItemId, firstItemId); + Assert.Equal(0, BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(position))); + position += 2; + ushort primaryExtentCount = BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(position)); + position += 2; + Assert.Equal(1, primaryExtentCount); + + int primaryOffsetField = position; + int insertionOffset = (int)BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(position)); + position += 4; + int primaryLengthField = position; + uint primaryLength = BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(position)); + position += 4; + + byte[] prefixNalUnit = PrependPrefixSeiNalUnit([], rbsp); + BinaryPrimitives.WriteUInt32BigEndian( + container.AsSpan(primaryLengthField), + checked(primaryLength + (uint)prefixNalUnit.Length)); + + // The fixture stores every extent as an absolute file offset. Inserting into the first extent shifts only + // later extents; its own offset remains the exact start at which the prefix NAL is inserted. + for (int itemIndex = 1; itemIndex < itemCount; itemIndex++) + { + position += 4; + ushort extentCount = BinaryPrimitives.ReadUInt16BigEndian(container.AsSpan(position)); + position += 2; + for (int extentIndex = 0; extentIndex < extentCount; extentIndex++) + { + int extentOffsetField = position; + uint extentOffset = BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(position)); + position += 8; + if (extentOffset > insertionOffset) + { + BinaryPrimitives.WriteUInt32BigEndian( + container.AsSpan(extentOffsetField), + checked(extentOffset + (uint)prefixNalUnit.Length)); + } + } + } + + Assert.Equal( + (uint)insertionOffset, + BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(primaryOffsetField))); + + uint compactMediaDataSize = BinaryPrimitives.ReadUInt32BigEndian(container.AsSpan(mediaDataOffset)); + ulong mediaDataSize = compactMediaDataSize == 1 + ? BinaryPrimitives.ReadUInt64BigEndian(container.AsSpan(mediaDataOffset + 8)) + : compactMediaDataSize; + + if (compactMediaDataSize == 1) + { + BinaryPrimitives.WriteUInt64BigEndian( + container.AsSpan(mediaDataOffset + 8), + checked(mediaDataSize + (uint)prefixNalUnit.Length)); + } + else + { + BinaryPrimitives.WriteUInt32BigEndian( + container.AsSpan(mediaDataOffset), + checked((uint)mediaDataSize + (uint)prefixNalUnit.Length)); + } + + byte[] result = new byte[container.Length + prefixNalUnit.Length]; + container.AsSpan(0, insertionOffset).CopyTo(result); + prefixNalUnit.CopyTo(result.AsSpan(insertionOffset)); + container.AsSpan(insertionOffset).CopyTo(result.AsSpan(insertionOffset + prefixNalUnit.Length)); + return result; + } + + /// + /// Finds a bounded ISO BMFF child box, including boxes that use a 64-bit extended size. + /// + private static int FindBoxOffset(ReadOnlySpan data, Heif4CharCode type, int offset, int length) + { + int endOffset = offset + length; + while (offset < endOffset) + { + uint compactSize = BinaryPrimitives.ReadUInt32BigEndian(data[offset..]); + ulong boxSize = compactSize == 1 + ? BinaryPrimitives.ReadUInt64BigEndian(data[(offset + 8)..]) + : compactSize; + + Assert.InRange(boxSize, 8UL, (ulong)(endOffset - offset)); + Heif4CharCode boxType = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(data[(offset + 4)..]); + if (boxType == type) + { + return offset; + } + + offset += (int)boxSize; + } + + return -1; + } + + /// + /// Prepends one valid prefix SEI NAL unit to a length-delimited HEVC item payload. + /// + private static byte[] PrependPrefixSeiNalUnit(ReadOnlySpan itemData, ReadOnlySpan rbsp) + { + int preventionByteCount = 0; + int consecutiveZeroBytes = 0; + foreach (byte value in rbsp) + { + if (consecutiveZeroBytes == 2 && value <= 3) + { + preventionByteCount++; + consecutiveZeroBytes = 0; + } + + consecutiveZeroBytes = value == 0 ? consecutiveZeroBytes + 1 : 0; + } + + const int lengthFieldLength = 4; + const int nalHeaderLength = 2; + int nalLength = nalHeaderLength + rbsp.Length + preventionByteCount; + byte[] result = new byte[lengthFieldLength + nalLength + itemData.Length]; + BinaryPrimitives.WriteUInt32BigEndian(result, (uint)nalLength); + result[lengthFieldLength] = 0x4E; + result[lengthFieldLength + 1] = 0x01; + int destinationOffset = lengthFieldLength + nalHeaderLength; + consecutiveZeroBytes = 0; + foreach (byte value in rbsp) + { + if (consecutiveZeroBytes == 2 && value <= 3) + { + result[destinationOffset++] = 3; + consecutiveZeroBytes = 0; + } + + result[destinationOffset++] = value; + consecutiveZeroBytes = value == 0 ? consecutiveZeroBytes + 1 : 0; + } + + itemData.CopyTo(result.AsSpan(destinationOffset)); + return result; + } + /// /// Adapts the first independently coded Annex B picture to the bounded hvc1 item contract used by the /// production decoder.