Browse Source

Apply HEIF clean aperture properties

pull/2633/head
James Jackson-South 1 week ago
parent
commit
cf0c4086f8
  1. 6
      HEIF_IMPLEMENTATION_PLAN.md
  2. 5
      src/ImageSharp/Formats/Heif/Heif4CharCode.cs
  3. 1
      src/ImageSharp/Formats/Heif/Heif4CharCode.tt
  4. 160
      src/ImageSharp/Formats/Heif/HeifCleanAperture.cs
  5. 60
      src/ImageSharp/Formats/Heif/HeifDecoderCore.cs
  6. 6
      src/ImageSharp/Formats/Heif/HeifItem.cs

6
HEIF_IMPLEMENTATION_PLAN.md

@ -85,7 +85,7 @@ This snapshot pins or classifies the available references and failures; it does
| `Av1InverseQuantizer` and `Av1InverseQuantizationLookup` | AV1 section 7.12.3 inverse quantization | libaom `aom_dsp/aom_dsp_common.h`, `av1/common/quant_common.c`, and `av1/decoder/decodetxb.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Select the per-segment matrix level, alias 64-pixel transform dimensions to their adjusted matrices, retain a flat level-15 matrix, and apply the five-bit inverse-matrix weight scale. The large managed lookup remains a single process-wide table. |
| `Av1Inverse2dTransformer` and `Av1InverseTransformerFactory` | AV1 section 7.11.2 inverse transform and reconstruction | libaom `av1/common/av1_inv_txfm1d.c`, `av1/common/av1_inv_txfm2d.c`, and `av1/common/idct.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar transform oracle for coefficient-row traversal, intermediate layout, stage ranges, clipping, and high-bit-depth sample addition. The managed 16-bit overload is also used as a parity oracle for the byte overload. |
| `HeifDecoderCore` box extension handling and `HeifDecoderCore`/`HeifEncoderCore` item-property associations | ISO/IEC 14496-12 box extensibility and section 8.11.14 item properties and `ipma` syntax | libavif `src/read.c` and `src/write.c` at `092276ce89098ead06db80975173191e5fee1826` | Skip unrecognized top-level and metadata child boxes, preserve the position of every property in `ipco`, reject an unrecognized property only when its item association marks it essential, associate properties by item ID, and read or write the essential bit plus one-based 7-bit or 15-bit property index according to the full-box flags. Independent HEIC, HIF, and AVIF fixtures provide the reader oracle; container-level identification of encoded output guards the writer independently of pixel roundtripping. |
| `HeifItem` rotation/mirror state and `HeifDecoderCore` presentation-transform parsing and application | HEIF image rotation and mirror properties; MIAF section 7.3.6.7 presentation order and section 7.3.9 essential transformative properties | libavif `src/read.c` property parsers and alpha-property validation plus `apps/shared/avifutil.c` transform application at `092276ce89098ead06db80975173191e5fee1826` | Validate the registered reserved bits, require essential associations, map counter-clockwise HEIF quarter turns to ImageSharp's optimized clockwise rotate modes, then mirror around the signaled axis after alpha composition. Reuse ImageSharp's existing rotation and flip processors for every pixel type. Retain only the two image-item property values; do not add a generic transform-box or ISO BMFF model. |
| `HeifCleanAperture`, `HeifItem` presentation state, and `HeifDecoderCore` transformative-property parsing and application | ISO/IEC 14496-12 section 12.1.4 clean aperture; HEIF image rotation and mirror properties; MIAF section 7.3.6.7 presentation order and section 7.3.9 essential transformative properties | libavif `src/avif.c` clean-aperture conversion, `src/read.c` property parsers and alpha-property validation, and `apps/shared/avifutil.c` transform application at `092276ce89098ead06db80975173191e5fee1826` | Resolve fractional clean-aperture dimensions and center offsets to exact bounded integer pixels, validate the registered rotation/mirror reserved bits, require essential associations, crop after auxiliary-alpha composition, map counter-clockwise HEIF quarter turns to ImageSharp's optimized clockwise rotate modes, then mirror around the signaled axis. Reuse ImageSharp's existing crop, rotation, and flip processors for every pixel type. Retain only the three image-item property values; do not add a generic transform-box or ISO BMFF model. |
| `HeifConstants.IsSupportedFileType`, `HeifImageFormatDetector`, and `HeifDecoderCore.CheckFileTypeBox` | ISO/IEC 14496-12 `FileTypeBox` syntax and the MP4 Registration Authority HEIF/AVIF still-image and sequence brand registrations | libavif `src/read.c` functions `avifParseFileTypeBox`, `avifFileTypeHasBrand`, and `avifFileTypeIsCompatible` at `092276ce89098ead06db80975173191e5fee1826` | Apply one rule to the major and compatible brands, accept the implemented still-image container and payload brands, and reject registered HEVC, AVIF, and JPEG sequence major brands as outside the image-item scope. The decoder validates the complete `ftyp` payload; the fixed-size format detector inspects the available prefix. |
| `HeifDecoderCore.ReadBoxHeader` and `HeifDecoderCore.ParseBoxHeader` | ISO/IEC 14496-12 section 4.2.2 basic box syntax | libavif `src/stream.c` functions `avifROStreamReadBoxHeaderPartial` and `avifROStreamReadBoxHeader` at `092276ce89098ead06db80975173191e5fee1826` | Resolve 32-bit, 64-bit, UUID, and top-level size-zero boxes into content lengths only after validating the complete variable-sized header and the remaining parent boundary. Nested size-zero boxes are invalid; large skips retain 64-bit offsets. |
| `HeifDecoderCore.ParseMetadata` | ISO/IEC 14496-12 `MetaBox` and HEIF item declarations, locations, properties, and associations | libavif `src/read.c` functions `avifParseMetaBox`, `avifMetaFindOrCreateItem`, `avifParseItemLocationBox`, and `avifParseItemPropertiesBox` at `092276ce89098ead06db80975173191e5fee1826` | Index unique recognized metadata children by type and payload location, then parse them in dependency order so physical placement does not control item lookup or property association. Duplicate unique children and truncated full-box headers are invalid. |
@ -116,8 +116,8 @@ This assessment is based on the current source after the upstream ImageSharp mer
- Item locations now support bounded file-relative and `idat`-relative storage, multiple ordered extents, versioned item IDs, 0/4/8-byte registered field sizes, and 64-bit offsets. Referenced-item construction method two and external data references remain explicitly unsupported.
- Grid derived-image decoding now parses both registered descriptor widths, resolves the ordered `dimg` cells, validates tile count, coding format, dimensions, canvas coverage, and edge overlap, then composes the output through ImageSharp row buffers. Unsupported grid tile codecs can select only a decodable thumbnail of the same primary grid. Independent AV1 and JPEG grid fixtures are still required, and HEVC grids remain blocked on the HEVC decoder.
- Alpha auxiliary decoding now recognizes `auxC`, `auxl`, and `prem`, supports a direct auxiliary image/grid and the per-color-grid-tile form, preserves normalized alpha through `L16`/`Rgba64`, box-resamples differing plane sizes, and reports alpha presence. HEVC alpha remains blocked on the HEVC image-item decoder, while independent AVIF alpha fixtures are still required against the incomplete AV1 reconstruction pipeline.
- Image rotation and mirror properties now validate their registered payloads and essential associations, affect Identify dimensions, and reuse ImageSharp's optimized rotate/flip processors after auxiliary alpha composition in the MIAF-defined order. Clean aperture remains missing, and independent transform vectors must still verify every rotation/mirror/alpha combination.
- HEVC and AV1 configuration, CICP color information, Exif/XMP, clean aperture, and several image-item properties/relationships are missing or parsed without affecting output.
- Clean aperture, image rotation, and image mirror properties now validate their registered payloads, exact integer crop geometry, and essential associations; affect Identify dimensions; and reuse ImageSharp's optimized crop/rotate/flip processors after auxiliary alpha composition in the MIAF-defined order. Independent transform vectors must still verify every crop/rotation/mirror/alpha combination.
- HEVC and AV1 configuration, CICP color information, Exif/XMP, and several image-item properties/relationships are missing or parsed without affecting output.
- Identify and decode now use the same bounded metadata parser and both validate the complete leading file type box. The parsed state is still mutable and Identify does not yet report the complete bit depth, color, profile, or transform model.
### HEVC decoder and encoder

5
src/ImageSharp/Formats/Heif/Heif4CharCode.cs

@ -93,6 +93,11 @@ public enum Heif4CharCode : uint
/// </summary>
Irot = 0x69726F74U,
/// <summary>
/// Clean Aperture.
/// </summary>
Clap = 0x636C6170U,
/// <summary>
/// Image Scaling.
/// </summary>

1
src/ImageSharp/Formats/Heif/Heif4CharCode.tt

@ -23,6 +23,7 @@
"av1C", "AV1 configuration",
"imir", "Image Mirror",
"irot", "Image Rotation",
"clap", "Clean Aperture",
"iscl", "Image Scaling",
"pasp", "Pixel Aspect Ratio",
"pixi", "Pixel Information",

160
src/ImageSharp/Formats/Heif/HeifCleanAperture.cs

@ -0,0 +1,160 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif;
/// <summary>
/// Describes the fractional dimensions and center offsets of a HEIF clean-aperture property.
/// </summary>
internal readonly struct HeifCleanAperture : IEquatable<HeifCleanAperture>
{
/// <summary>
/// Initializes a new instance of the <see cref="HeifCleanAperture"/> struct.
/// </summary>
/// <param name="widthNumerator">The clean-aperture width numerator.</param>
/// <param name="widthDenominator">The clean-aperture width denominator.</param>
/// <param name="heightNumerator">The clean-aperture height numerator.</param>
/// <param name="heightDenominator">The clean-aperture height denominator.</param>
/// <param name="horizontalOffsetNumerator">The horizontal center-offset numerator.</param>
/// <param name="horizontalOffsetDenominator">The horizontal center-offset denominator.</param>
/// <param name="verticalOffsetNumerator">The vertical center-offset numerator.</param>
/// <param name="verticalOffsetDenominator">The vertical center-offset denominator.</param>
public HeifCleanAperture(
int widthNumerator,
int widthDenominator,
int heightNumerator,
int heightDenominator,
int horizontalOffsetNumerator,
int horizontalOffsetDenominator,
int verticalOffsetNumerator,
int verticalOffsetDenominator)
{
this.WidthNumerator = widthNumerator;
this.WidthDenominator = widthDenominator;
this.HeightNumerator = heightNumerator;
this.HeightDenominator = heightDenominator;
this.HorizontalOffsetNumerator = horizontalOffsetNumerator;
this.HorizontalOffsetDenominator = horizontalOffsetDenominator;
this.VerticalOffsetNumerator = verticalOffsetNumerator;
this.VerticalOffsetDenominator = verticalOffsetDenominator;
}
/// <summary>
/// Gets the clean-aperture width numerator.
/// </summary>
public int WidthNumerator { get; }
/// <summary>
/// Gets the clean-aperture width denominator.
/// </summary>
public int WidthDenominator { get; }
/// <summary>
/// Gets the clean-aperture height numerator.
/// </summary>
public int HeightNumerator { get; }
/// <summary>
/// Gets the clean-aperture height denominator.
/// </summary>
public int HeightDenominator { get; }
/// <summary>
/// Gets the horizontal center-offset numerator.
/// </summary>
public int HorizontalOffsetNumerator { get; }
/// <summary>
/// Gets the horizontal center-offset denominator.
/// </summary>
public int HorizontalOffsetDenominator { get; }
/// <summary>
/// Gets the vertical center-offset numerator.
/// </summary>
public int VerticalOffsetNumerator { get; }
/// <summary>
/// Gets the vertical center-offset denominator.
/// </summary>
public int VerticalOffsetDenominator { get; }
/// <summary>
/// Converts the clean aperture to an integer crop rectangle within the coded image extent.
/// </summary>
/// <param name="imageExtent">The coded image dimensions.</param>
/// <returns>The clean-aperture crop rectangle.</returns>
/// <exception cref="InvalidImageContentException">
/// The clean-aperture fractions do not describe an integer rectangle inside the coded image extent.
/// </exception>
public Rectangle ToRectangle(Size imageExtent)
{
if (this.WidthNumerator <= 0 || this.HeightNumerator <= 0 ||
this.WidthDenominator <= 0 || this.HeightDenominator <= 0 ||
this.HorizontalOffsetDenominator <= 0 || this.VerticalOffsetDenominator <= 0)
{
throw new InvalidImageContentException("The clean aperture dimensions and denominators must be positive.");
}
if ((this.WidthNumerator % this.WidthDenominator) != 0 ||
(this.HeightNumerator % this.HeightDenominator) != 0)
{
throw new InvalidImageContentException("The clean aperture dimensions must resolve to integer pixels.");
}
int width = this.WidthNumerator / this.WidthDenominator;
int height = this.HeightNumerator / this.HeightDenominator;
// Express each top-left coordinate over twice the offset denominator. This is the exact
// center-plus-offset-minus-half-size equation without floating-point rounding.
long xNumerator = ((long)(imageExtent.Width - width) * this.HorizontalOffsetDenominator) +
(2L * this.HorizontalOffsetNumerator);
long xDenominator = 2L * this.HorizontalOffsetDenominator;
long yNumerator = ((long)(imageExtent.Height - height) * this.VerticalOffsetDenominator) +
(2L * this.VerticalOffsetNumerator);
long yDenominator = 2L * this.VerticalOffsetDenominator;
if ((xNumerator % xDenominator) != 0 || (yNumerator % yDenominator) != 0)
{
throw new InvalidImageContentException("The clean aperture offsets must resolve to integer pixels.");
}
long x = xNumerator / xDenominator;
long y = yNumerator / yDenominator;
if (x < 0 || y < 0 || x + width > imageExtent.Width || y + height > imageExtent.Height)
{
throw new InvalidImageContentException("The clean aperture lies outside the coded image extent.");
}
return new Rectangle((int)x, (int)y, width, height);
}
/// <inheritdoc/>
public bool Equals(HeifCleanAperture other)
=> this.WidthNumerator == other.WidthNumerator &&
this.WidthDenominator == other.WidthDenominator &&
this.HeightNumerator == other.HeightNumerator &&
this.HeightDenominator == other.HeightDenominator &&
this.HorizontalOffsetNumerator == other.HorizontalOffsetNumerator &&
this.HorizontalOffsetDenominator == other.HorizontalOffsetDenominator &&
this.VerticalOffsetNumerator == other.VerticalOffsetNumerator &&
this.VerticalOffsetDenominator == other.VerticalOffsetDenominator;
/// <inheritdoc/>
public override bool Equals(object? obj) => obj is HeifCleanAperture other && this.Equals(other);
/// <inheritdoc/>
public override int GetHashCode()
{
HashCode hashCode = default;
hashCode.Add(this.WidthNumerator);
hashCode.Add(this.WidthDenominator);
hashCode.Add(this.HeightNumerator);
hashCode.Add(this.HeightDenominator);
hashCode.Add(this.HorizontalOffsetNumerator);
hashCode.Add(this.HorizontalOffsetDenominator);
hashCode.Add(this.VerticalOffsetNumerator);
hashCode.Add(this.VerticalOffsetDenominator);
return hashCode.ToHashCode();
}
}

60
src/ImageSharp/Formats/Heif/HeifDecoderCore.cs

@ -844,6 +844,20 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
this.av1CodecConfiguration = new(boxBuffer);
properties.Add(new KeyValuePair<Heif4CharCode, object>(Heif4CharCode.Av1C, new object()));
break;
case Heif4CharCode.Clap:
EnsureBufferRemaining(boxBuffer, 0, 32, "clean aperture");
HeifCleanAperture cleanAperture = new(
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer)),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[4..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[8..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[12..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[16..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[20..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[24..])),
unchecked((int)BinaryPrimitives.ReadUInt32BigEndian(boxBuffer[28..])));
properties.Add(new KeyValuePair<Heif4CharCode, object>(Heif4CharCode.Clap, cleanAperture));
break;
case Heif4CharCode.Irot:
EnsureBufferRemaining(boxBuffer, 0, 1, "image rotation");
if ((boxBuffer[0] & 0xFC) != 0)
@ -956,7 +970,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
throw new InvalidImageContentException($"Item {itemId} associates unknown essential property '{PrettyPrint(prop.Key)}'.");
}
if (!essential && prop.Key is Heif4CharCode.Irot or Heif4CharCode.Imir)
if (!essential && prop.Key is Heif4CharCode.Clap or Heif4CharCode.Irot or Heif4CharCode.Imir)
{
throw new InvalidImageContentException($"Item {itemId} associates nonessential transformative property '{PrettyPrint(prop.Key)}'.");
}
@ -982,6 +996,14 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
item.AuxiliaryType = (string)prop.Value;
break;
case Heif4CharCode.Clap:
if (item.CleanAperture is not null)
{
throw new InvalidImageContentException($"Item {itemId} associates more than one clean aperture property.");
}
item.CleanAperture = (HeifCleanAperture)prop.Value;
break;
case Heif4CharCode.Irot:
if (item.RotationAngle is not null)
{
@ -1360,17 +1382,20 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
}
/// <summary>
/// Gets the dimensions of an image item after its rotation property is applied.
/// Gets the dimensions of an image item after its clean-aperture and rotation properties are applied.
/// </summary>
/// <param name="item">The image item whose presentation dimensions are requested.</param>
/// <returns>The item dimensions after an optional quarter-turn rotation.</returns>
/// <returns>The item dimensions after the optional crop and quarter-turn rotation.</returns>
private static Size GetPresentationExtent(HeifItem item)
=> item.RotationAngle is not null && (item.RotationAngle.Value & 1) != 0
? new Size(item.Extent.Height, item.Extent.Width)
: item.Extent;
{
Size extent = item.CleanAperture is not null ? item.CleanAperture.Value.ToRectangle(item.Extent).Size : item.Extent;
return item.RotationAngle is not null && (item.RotationAngle.Value & 1) != 0
? new Size(extent.Height, extent.Width)
: extent;
}
/// <summary>
/// Applies the rotation and mirror properties associated with an image item.
/// Applies the clean-aperture, rotation, and mirror properties associated with an image item.
/// </summary>
/// <typeparam name="TPixel">The image pixel format.</typeparam>
/// <param name="image">The decoded image item.</param>
@ -1378,6 +1403,15 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
private static void ApplyPresentationTransforms<TPixel>(Image<TPixel> image, HeifItem item)
where TPixel : unmanaged, IPixel<TPixel>
{
if (item.CleanAperture is not null)
{
Rectangle cropRectangle = item.CleanAperture.Value.ToRectangle(image.Size);
if (cropRectangle != image.Bounds)
{
image.Mutate(context => context.Crop(cropRectangle));
}
}
if (item.RotationAngle is not null)
{
// HEIF angles count quarter turns counter-clockwise, while ImageSharp's optimized rotate modes are clockwise.
@ -1421,8 +1455,16 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
{
// libavif releases through 1.3 omitted alpha transform associations, so accept complete absence for
// compatibility. If either property is present, it must match the color item before plane composition.
if ((alphaItem.RotationAngle is not null || alphaItem.MirrorAxis is not null) &&
(alphaItem.RotationAngle != colorItem.RotationAngle || alphaItem.MirrorAxis != colorItem.MirrorAxis))
bool alphaHasTransforms = alphaItem.CleanAperture is not null ||
alphaItem.RotationAngle is not null ||
alphaItem.MirrorAxis is not null;
bool cleanAperturesMatch = alphaItem.CleanAperture is null
? colorItem.CleanAperture is null
: colorItem.CleanAperture is not null && alphaItem.CleanAperture.Value.Equals(colorItem.CleanAperture.Value);
if (alphaHasTransforms &&
(!cleanAperturesMatch || alphaItem.RotationAngle != colorItem.RotationAngle || alphaItem.MirrorAxis != colorItem.MirrorAxis))
{
throw new ImageFormatException("The alpha auxiliary image and color image use different presentation transforms.");
}

6
src/ImageSharp/Formats/Heif/HeifItem.cs

@ -55,6 +55,12 @@ internal class HeifItem(Heif4CharCode type, uint id)
/// </summary>
public Size PixelAspectRatio { get; set; }
/// <summary>
/// Gets or sets the clean-aperture crop applied before image rotation and mirroring, or
/// <see langword="null"/> when no clean-aperture property is associated with the item.
/// </summary>
public HeifCleanAperture? CleanAperture { get; set; }
/// <summary>
/// Gets or sets the number of 90-degree counter-clockwise rotations applied to the image, or
/// <see langword="null"/> when no image-rotation property is associated with the item.

Loading…
Cancel
Save