Browse Source

Checkpoint AV1 inter encoding and shared search operators

Preserve the current encoder, container sequence, and decoder integration work. Record focused Release verification and the remaining interpolation/conformance work in the implementation plan. Non-regular interpolation runtime verification remains open.
pull/2633/head
James Jackson-South 4 weeks ago
parent
commit
433afd1a92
  1. 55
      HEIF_IMPLEMENTATION_PLAN.md
  2. 78
      src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs
  3. 45
      src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs
  4. 79
      src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs
  5. 72
      src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs
  6. 4
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs
  7. 104
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs
  8. 225
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs
  9. 61
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs
  10. 529
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs
  11. 53
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs
  12. 45
      src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs
  13. 10
      src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs
  14. 459
      src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs
  15. 9
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs
  16. 15
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs
  17. 153
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs
  18. 8
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs
  19. 654
      src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs
  20. 92
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs
  21. 38
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs
  22. 7
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs
  23. 35
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderInterPredictionWorkspace.cs
  24. 1700
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  25. 586
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs
  26. 98
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  27. 570
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs
  28. 3
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs
  29. 1751
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs
  30. 170
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs
  31. 158
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs
  32. 220
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs
  33. 383
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs
  34. 2
      src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs
  35. 58
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs
  36. 12
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs
  37. 18
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs
  38. 126
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs
  39. 62
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs
  40. 15
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs
  41. 2
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs
  42. 9
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs
  43. 337
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs
  44. 102
      src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs
  45. 166
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs
  46. 212
      src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs
  47. 454
      src/ImageSharp/Formats/Heif/HeifDecoderCore.cs
  48. 8
      src/ImageSharp/Formats/Heif/HeifEncoder.cs
  49. 265
      src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs
  50. 623
      src/ImageSharp/Formats/Heif/HeifEncoderCore.cs
  51. 15
      src/ImageSharp/Formats/Heif/HeifItem.cs
  52. 16
      src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs
  53. 33
      src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs
  54. 20
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs
  55. 174
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs
  56. 282
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs
  57. 187
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs
  58. 218
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs
  59. 44
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs
  60. 224
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs
  61. 31
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs
  62. 140
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs
  63. 77
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs
  64. 4
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs
  65. 105
      tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs
  66. 476
      tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs

55
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

78
src/ImageSharp/Formats/Heif/Av1/Av1BitStreamWriter.cs

@ -173,6 +173,84 @@ internal ref struct Av1BitStreamWriter
} }
} }
/// <summary>
/// Writes a finite subexponential value recentered around a signed reference value.
/// </summary>
/// <param name="value">The signed value to write.</param>
/// <param name="valueMagnitude">One greater than the maximum absolute value in the signed domain.</param>
/// <param name="groupBitCount">The bit width of the first subexponential group.</param>
/// <param name="reference">The signed reference value around which smaller codewords are concentrated.</param>
public void WriteSignedReferenceSubexponential(int value, int valueMagnitude, int groupBitCount, int reference)
{
int shiftedReference = reference + valueMagnitude - 1;
int shiftedValue = value + valueMagnitude - 1;
int scaledValueCount = (valueMagnitude << 1) - 1;
int recenteredValue = RecenterFiniteNonNegative(scaledValueCount, shiftedReference, shiftedValue);
this.WriteSubexponential(recenteredValue, scaledValueCount, groupBitCount);
}
/// <summary>
/// Writes one value using a finite sequence of exponentially growing code groups.
/// </summary>
private void WriteSubexponential(int value, int valueCount, int groupBitCount)
{
int groupIndex = 0;
int groupStart = 0;
while (true)
{
// The first two groups retain the initial width. Later groups grow one bit at a time until the
// finite tail is small enough for the exact non-symmetric alphabet.
int bitCount = groupIndex == 0 ? groupBitCount : groupBitCount + groupIndex - 1;
int groupSize = 1 << bitCount;
if (valueCount <= groupStart + (3 * groupSize))
{
this.WriteNonSymmetric((uint)(value - groupStart), (uint)(valueCount - groupStart));
return;
}
bool useLaterGroup = value >= groupStart + groupSize;
this.WriteBoolean(useLaterGroup);
if (!useLaterGroup)
{
this.WriteLiteral((uint)(value - groupStart), bitCount);
return;
}
groupIndex++;
groupStart += groupSize;
}
}
/// <summary>
/// Maps an unsigned value to increasing distance from a reference inside a finite domain.
/// </summary>
private static int RecenterFiniteNonNegative(int valueCount, int reference, int value)
{
if ((reference << 1) <= valueCount)
{
return RecenterNonNegative(reference, value);
}
return RecenterNonNegative(valueCount - 1 - reference, valueCount - 1 - value);
}
/// <summary>
/// Maps an unsigned value to alternating positions around a nonnegative reference.
/// </summary>
private static int RecenterNonNegative(int reference, int value)
{
if (value > (reference << 1))
{
return value;
}
return value >= reference
? (value - reference) << 1
: ((reference - value) << 1) - 1;
}
/// <summary> /// <summary>
/// Appends one bit to the partially assembled output byte. /// Appends one bit to the partially assembled output byte.
/// </summary> /// </summary>

45
src/ImageSharp/Formats/Heif/Av1/Av1Constants.cs

@ -21,6 +21,41 @@ internal static class Av1Constants
/// </summary> /// </summary>
public const int LevelBits = 5; public const int LevelBits = 5;
/// <summary>
/// The number of bits used for the zero-based operating-point count.
/// </summary>
public const int OperatingPointCountBits = 5;
/// <summary>
/// The number of bits used for an operating-point layer-selection mask.
/// </summary>
public const int OperatingPointIdcBits = 12;
/// <summary>
/// The number of bits used for a frame type.
/// </summary>
public const int FrameTypeBits = 2;
/// <summary>
/// The first sequence-level index that carries an explicit tier bit.
/// </summary>
public const int SequenceTierMinimumLevelIndex = 8;
/// <summary>
/// The number of bits used to select a frame from the eight-slot reference map.
/// </summary>
public const int ReferenceFrameIndexBits = 3;
/// <summary>
/// The sequence-header value that lets each frame choose whether to use screen-content tools.
/// </summary>
public const int SelectScreenContentTools = 2;
/// <summary>
/// The sequence-header value that lets each applicable frame choose whether to require integer motion vectors.
/// </summary>
public const int SelectIntegerMotionVector = 2;
/// <summary> /// <summary>
/// The maximum number of operating points declared by one AV1 sequence header. /// The maximum number of operating points declared by one AV1 sequence header.
/// </summary> /// </summary>
@ -46,6 +81,11 @@ internal static class Av1Constants
/// </summary> /// </summary>
public const int ReferencesPerFrame = 7; public const int ReferencesPerFrame = 7;
/// <summary>
/// The largest frame width or height representable by the 16-bit AV1 dimension syntax.
/// </summary>
public const int MaxFrameDimension = 1 << 16;
/// <summary> /// <summary>
/// The maximum area of a tile in units of luma samples. /// The maximum area of a tile in units of luma samples.
/// </summary> /// </summary>
@ -66,6 +106,11 @@ internal static class Av1Constants
/// </summary> /// </summary>
public const int MaxTileRowCount = 64; public const int MaxTileRowCount = 64;
/// <summary>
/// The number of 64x64 CDEF filter units in a 128x128 superblock.
/// </summary>
public const int CdefUnitsPerSuperblock = 4;
/// <summary> /// <summary>
/// The number of frames that can be stored for future reference. /// The number of frames that can be stored for future reference.
/// </summary> /// </summary>

79
src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs

@ -261,6 +261,52 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
return this.ConvertToFrame<TPixel>(outputFrame.FrameBuffer, outputFrame.FrameHeader, effectiveColorProfile); return this.ConvertToFrame<TPixel>(outputFrame.FrameBuffer, outputFrame.FrameHeader, effectiveColorProfile);
} }
/// <summary>
/// Decodes the next visible sample in a bounded AV1 image sequence directly into a caller-owned frame.
/// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <param name="buffer">The complete AV1 sample payload.</param>
/// <param name="containerColorProfile">The container color description.</param>
/// <param name="codecConfiguration">The AV1 sample-entry configuration.</param>
/// <param name="expectedCodedSize">The coded dimensions declared by the visual sample entry.</param>
/// <param name="sourceRectangle">The clean-aperture region mapped to the complete destination frame.</param>
/// <param name="destination">The caller-owned packed-pixel frame receiving the presented sample.</param>
public void DecodeSequenceFrame<TPixel>(
Span<byte> buffer,
CicpProfile? containerColorProfile,
Av1CodecConfiguration? codecConfiguration,
Size expectedCodedSize,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination)
where TPixel : unmanaged, IPixel<TPixel>
{
CicpProfile effectiveColorProfile = this.DecodePayload(
buffer,
containerColorProfile,
codecConfiguration,
null,
requireShownFrame: true);
Av1ReferenceFrame outputFrame = this.referenceFrames.ResolveOutput();
Size codedSize = new(
outputFrame.FrameHeader.FrameSize.SuperResolutionUpscaledWidth,
outputFrame.FrameHeader.FrameSize.FrameHeight);
if (codedSize != expectedCodedSize)
{
throw new InvalidImageContentException(
"The decoded image-sequence sample dimensions do not match its visual sample entry.");
}
Av1YuvConverter.ConvertRegionToRgb(
this.configuration,
outputFrame.FrameBuffer,
sourceRectangle,
destination);
destination.Metadata.CicpProfile = effectiveColorProfile.DeepClone();
}
/// <summary> /// <summary>
/// Decodes one non-presented AV1 image-sequence sample while retaining its reference state. /// Decodes one non-presented AV1 image-sequence sample while retaining its reference state.
/// </summary> /// </summary>
@ -286,6 +332,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
/// <param name="containerColorProfile">The container color description.</param> /// <param name="containerColorProfile">The container color description.</param>
/// <param name="codecConfiguration">The AV1 sample-entry configuration.</param> /// <param name="codecConfiguration">The AV1 sample-entry configuration.</param>
/// <param name="expectedCodedSize">The required coded dimensions.</param> /// <param name="expectedCodedSize">The required coded dimensions.</param>
/// <param name="sourceRectangle">The clean-aperture luma region mapped to the destination.</param>
/// <param name="destination">The packed color frame receiving alpha values.</param> /// <param name="destination">The packed color frame receiving alpha values.</param>
/// <param name="outputSize">The complete presented size of the auxiliary image.</param> /// <param name="outputSize">The complete presented size of the auxiliary image.</param>
/// <param name="destinationRectangle">The destination region receiving the alpha image.</param> /// <param name="destinationRectangle">The destination region receiving the alpha image.</param>
@ -295,6 +342,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
CicpProfile? containerColorProfile, CicpProfile? containerColorProfile,
Av1CodecConfiguration? codecConfiguration, Av1CodecConfiguration? codecConfiguration,
Size expectedCodedSize, Size expectedCodedSize,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination, ImageFrame<TPixel> destination,
Size outputSize, Size outputSize,
Rectangle destinationRectangle, Rectangle destinationRectangle,
@ -312,6 +360,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
this.ComposeAlpha( this.ComposeAlpha(
outputFrame.FrameBuffer, outputFrame.FrameBuffer,
expectedCodedSize, expectedCodedSize,
sourceRectangle,
destination, destination,
outputSize, outputSize,
destinationRectangle, destinationRectangle,
@ -424,6 +473,35 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
Rectangle destinationRectangle, Rectangle destinationRectangle,
bool premultiplied) bool premultiplied)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
=> this.ComposeAlpha(
frameBuffer,
expectedCodedSize,
new Rectangle(0, 0, frameBuffer.Width, frameBuffer.Height),
destination,
outputSize,
destinationRectangle,
premultiplied);
/// <summary>
/// Composes one decoded monochrome region into a packed color frame.
/// </summary>
/// <typeparam name="TPixel">The destination color pixel type.</typeparam>
/// <param name="frameBuffer">The decoded monochrome planes.</param>
/// <param name="expectedCodedSize">The required coded dimensions.</param>
/// <param name="sourceRectangle">The luma region mapped to the destination rectangle.</param>
/// <param name="destination">The packed color frame receiving alpha values.</param>
/// <param name="outputSize">The complete presented size of the auxiliary image.</param>
/// <param name="destinationRectangle">The destination region receiving the alpha image.</param>
/// <param name="premultiplied">Whether stored color samples must be converted to unassociated alpha.</param>
private void ComposeAlpha<TPixel>(
Av1FrameBuffer<byte> frameBuffer,
Size expectedCodedSize,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination,
Size outputSize,
Rectangle destinationRectangle,
bool premultiplied)
where TPixel : unmanaged, IPixel<TPixel>
{ {
if (expectedCodedSize != default && (frameBuffer.Width != expectedCodedSize.Width || frameBuffer.Height != expectedCodedSize.Height)) if (expectedCodedSize != default && (frameBuffer.Width != expectedCodedSize.Width || frameBuffer.Height != expectedCodedSize.Height))
{ {
@ -440,6 +518,7 @@ internal sealed class Av1Decoder : IAv1TileReader, IDisposable
Av1YuvConverter.ComposeAlpha( Av1YuvConverter.ComposeAlpha(
this.configuration, this.configuration,
frameBuffer, frameBuffer,
sourceRectangle,
destination, destination,
outputSize, outputSize,
destinationRectangle, destinationRectangle,

72
src/ImageSharp/Formats/Heif/Av1/Color/Av1YuvConverter.cs

@ -92,6 +92,48 @@ internal static class Av1YuvConverter
mode); mode);
} }
/// <summary>
/// Converts a rectangular region of reconstructed component planes directly to packed pixels.
/// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
/// <param name="frameBuffer">The reconstructed AV1 frame.</param>
/// <param name="sourceRectangle">The luma-sample region mapped to the complete destination frame.</param>
/// <param name="image">The destination image frame.</param>
public static void ConvertRegionToRgb<TPixel>(
Configuration configuration,
Av1FrameBuffer<byte> frameBuffer,
Rectangle sourceRectangle,
ImageFrame<TPixel> image)
where TPixel : unmanaged, IPixel<TPixel>
{
HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out HeifColorConversionMode mode);
if (frameBuffer.BitDepth == Av1BitDepth.EightBit)
{
Av1PlanarSampleBuffer<byte> buffer = new(frameBuffer);
HeifPlanarColorConverter.ConvertToRgb<TPixel, Av1PlanarSampleBuffer<byte>, byte, HeifByteSampleConverter>(
configuration,
buffer,
image,
in parameters,
mode,
sourceRectangle.X,
sourceRectangle.Y);
return;
}
Av1PlanarSampleBuffer<ushort> highBitDepthBuffer = new(frameBuffer);
HeifPlanarColorConverter.ConvertToRgb<TPixel, Av1PlanarSampleBuffer<ushort>>(
configuration,
highBitDepthBuffer,
image,
in parameters,
mode,
sourceRectangle.X,
sourceRectangle.Y);
}
/// <summary> /// <summary>
/// Composes the reconstructed luma plane into a packed color frame as auxiliary alpha. /// Composes the reconstructed luma plane into a packed color frame as auxiliary alpha.
/// </summary> /// </summary>
@ -110,9 +152,37 @@ internal static class Av1YuvConverter
Rectangle destinationRectangle, Rectangle destinationRectangle,
bool premultiplied) bool premultiplied)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
=> ComposeAlpha(
configuration,
frameBuffer,
new Rectangle(0, 0, frameBuffer.Width, frameBuffer.Height),
destination,
outputSize,
destinationRectangle,
premultiplied);
/// <summary>
/// Composes a rectangular reconstructed luma region into a packed color frame as auxiliary alpha.
/// </summary>
/// <typeparam name="TPixel">The destination color pixel type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
/// <param name="frameBuffer">The reconstructed AV1 frame containing the alpha luma plane.</param>
/// <param name="sourceRectangle">The luma-sample region mapped to the destination rectangle.</param>
/// <param name="destination">The packed color frame receiving alpha values.</param>
/// <param name="outputSize">The complete presented size of the auxiliary image or grid tile.</param>
/// <param name="destinationRectangle">The destination region receiving the presented alpha image.</param>
/// <param name="premultiplied">Whether stored color samples must be converted to unassociated alpha.</param>
public static void ComposeAlpha<TPixel>(
Configuration configuration,
Av1FrameBuffer<byte> frameBuffer,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination,
Size outputSize,
Rectangle destinationRectangle,
bool premultiplied)
where TPixel : unmanaged, IPixel<TPixel>
{ {
HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out _); HeifColorConversionParameters parameters = GetConversionParameters(frameBuffer.ColorConfig, out _);
Rectangle sourceRectangle = new(0, 0, frameBuffer.Width, frameBuffer.Height);
if (frameBuffer.BitDepth == Av1BitDepth.EightBit) if (frameBuffer.BitDepth == Av1BitDepth.EightBit)
{ {
Av1PlanarSampleBuffer<byte> buffer = new(frameBuffer); Av1PlanarSampleBuffer<byte> buffer = new(frameBuffer);

4
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1FrameEntropyContext.cs

@ -512,8 +512,8 @@ internal sealed class Av1FrameEntropyContext
{ {
int qContext = GetQContext(qIndex); int qContext = GetQContext(qIndex);
// The prototypes are never exposed to a range reader. Copying their state lets a decoder session reuse the // The prototypes are never exposed to a range reader or writer. Copying their state lets each codec session
// same three mutable object graphs even when successive frames select different coefficient-model bands. // reuse its mutable object graphs even when successive frames select different coefficient-model bands.
this.CopyFrom(DefaultPrototypes[qContext]); this.CopyFrom(DefaultPrototypes[qContext]);
} }

104
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1MotionVectorContext.cs

@ -41,10 +41,7 @@ internal sealed class Av1MotionVectorContext
/// <param name="symbol">The zero-based symbol.</param> /// <param name="symbol">The zero-based symbol.</param>
/// <param name="distribution">The live symbol distribution.</param> /// <param name="distribution">The live symbol distribution.</param>
/// <returns>The symbol cost in 1/512-bit units, or zero when writing.</returns> /// <returns>The symbol cost in 1/512-bit units, or zero when writing.</returns>
public static abstract int ProcessSymbol( public static abstract int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution);
Av1SymbolWriter writer,
int symbol,
Av1Distribution distribution);
} }
/// <summary> /// <summary>
@ -103,13 +100,14 @@ internal sealed class Av1MotionVectorContext
} }
/// <summary> /// <summary>
/// Writes an integer displacement vector relative to a spatially derived reference. /// Writes a motion vector relative to a spatially derived reference.
/// </summary> /// </summary>
/// <param name="writer">The tile range encoder.</param> /// <param name="writer">The tile range encoder.</param>
/// <param name="value">The displacement vector to encode.</param> /// <param name="value">The displacement vector to encode.</param>
/// <param name="reference">The spatially derived reference vector.</param> /// <param name="reference">The spatially derived reference vector.</param>
public void Write(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference) /// <param name="precision">The fractional precision selected by the frame header.</param>
=> _ = this.Process<MotionVectorWriteOperation>(writer, value, reference); public void Write(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision)
=> _ = this.Process<MotionVectorWriteOperation>(writer, value, reference, precision);
/// <summary> /// <summary>
/// Measures a motion-vector delta against the live distributions without changing them. /// Measures a motion-vector delta against the live distributions without changing them.
@ -117,17 +115,15 @@ internal sealed class Av1MotionVectorContext
/// <param name="writer">The tile range encoder associated with the live context.</param> /// <param name="writer">The tile range encoder associated with the live context.</param>
/// <param name="value">The motion vector to measure.</param> /// <param name="value">The motion vector to measure.</param>
/// <param name="reference">The spatially derived reference vector.</param> /// <param name="reference">The spatially derived reference vector.</param>
/// <param name="precision">The fractional precision selected by the frame header.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns> /// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetCost(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference) public int GetCost(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision)
=> this.Process<MotionVectorCostOperation>(writer, value, reference); => this.Process<MotionVectorCostOperation>(writer, value, reference, precision);
/// <summary> /// <summary>
/// Processes one complete motion-vector delta through a closed symbol operation. /// Processes one complete motion-vector delta through a closed symbol operation.
/// </summary> /// </summary>
private int Process<TOperation>( private int Process<TOperation>(Av1SymbolWriter writer, Av1MotionVector value, Av1MotionVector reference, Av1MotionVectorPrecision precision)
Av1SymbolWriter writer,
Av1MotionVector value,
Av1MotionVector reference)
where TOperation : struct, IMotionVectorSymbolOperation where TOperation : struct, IMotionVectorSymbolOperation
{ {
int row = value.Row - reference.Row; int row = value.Row - reference.Row;
@ -140,12 +136,12 @@ internal sealed class Av1MotionVectorContext
int rate = TOperation.ProcessSymbol(writer, jointType, this.Joint); int rate = TOperation.ProcessSymbol(writer, jointType, this.Joint);
if (row != 0) if (row != 0)
{ {
rate += this.Vertical.Process<TOperation>(writer, row); rate += this.Vertical.Process<TOperation>(writer, row, precision);
} }
if (column != 0) if (column != 0)
{ {
rate += this.Horizontal.Process<TOperation>(writer, column); rate += this.Horizontal.Process<TOperation>(writer, column, precision);
} }
return rate; return rate;
@ -157,10 +153,7 @@ internal sealed class Av1MotionVectorContext
private readonly struct MotionVectorWriteOperation : IMotionVectorSymbolOperation private readonly struct MotionVectorWriteOperation : IMotionVectorSymbolOperation
{ {
/// <inheritdoc/> /// <inheritdoc/>
public static int ProcessSymbol( public static int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution)
Av1SymbolWriter writer,
int symbol,
Av1Distribution distribution)
{ {
writer.WriteSymbol(symbol, distribution); writer.WriteSymbol(symbol, distribution);
return 0; return 0;
@ -173,10 +166,7 @@ internal sealed class Av1MotionVectorContext
private readonly struct MotionVectorCostOperation : IMotionVectorSymbolOperation private readonly struct MotionVectorCostOperation : IMotionVectorSymbolOperation
{ {
/// <inheritdoc/> /// <inheritdoc/>
public static int ProcessSymbol( public static int ProcessSymbol(Av1SymbolWriter writer, int symbol, Av1Distribution distribution)
Av1SymbolWriter writer,
int symbol,
Av1Distribution distribution)
=> Av1ProbabilityCost.GetSymbolCost(distribution, symbol); => Av1ProbabilityCost.GetSymbolCost(distribution, symbol);
} }
@ -350,48 +340,72 @@ internal sealed class Av1MotionVectorContext
} }
/// <summary> /// <summary>
/// Writes one signed integer-precision component. /// Writes one signed motion-vector component.
/// </summary> /// </summary>
/// <param name="writer">The tile range encoder.</param> /// <param name="writer">The tile range encoder.</param>
/// <param name="value">The nonzero component in one-eighth-sample units.</param> /// <param name="value">The nonzero component in one-eighth-sample units.</param>
public void Write(Av1SymbolWriter writer, int value) /// <param name="precision">The fractional precision selected by the frame header.</param>
=> _ = this.Process<MotionVectorWriteOperation>(writer, value); public void Write(Av1SymbolWriter writer, int value, Av1MotionVectorPrecision precision)
=> _ = this.Process<MotionVectorWriteOperation>(writer, value, precision);
/// <summary> /// <summary>
/// Processes one nonzero signed component through the shared motion-vector symbol operation. /// Processes one nonzero signed component through the shared motion-vector symbol operation.
/// </summary> /// </summary>
public int Process<TOperation>(Av1SymbolWriter writer, int value) public int Process<TOperation>(Av1SymbolWriter writer, int value, Av1MotionVectorPrecision precision)
where TOperation : struct, IMotionVectorSymbolOperation where TOperation : struct, IMotionVectorSymbolOperation
{ {
int magnitude = Math.Abs(value); int magnitude = Math.Abs(value);
DebugGuard.IsTrue(magnitude > 0 && (magnitude & 7) == 0, "Displacement-vector components must use whole-sample precision."); int precisionMask = precision == Av1MotionVectorPrecision.Integer
? 7
// Class zero contains the two whole-sample magnitudes 8 and 16. Above it, the highest set bit of magnitude : precision == Av1MotionVectorPrecision.QuarterSample ? 1 : 0;
// minus one selects the doubling range; subtracting three converts the eighth-sample bit index to the class.
int magnitudeClass = magnitude <= (ClassZeroSize << 3) ? 0 : Av1Math.MostSignificantBit((uint)(magnitude - 1)) - 3; DebugGuard.IsTrue(
magnitude > 0 && (magnitude & precisionMask) == 0,
"Motion-vector components must match the frame precision.");
// The coded value is magnitude minus one. Its whole-sample portion selects the doubling class,
// while the remainder carries integer offset, fractional phase, and the high-precision bit.
int codedMagnitude = magnitude - 1;
uint classValue = (uint)(codedMagnitude >> 3);
int magnitudeClass = classValue == 0 ? 0 : Av1Math.MostSignificantBit(classValue);
DebugGuard.MustBeLessThan(magnitudeClass, MagnitudeClassCount, nameof(magnitudeClass)); DebugGuard.MustBeLessThan(magnitudeClass, MagnitudeClassCount, nameof(magnitudeClass));
int magnitudeBase = magnitudeClass == 0 ? 0 : ClassZeroSize << (magnitudeClass + 2);
int offset = codedMagnitude - magnitudeBase;
int integerOffset = offset >> 3;
int fractional = (offset >> 1) & 3;
int highPrecision = offset & 1;
int rate = TOperation.ProcessSymbol(writer, value < 0 ? 1 : 0, this.Sign); int rate = TOperation.ProcessSymbol(writer, value < 0 ? 1 : 0, this.Sign);
rate += TOperation.ProcessSymbol(writer, magnitudeClass, this.MagnitudeClass); rate += TOperation.ProcessSymbol(writer, magnitudeClass, this.MagnitudeClass);
if (magnitudeClass == 0) if (magnitudeClass == 0)
{ {
rate += TOperation.ProcessSymbol(writer, (magnitude >> 3) - 1, this.ClassZero); rate += TOperation.ProcessSymbol(writer, integerOffset, this.ClassZero);
return rate; }
else
{
for (int bit = 0; bit < magnitudeClass; bit++)
{
// Integer offsets are transmitted least-significant bit first through independent models.
rate += TOperation.ProcessSymbol(writer, (integerOffset >> bit) & 1, this.OffsetBits[bit]);
}
} }
// Remove the class base and the implicit low-bit value 7 plus the final one before coding the remaining if (precision != Av1MotionVectorPrecision.Integer)
// whole-sample offset least-significant bit first. {
int magnitudeBase = ClassZeroSize << (magnitudeClass + 2); Av1Distribution fractionalDistribution = magnitudeClass == 0
int integerOffset = (magnitude - magnitudeBase - 8) >> 3; ? this.ClassZeroFractional[integerOffset]
: this.Fractional;
for (int bit = 0; bit < magnitudeClass; bit++) rate += TOperation.ProcessSymbol(writer, fractional, fractionalDistribution);
}
if (precision == Av1MotionVectorPrecision.EighthSample)
{ {
// The decoder reconstructs offsets least-significant bit first, so each adaptive bit model must be Av1Distribution highPrecisionDistribution = magnitudeClass == 0
// updated in the same order during encoding. ? this.ClassZeroHighPrecision
rate += TOperation.ProcessSymbol( : this.HighPrecision;
writer,
(integerOffset >> bit) & 1, rate += TOperation.ProcessSymbol(writer, highPrecision, highPrecisionDistribution);
this.OffsetBits[bit]);
} }
return rate; return rate;

225
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1RateDistortion.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
@ -10,6 +11,108 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
/// </summary> /// </summary>
internal static class Av1RateDistortion internal static class Av1RateDistortion
{ {
/// <summary>
/// Each fitted curve contains 65 equally spaced samples, including the cubic interpolation endpoints.
/// </summary>
private const int ModelCurveLength = 65;
/// <summary>
/// Gets the rate-curve category for each AV1 block geometry.
/// </summary>
private static ReadOnlySpan<byte> ModelRateCategories => [0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 1, 1, 2, 2, 3, 3];
/// <summary>
/// Gets the four block-size rate curves in fixed-point bit-cost units per sample.
/// </summary>
private static ReadOnlySpan<double> ModelRateCurves =>
[
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 118.257702, 120.210658, 121.434853, 122.100487,
122.377758, 122.436865, 72.290102, 96.974289, 101.652727,
126.830141, 140.417377, 157.644879, 184.315291, 215.823873,
262.300169, 335.919859, 420.624173, 519.185032, 619.854243,
726.053595, 827.663369, 933.127475, 1037.988755, 1138.839609,
1233.342933, 1333.508064, 1428.760126, 1533.396364, 1616.952052,
1744.539319, 1803.413586, 1951.466618, 1994.227838, 2086.031680,
2148.635443, 2239.068450, 2222.590637, 2338.859809, 2402.929011,
2418.727875, 2435.342670, 2471.159469, 2523.187446, 2591.183827,
2674.905840, 2774.110714, 2888.555675, 3017.997952, 3162.194773,
3320.903365, 3493.880956, 3680.884773, 3881.672045, 4096.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 13.087244, 15.919735, 25.930313, 24.412411,
28.567417, 29.924194, 30.857010, 32.742979, 36.382570,
39.210386, 42.265690, 47.378572, 57.014850, 82.740067,
137.346562, 219.968084, 316.781856, 415.643773, 516.706538,
614.914364, 714.303763, 815.512135, 911.210485, 1008.501528,
1109.787854, 1213.772279, 1322.922561, 1414.752579, 1510.505641,
1615.741888, 1697.989032, 1780.123933, 1847.453790, 1913.742309,
1960.828122, 2047.500168, 2085.454095, 2129.230668, 2158.171824,
2182.231724, 2217.684864, 2269.589211, 2337.264824, 2420.618694,
2519.557814, 2633.989178, 2763.819779, 2908.956609, 3069.306660,
3244.776927, 3435.274401, 3640.706076, 3860.978945, 4096.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 4.656893, 5.123633, 5.594132, 6.162376,
6.918433, 7.768444, 8.739415, 10.105862, 11.477328,
13.236604, 15.421030, 19.093623, 25.801871, 46.724612,
98.841054, 181.113466, 272.586364, 359.499769, 445.546343,
525.944439, 605.188743, 681.793483, 756.668359, 838.486885,
926.950356, 1015.482542, 1113.353926, 1204.897193, 1288.871992,
1373.464145, 1455.746628, 1527.796460, 1588.475066, 1658.144771,
1710.302500, 1807.563351, 1863.197608, 1927.281616, 1964.450872,
2022.719898, 2100.041145, 2185.205712, 2280.993936, 2387.616216,
2505.282950, 2634.204540, 2774.591385, 2926.653884, 3090.602436,
3266.647443, 3454.999303, 3655.868416, 3869.465182, 4096.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 0.000000, 0.000000, 0.000000, 0.000000,
0.000000, 0.337370, 0.391916, 0.468839, 0.566334,
0.762564, 1.069225, 1.384361, 1.787581, 2.293948,
3.251909, 4.412991, 8.050068, 11.606073, 27.668092,
65.227758, 128.463938, 202.097653, 262.715851, 312.464873,
355.601398, 400.609054, 447.201352, 495.761568, 552.871938,
619.067625, 691.984883, 773.753288, 860.628503, 946.262808,
1019.805896, 1106.061360, 1178.422145, 1244.852258, 1302.173987,
1399.650266, 1548.092912, 1545.928652, 1670.817500, 1694.523823,
1779.195362, 1882.155494, 1990.662097, 2108.325181, 2235.456119,
2372.366287, 2519.367059, 2676.769812, 2844.885918, 3024.026754,
3214.503695, 3416.628115, 3630.711389, 3857.064892, 4096.000000,
];
/// <summary>
/// Gets the low- and high-error distortion curves in sixteenth-sample-error units.
/// </summary>
private static ReadOnlySpan<double> ModelDistortionCurves =>
[
16.000000, 15.962891, 15.925174, 15.886888, 15.848074,
15.808770, 15.769015, 15.728850, 15.688313, 15.647445,
15.606284, 15.564870, 15.525918, 15.483820, 15.373330,
15.126844, 14.637442, 14.184387, 13.560070, 12.880717,
12.165995, 11.378144, 10.438769, 9.130790, 7.487633,
5.688649, 4.267515, 3.196300, 2.434201, 1.834064,
1.369920, 1.035921, 0.775279, 0.574895, 0.427232,
0.314123, 0.233236, 0.171440, 0.128188, 0.092762,
0.067569, 0.049324, 0.036330, 0.027008, 0.019853,
0.015539, 0.011093, 0.008733, 0.007624, 0.008105,
0.005427, 0.004065, 0.003427, 0.002848, 0.002328,
0.001865, 0.001457, 0.001103, 0.000801, 0.000550,
0.000348, 0.000193, 0.000085, 0.000021, 0.000000,
16.000000, 15.996116, 15.984769, 15.966413, 15.941505,
15.910501, 15.873856, 15.832026, 15.785466, 15.734633,
15.679981, 15.621967, 15.560961, 15.460157, 15.288367,
15.052462, 14.466922, 13.921212, 13.073692, 12.222005,
11.237799, 9.985848, 8.898823, 7.423519, 5.995325,
4.773152, 3.744032, 2.938217, 2.294526, 1.762412,
1.327145, 1.020728, 0.765535, 0.570548, 0.425833,
0.313825, 0.232959, 0.171324, 0.128174, 0.092750,
0.067558, 0.049319, 0.036330, 0.027008, 0.019853,
0.015539, 0.011093, 0.008733, 0.007624, 0.008105,
0.005427, 0.004065, 0.003427, 0.002848, 0.002328,
0.001865, 0.001457, 0.001103, 0.000801, 0.000550,
0.000348, 0.000193, 0.000085, 0.000021, -0.000000,
];
/// <summary> /// <summary>
/// Gets the key-frame rate multiplier for an AV1 quantizer and sample precision. /// Gets the key-frame rate multiplier for an AV1 quantizer and sample precision.
/// </summary> /// </summary>
@ -32,6 +135,28 @@ internal static class Av1RateDistortion
return (int)Math.Max(multiplier, 1); return (int)Math.Max(multiplier, 1);
} }
/// <summary>
/// Gets the inter-frame rate multiplier for an AV1 quantizer and sample precision.
/// </summary>
/// <param name="qIndex">The segment quantizer index.</param>
/// <param name="bitDepth">The coded sample bit depth.</param>
/// <returns>The rate multiplier.</returns>
public static int GetInterFrameRateMultiplier(int qIndex, Av1BitDepth bitDepth)
{
int quantizer = Av1QuantizationLookup.GetDcQuant(qIndex, 0, bitDepth);
// Ordinary inter frames use a slightly lower rate weight than key frames, preserving more residual detail.
// Distortion remains normalized to the eight-bit domain before it is combined with this value.
long multiplier = (long)((quantizer * (long)quantizer) * (3.2 + (0.0015 * quantizer)));
int shift = (bitDepth.GetBitCount() - 8) * 2;
if (shift > 0)
{
multiplier = (multiplier + (1L << (shift - 1))) >> shift;
}
return (int)Math.Max(multiplier, 1);
}
/// <summary> /// <summary>
/// Gets a rate-distortion cost using the encoder probability-cost precision. /// Gets a rate-distortion cost using the encoder probability-cost precision.
/// </summary> /// </summary>
@ -86,10 +211,7 @@ internal static class Av1RateDistortion
/// <param name="motionVectorRate">The motion-vector syntax rate in 1/512-bit units.</param> /// <param name="motionVectorRate">The motion-vector syntax rate in 1/512-bit units.</param>
/// <param name="sumOfAbsoluteDifferences">The unnormalized sample-domain absolute difference.</param> /// <param name="sumOfAbsoluteDifferences">The unnormalized sample-domain absolute difference.</param>
/// <returns>The absolute difference plus the motion-vector search cost.</returns> /// <returns>The absolute difference plus the motion-vector search cost.</returns>
public static int GetMotionSearchSadCost( public static int GetMotionSearchSadCost(int sadPerBit, int motionVectorRate, int sumOfAbsoluteDifferences)
int sadPerBit,
int motionVectorRate,
int sumOfAbsoluteDifferences)
{ {
const int MotionRateShift = 9; const int MotionRateShift = 9;
@ -99,4 +221,99 @@ internal static class Av1RateDistortion
int motionError = (int)((weightedRate + (1 << (MotionRateShift - 1))) >> MotionRateShift); int motionError = (int)((weightedRate + (1 << (MotionRateShift - 1))) >> MotionRateShift);
return sumOfAbsoluteDifferences + motionError; return sumOfAbsoluteDifferences + motionError;
} }
/// <summary>
/// Estimates residual rate and distortion from prediction error without running transforms or quantization.
/// </summary>
/// <param name="blockSize">The plane block geometry selecting the fitted rate curve.</param>
/// <param name="squaredError">The visible prediction error normalized to eight-bit precision.</param>
/// <param name="sampleCount">The number of visible samples contributing to the error.</param>
/// <param name="acQuantizer">The plane AC dequantization step at native sample precision.</param>
/// <param name="bitDepth">The native sample precision.</param>
/// <param name="rateMultiplier">The block's rate-distortion multiplier.</param>
/// <param name="rate">The estimated residual rate in 1/512-bit units.</param>
/// <param name="distortion">The estimated residual distortion in sixteenth-sample-error units.</param>
public static void ModelPredictionError(
Av1BlockSize blockSize,
long squaredError,
int sampleCount,
int acQuantizer,
Av1BitDepth bitDepth,
int rateMultiplier,
out int rate,
out long distortion)
{
if (squaredError == 0)
{
rate = 0;
distortion = 0;
return;
}
const double CurveStart = -15.5;
const double CurveStep = 0.5;
const double EndpointMargin = 1E-6;
const double HighErrorThreshold = 16;
const int DistortionScaleShift = 4;
// Transform dequantizers are scaled by eight. Normalize both their precision and the prediction error
// before taking the logarithmic feature, so the same fitted curves serve eight-, ten-, and twelve-bit input.
int quantizerStep = Math.Max(acQuantizer >> (bitDepth.GetBitCount() - 5), 1);
double normalizedError = (double)squaredError / sampleCount;
double feature = Math.Log2(normalizedError / ((double)quantizerStep * quantizerStep));
double lastCurvePosition = CurveStart + ((ModelCurveLength - 1) * CurveStep);
feature = Math.Clamp(feature, CurveStart + CurveStep + EndpointMargin, lastCurvePosition - CurveStep - EndpointMargin);
double position = (feature - CurveStart) / CurveStep;
int index = (int)position;
double fraction = position - index;
int rateCategory = ModelRateCategories[(int)blockSize];
int distortionCategory = normalizedError > HighErrorThreshold ? 1 : 0;
ReadOnlySpan<double> ratePoints = ModelRateCurves.Slice((rateCategory * ModelCurveLength) + index - 1, 4);
ReadOnlySpan<double> distortionPoints = ModelDistortionCurves.Slice((distortionCategory * ModelCurveLength) + index - 1, 4);
double rateEstimate;
double distortionEstimate;
if (Vector128.IsHardwareAccelerated)
{
// The two lanes evaluate rate and distortion together. Keep the cubic polynomial's operation order,
// including its separate multiplies and adds, so vector and scalar rounding agree at decision boundaries.
Vector128<double> p0 = Vector128.Create(ratePoints[0], distortionPoints[0]);
Vector128<double> p1 = Vector128.Create(ratePoints[1], distortionPoints[1]);
Vector128<double> p2 = Vector128.Create(ratePoints[2], distortionPoints[2]);
Vector128<double> p3 = Vector128.Create(ratePoints[3], distortionPoints[3]);
Vector128<double> x = Vector128.Create(fraction);
Vector128<double> cubic = (Vector128.Create(3.0) * (p1 - p2)) + p3 - p0;
Vector128<double> quadratic = (Vector128.Create(2.0) * p0) - (Vector128.Create(5.0) * p1) + (Vector128.Create(4.0) * p2) - p3;
Vector128<double> result = p1 + (Vector128.Create(0.5) * x * (p2 - p0 + (x * (quadratic + (x * cubic)))));
rateEstimate = result.GetElement(0);
distortionEstimate = result.GetElement(1);
}
else
{
rateEstimate = InterpolateModelCurve(ratePoints, fraction);
distortionEstimate = InterpolateModelCurve(distortionPoints, fraction);
}
rate = (int)(Math.Max(0, rateEstimate * sampleCount) + 0.5);
distortion = (long)(Math.Max(0, (distortionEstimate * normalizedError) * sampleCount) + 0.5);
long skipDistortion = squaredError << DistortionScaleShift;
// A modeled coded residual is useful only if it beats leaving the prediction unchanged. Preserve the
// reference model's zero-rate rule instead of returning an artificially low distortion for a skipped block.
if (rate == 0 || GetCost(rateMultiplier, rate, distortion) >= GetCost(rateMultiplier, 0, skipDistortion))
{
rate = 0;
distortion = skipDistortion;
}
}
/// <summary>
/// Evaluates one fitted curve's cubic segment without fusing arithmetic operations.
/// </summary>
private static double InterpolateModelCurve(ReadOnlySpan<double> points, double fraction)
{
double cubic = (3.0 * (points[1] - points[2])) + points[3] - points[0];
double quadratic = (2.0 * points[0]) - (5.0 * points[1]) + (4.0 * points[2]) - points[3];
return points[1] + (0.5 * fraction * (points[2] - points[0] + (fraction * (quadratic + (fraction * cubic)))));
}
} }

61
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolContextHelper.cs

@ -46,7 +46,7 @@ internal static class Av1SymbolContextHelper
/// <summary> /// <summary>
/// The number of interpolation filters selectable by per-block switchable syntax. /// The number of interpolation filters selectable by per-block switchable syntax.
/// </summary> /// </summary>
private const int SwitchableInterpolationFilterCount = 3; public const int SwitchableInterpolationFilterCount = 3;
/// <summary> /// <summary>
/// The number of transform types represented by each flattened transform-set row. /// The number of transform types represented by each flattened transform-set row.
@ -961,25 +961,70 @@ internal static class Av1SymbolContextHelper
Av1BlockModeInfo? left, Av1BlockModeInfo? left,
int direction) int direction)
{ {
const int filterContextCount = SwitchableInterpolationFilterCount + 1;
const int horizontalContextOffset = filterContextCount * 2;
ReadOnlySpan<Av1ReferenceFrameType> referenceFrames = modeInfo.ReferenceFrames; ReadOnlySpan<Av1ReferenceFrameType> referenceFrames = modeInfo.ReferenceFrames;
Av1ReferenceFrameType primaryReference = referenceFrames[0]; Av1ReferenceFrameType primaryReference = referenceFrames[0];
bool isCompound = referenceFrames[1] > Av1ReferenceFrameType.Intra; bool isCompound = referenceFrames[1] > Av1ReferenceFrameType.Intra;
// The sixteen rows are laid out as single vertical, compound vertical, single horizontal, then compound
// horizontal, with four neighbor states in each group.
int context = (isCompound ? filterContextCount : 0) + (direction * horizontalContextOffset);
int leftFilter = GetReferenceInterpolationFilterContext(left, primaryReference, direction); int leftFilter = GetReferenceInterpolationFilterContext(left, primaryReference, direction);
int aboveFilter = GetReferenceInterpolationFilterContext(above, primaryReference, direction); int aboveFilter = GetReferenceInterpolationFilterContext(above, primaryReference, direction);
return GetSwitchableInterpolationContext(aboveFilter, leftFilter, isCompound, direction);
}
/// <summary>
/// Gets the switchable interpolation-filter context from the encoder's packed single-reference neighbors.
/// </summary>
/// <param name="modeInfo">The current inter block.</param>
/// <param name="macroBlock">The current block's available spatial neighbors.</param>
/// <param name="direction">Zero for the vertical filter or one for the horizontal filter.</param>
/// <returns>The single-reference context for the selected direction.</returns>
public static int GetSwitchableInterpolationContext(
Av1EncoderBlockModeInfo modeInfo,
Av1MacroBlockD macroBlock,
int direction)
{
int aboveFilter = SwitchableInterpolationFilterCount;
int leftFilter = SwitchableInterpolationFilterCount;
if (macroBlock.IsUpAvailable)
{
Av1EncoderBlockModeInfo above = macroBlock.GetRelativeModeInfo(-macroBlock.ModeInfoStride).Block;
if (above.ReferenceFrame == modeInfo.ReferenceFrame)
{
aboveFilter = (int)(direction == 0 ? above.VerticalInterpolationFilter : above.HorizontalInterpolationFilter);
}
}
if (macroBlock.IsLeftAvailable)
{
Av1EncoderBlockModeInfo left = macroBlock.GetRelativeModeInfo(-1).Block;
if (left.ReferenceFrame == modeInfo.ReferenceFrame)
{
leftFilter = (int)(direction == 0 ? left.VerticalInterpolationFilter : left.HorizontalInterpolationFilter);
}
}
return GetSwitchableInterpolationContext(aboveFilter, leftFilter, isCompound: false, direction);
}
/// <summary>
/// Combines the two neighboring filter states into the shared encoder and decoder context layout.
/// </summary>
private static int GetSwitchableInterpolationContext(int aboveFilter, int leftFilter, bool isCompound, int direction)
{
const int filterContextCount = SwitchableInterpolationFilterCount + 1;
const int horizontalContextOffset = filterContextCount * 2;
// The sixteen rows are single vertical, compound vertical, single horizontal, then compound horizontal,
// with four neighbor states in each group. Both storage representations must use this same mapping.
int context = (isCompound ? filterContextCount : 0) + (direction * horizontalContextOffset);
if (leftFilter == aboveFilter) if (leftFilter == aboveFilter)
{ {
return context + leftFilter; return context + leftFilter;
} }
// The fourth neighbor state is not a selectable Bilinear filter. It is the value the reference decoder uses when a neighbor // The fourth neighbor state means no matching primary reference or disagreement between contributing
// does not share the current primary reference, and when two contributing neighbors selected different filters. // neighbors. It is not the Bilinear filter, which is absent from the switchable alphabet.
if (leftFilter == SwitchableInterpolationFilterCount) if (leftFilter == SwitchableInterpolationFilterCount)
{ {
return context + aboveFilter; return context + aboveFilter;

529
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs

@ -6,6 +6,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
@ -22,6 +23,11 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// </summary> /// </summary>
private const int MaximumCoefficientContextCount = (Av1Constants.MaxTransformSize / 2) * (Av1Constants.MaxTransformSize / 2); private const int MaximumCoefficientContextCount = (Av1Constants.MaxTransformSize / 2) * (Av1Constants.MaxTransformSize / 2);
/// <summary>
/// Owns every mutable tile distribution and restores normative defaults without rebuilding the object graph.
/// </summary>
private readonly Av1FrameEntropyContext entropyContext;
/// <summary> /// <summary>
/// The tile-adaptive intra-block-copy distribution. /// The tile-adaptive intra-block-copy distribution.
/// </summary> /// </summary>
@ -30,7 +36,32 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <summary> /// <summary>
/// The tile-adaptive integer displacement-vector context. /// The tile-adaptive integer displacement-vector context.
/// </summary> /// </summary>
private readonly Av1MotionVectorContext displacementVector = new(); private readonly Av1MotionVectorContext displacementVector;
/// <summary>
/// The tile-adaptive normal inter motion-vector context.
/// </summary>
private readonly Av1MotionVectorContext motionVector;
/// <summary>
/// The tile-adaptive NEWMV branch distributions.
/// </summary>
private readonly Av1Distribution[] newMotionVector;
/// <summary>
/// The tile-adaptive GLOBALMV branch distributions.
/// </summary>
private readonly Av1Distribution[] zeroMotionVector;
/// <summary>
/// The tile-adaptive NEARESTMV branch distributions.
/// </summary>
private readonly Av1Distribution[] referenceMotionVector;
/// <summary>
/// The tile-adaptive dynamic-reference-list distributions.
/// </summary>
private readonly Av1Distribution[] dynamicReferenceList;
/// <summary> /// <summary>
/// The tile-adaptive partition-type distributions. /// The tile-adaptive partition-type distributions.
@ -42,6 +73,21 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// </summary> /// </summary>
private readonly Av1Distribution[][] keyFrameYMode; private readonly Av1Distribution[][] keyFrameYMode;
/// <summary>
/// The tile-adaptive inter-frame intra luma-mode distributions.
/// </summary>
private readonly Av1Distribution[] frameYMode;
/// <summary>
/// The tile-adaptive intra-versus-inter distributions.
/// </summary>
private readonly Av1Distribution[] intraInter;
/// <summary>
/// The tile-adaptive single-reference branch distributions.
/// </summary>
private readonly Av1Distribution[][] singleReference;
/// <summary> /// <summary>
/// The tile-adaptive chroma intra-mode distributions. /// The tile-adaptive chroma intra-mode distributions.
/// </summary> /// </summary>
@ -82,11 +128,6 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// </summary> /// </summary>
private readonly Av1Distribution filterIntraMode; private readonly Av1Distribution filterIntraMode;
/// <summary>
/// The palette probability state, created only when screen-content coding uses it.
/// </summary>
private PaletteEntropyContext? paletteEntropyContext;
/// <summary> /// <summary>
/// The tile-adaptive absolute quantizer delta distribution. /// The tile-adaptive absolute quantizer delta distribution.
/// </summary> /// </summary>
@ -157,20 +198,15 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// </summary> /// </summary>
private bool isDisposed; private bool isDisposed;
/// <summary>
/// The configuration providing lazily allocated coefficient scratch.
/// </summary>
private readonly Configuration configuration;
/// <summary> /// <summary>
/// The reusable padded coefficient levels used to derive entropy contexts. /// The reusable padded coefficient levels used to derive entropy contexts.
/// </summary> /// </summary>
private Av1LevelBuffer? levels; private readonly Av1LevelBuffer levels;
/// <summary> /// <summary>
/// The reusable raster-order coefficient contexts for one transform. /// The reusable raster-order coefficient contexts for one transform.
/// </summary> /// </summary>
private IMemoryOwner<sbyte>? coefficientContexts; private readonly IMemoryOwner<sbyte> coefficientContexts;
/// <summary> /// <summary>
/// The range writer producing the current tile payload. /// The range writer producing the current tile payload.
@ -183,7 +219,7 @@ internal sealed class Av1SymbolEncoder : IDisposable
private readonly int baseQIndex; private readonly int baseQIndex;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="Av1SymbolEncoder"/> class for one AV1 tile. /// Initializes a new instance of the <see cref="Av1SymbolEncoder"/> class with reusable tile state.
/// </summary> /// </summary>
/// <param name="configuration">The configuration providing output and temporary memory.</param> /// <param name="configuration">The configuration providing output and temporary memory.</param>
/// <param name="bufferLength">The complete fixed output allocation length in bytes.</param> /// <param name="bufferLength">The complete fixed output allocation length in bytes.</param>
@ -191,34 +227,50 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <param name="updateCdf">A value indicating whether encoded symbols adapt their tile distributions.</param> /// <param name="updateCdf">A value indicating whether encoded symbols adapt their tile distributions.</param>
public Av1SymbolEncoder(Configuration configuration, int bufferLength, int qIndex, bool updateCdf) public Av1SymbolEncoder(Configuration configuration, int bufferLength, int qIndex, bool updateCdf)
{ {
this.configuration = configuration; this.entropyContext = new Av1FrameEntropyContext(qIndex);
// Every default accessor creates independently mutable state. Encoding and decoding therefore begin from // Encoder and decoder now share the same mutable context shape. Every field aliases that single graph so
// equivalent tile-local models without constructing and immediately deep-copying a second object graph. // sequence samples can restore normative defaults without replacing any distribution or array.
this.tileIntraBlockCopy = Av1DefaultDistributions.IntraBlockCopy; this.tileIntraBlockCopy = this.entropyContext.IntraBlockCopy;
this.tilePartitionTypes = Av1DefaultDistributions.PartitionTypes; this.motionVector = this.entropyContext.MotionVector;
this.keyFrameYMode = Av1DefaultDistributions.KeyFrameYMode; this.displacementVector = this.entropyContext.DisplacementVector;
this.uvMode = Av1DefaultDistributions.UvMode; this.tilePartitionTypes = this.entropyContext.PartitionTypes;
this.filterIntra = Av1DefaultDistributions.FilterIntra; this.keyFrameYMode = this.entropyContext.KeyFrameYMode;
this.filterIntraMode = Av1DefaultDistributions.FilterIntraMode; this.frameYMode = this.entropyContext.FrameYMode;
this.deltaQuantizerAbsolute = Av1DefaultDistributions.DeltaQuantizerAbsolute; this.intraInter = this.entropyContext.IntraInter;
this.intraExtendedTransform = Av1DefaultDistributions.IntraExtendedTransform; this.singleReference = this.entropyContext.SingleReference;
this.interExtendedTransform = Av1DefaultDistributions.InterExtendedTransform; this.newMotionVector = this.entropyContext.NewMv;
this.transformSize = Av1DefaultDistributions.TransformSize; this.zeroMotionVector = this.entropyContext.ZeroMv;
this.transformPartition = Av1DefaultDistributions.TransformPartition; this.referenceMotionVector = this.entropyContext.RefMv;
this.segmentId = Av1DefaultDistributions.SegmentId; this.dynamicReferenceList = this.entropyContext.Drl;
this.angleDelta = Av1DefaultDistributions.AngleDelta; this.uvMode = this.entropyContext.UvMode;
this.skip = Av1DefaultDistributions.Skip; this.filterIntra = this.entropyContext.FilterIntra;
this.skipMode = Av1DefaultDistributions.SkipMode; this.filterIntraMode = this.entropyContext.FilterIntraMode;
this.chromaFromLumaSign = Av1DefaultDistributions.ChromaFromLumaSign; this.deltaQuantizerAbsolute = this.entropyContext.DeltaQuantizerAbsolute;
this.chromaFromLumaAlpha = Av1DefaultDistributions.ChromaFromLumaAlpha; this.intraExtendedTransform = this.entropyContext.IntraExtendedTransform;
this.transformBlockSkip = Av1DefaultDistributions.GetTransformBlockSkip(qIndex); this.interExtendedTransform = this.entropyContext.InterExtendedTransform;
this.endOfBlockFlag = Av1DefaultDistributions.GetEndOfBlockFlag(qIndex); this.transformSize = this.entropyContext.TransformSize;
this.coefficientsBaseRange = Av1DefaultDistributions.GetCoefficientsBaseRange(qIndex); this.transformPartition = this.entropyContext.TransformPartition;
this.coefficientsBase = Av1DefaultDistributions.GetCoefficientsBase(qIndex); this.segmentId = this.entropyContext.SegmentId;
this.coefficientsBaseEndOfBlock = Av1DefaultDistributions.GetBaseEndOfBlock(qIndex); this.angleDelta = this.entropyContext.AngleDelta;
this.dcSign = Av1DefaultDistributions.GetDcSign(qIndex); this.skip = this.entropyContext.Skip;
this.endOfBlockExtra = Av1DefaultDistributions.GetEndOfBlockExtra(qIndex); this.skipMode = this.entropyContext.SkipMode;
this.chromaFromLumaSign = this.entropyContext.ChromaFromLumaSign;
this.chromaFromLumaAlpha = this.entropyContext.ChromaFromLumaAlpha;
this.transformBlockSkip = this.entropyContext.TransformBlockSkip;
this.endOfBlockFlag = this.entropyContext.EndOfBlockFlag;
this.coefficientsBaseRange = this.entropyContext.CoefficientsBaseRange;
this.coefficientsBase = this.entropyContext.CoefficientsBase;
this.coefficientsBaseEndOfBlock = this.entropyContext.BaseEndOfBlock;
this.dcSign = this.entropyContext.DcSign;
this.endOfBlockExtra = this.entropyContext.EndOfBlockExtra;
// Transform dimensions are bounded by the AV1 coefficient-coding rules, so the complete entropy scratch
// is known with the tile output capacity and remains valid for every transform in every sequence sample.
this.levels = new Av1LevelBuffer(configuration);
this.coefficientContexts =
configuration.MemoryAllocator.Allocate<sbyte>(MaximumCoefficientContextCount);
this.writer = new(configuration, bufferLength, updateCdf); this.writer = new(configuration, bufferLength, updateCdf);
this.baseQIndex = qIndex; this.baseQIndex = qIndex;
} }
@ -287,6 +339,25 @@ internal sealed class Av1SymbolEncoder : IDisposable
int colorOrderIndex); int colorOrderIndex);
} }
/// <summary>
/// Restores the initial tile distributions and range coder while retaining their complete object graph and buffers.
/// </summary>
public void Reset()
{
this.entropyContext.ResetToDefaults(this.baseQIndex);
this.writer.Reset();
}
/// <summary>
/// Restores the initial tile distributions and begins the next tile at an offset in the retained output buffer.
/// </summary>
/// <param name="outputOffset">The first output byte available to the next tile.</param>
public void Reset(int outputOffset)
{
this.entropyContext.ResetToDefaults(this.baseQIndex);
this.writer.Reset(outputOffset);
}
/// <summary> /// <summary>
/// Writes an unsigned fixed-width literal to the tile entropy stream. /// Writes an unsigned fixed-width literal to the tile entropy stream.
/// </summary> /// </summary>
@ -342,9 +413,8 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <returns>The rate cost in 1/512-bit units.</returns> /// <returns>The rate cost in 1/512-bit units.</returns>
public int GetPaletteYModeCost(bool usePalette, int blockSizeContext, int neighborContext) public int GetPaletteYModeCost(bool usePalette, int blockSizeContext, int neighborContext)
{ {
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
return Av1ProbabilityCost.GetSymbolCost( return Av1ProbabilityCost.GetSymbolCost(
context.YMode[blockSizeContext][neighborContext], this.entropyContext.PaletteYMode[blockSizeContext][neighborContext],
usePalette ? 1 : 0); usePalette ? 1 : 0);
} }
@ -357,8 +427,7 @@ internal sealed class Av1SymbolEncoder : IDisposable
public void WritePaletteYMode(bool usePalette, int blockSizeContext, int neighborContext) public void WritePaletteYMode(bool usePalette, int blockSizeContext, int neighborContext)
{ {
ref Av1SymbolWriter w = ref this.writer; ref Av1SymbolWriter w = ref this.writer;
PaletteEntropyContext context = this.paletteEntropyContext ??= new(); w.WriteSymbol(usePalette, this.entropyContext.PaletteYMode[blockSizeContext][neighborContext]);
w.WriteSymbol(usePalette, context.YMode[blockSizeContext][neighborContext]);
} }
/// <summary> /// <summary>
@ -369,9 +438,8 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <returns>The rate cost in 1/512-bit units.</returns> /// <returns>The rate cost in 1/512-bit units.</returns>
public int GetPaletteUvModeCost(bool usePalette, bool hasLumaPalette) public int GetPaletteUvModeCost(bool usePalette, bool hasLumaPalette)
{ {
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
return Av1ProbabilityCost.GetSymbolCost( return Av1ProbabilityCost.GetSymbolCost(
context.UvMode[hasLumaPalette ? 1 : 0], this.entropyContext.PaletteUvMode[hasLumaPalette ? 1 : 0],
usePalette ? 1 : 0); usePalette ? 1 : 0);
} }
@ -383,8 +451,7 @@ internal sealed class Av1SymbolEncoder : IDisposable
public void WritePaletteUvMode(bool usePalette, bool hasLumaPalette) public void WritePaletteUvMode(bool usePalette, bool hasLumaPalette)
{ {
ref Av1SymbolWriter w = ref this.writer; ref Av1SymbolWriter w = ref this.writer;
PaletteEntropyContext context = this.paletteEntropyContext ??= new(); w.WriteSymbol(usePalette, this.entropyContext.PaletteUvMode[hasLumaPalette ? 1 : 0]);
w.WriteSymbol(usePalette, context.UvMode[hasLumaPalette ? 1 : 0]);
} }
/// <summary> /// <summary>
@ -396,10 +463,9 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <returns>The rate cost in 1/512-bit units.</returns> /// <returns>The rate cost in 1/512-bit units.</returns>
public int GetPaletteSizeCost(int paletteSize, int blockSizeContext, Av1PlaneType planeType) public int GetPaletteSizeCost(int paletteSize, int blockSizeContext, Av1PlaneType planeType)
{ {
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
Av1Distribution distribution = planeType == Av1PlaneType.Y Av1Distribution distribution = planeType == Av1PlaneType.Y
? context.YSize[blockSizeContext] ? this.entropyContext.PaletteYSize[blockSizeContext]
: context.UvSize[blockSizeContext]; : this.entropyContext.PaletteUvSize[blockSizeContext];
return Av1ProbabilityCost.GetSymbolCost(distribution, paletteSize - 2); return Av1ProbabilityCost.GetSymbolCost(distribution, paletteSize - 2);
} }
@ -413,10 +479,9 @@ internal sealed class Av1SymbolEncoder : IDisposable
public void WritePaletteSize(int paletteSize, int blockSizeContext, Av1PlaneType planeType) public void WritePaletteSize(int paletteSize, int blockSizeContext, Av1PlaneType planeType)
{ {
ref Av1SymbolWriter w = ref this.writer; ref Av1SymbolWriter w = ref this.writer;
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
Av1Distribution distribution = planeType == Av1PlaneType.Y Av1Distribution distribution = planeType == Av1PlaneType.Y
? context.YSize[blockSizeContext] ? this.entropyContext.PaletteYSize[blockSizeContext]
: context.UvSize[blockSizeContext]; : this.entropyContext.PaletteUvSize[blockSizeContext];
w.WriteSymbol(paletteSize - 2, distribution); w.WriteSymbol(paletteSize - 2, distribution);
} }
@ -435,10 +500,9 @@ internal sealed class Av1SymbolEncoder : IDisposable
int colorContext, int colorContext,
Av1PlaneType planeType) Av1PlaneType planeType)
{ {
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
Av1Distribution distribution = planeType == Av1PlaneType.Y Av1Distribution distribution = planeType == Av1PlaneType.Y
? context.YColorIndex[paletteSize - 2][colorContext] ? this.entropyContext.PaletteYColorIndex[paletteSize - 2][colorContext]
: context.UvColorIndex[paletteSize - 2][colorContext]; : this.entropyContext.PaletteUvColorIndex[paletteSize - 2][colorContext];
return Av1ProbabilityCost.GetSymbolCost(distribution, colorOrderIndex); return Av1ProbabilityCost.GetSymbolCost(distribution, colorOrderIndex);
} }
@ -457,10 +521,9 @@ internal sealed class Av1SymbolEncoder : IDisposable
Av1PlaneType planeType) Av1PlaneType planeType)
{ {
ref Av1SymbolWriter w = ref this.writer; ref Av1SymbolWriter w = ref this.writer;
PaletteEntropyContext context = this.paletteEntropyContext ??= new();
Av1Distribution distribution = planeType == Av1PlaneType.Y Av1Distribution distribution = planeType == Av1PlaneType.Y
? context.YColorIndex[paletteSize - 2][colorContext] ? this.entropyContext.PaletteYColorIndex[paletteSize - 2][colorContext]
: context.UvColorIndex[paletteSize - 2][colorContext]; : this.entropyContext.PaletteUvColorIndex[paletteSize - 2][colorContext];
w.WriteSymbol(colorOrderIndex, distribution); w.WriteSymbol(colorOrderIndex, distribution);
} }
@ -695,7 +758,7 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <param name="value">The displacement vector to encode.</param> /// <param name="value">The displacement vector to encode.</param>
/// <param name="reference">The spatially derived reference vector.</param> /// <param name="reference">The spatially derived reference vector.</param>
public void WriteDisplacementVector(Av1MotionVector value, Av1MotionVector reference) public void WriteDisplacementVector(Av1MotionVector value, Av1MotionVector reference)
=> this.displacementVector.Write(this.writer, value, reference); => this.displacementVector.Write(this.writer, value, reference, Av1MotionVectorPrecision.Integer);
/// <summary> /// <summary>
/// Measures an integer intra-block-copy displacement vector against the live distributions. /// Measures an integer intra-block-copy displacement vector against the live distributions.
@ -707,7 +770,11 @@ internal sealed class Av1SymbolEncoder : IDisposable
{ {
const int DisplacementVectorCostWeight = 120; const int DisplacementVectorCostWeight = 120;
const int WeightShift = 7; const int WeightShift = 7;
int rate = this.displacementVector.GetCost(this.writer, value, reference); int rate = this.displacementVector.GetCost(
this.writer,
value,
reference,
Av1MotionVectorPrecision.Integer);
// Displacement syntax uses a 120/128 discount during mode search; adding half the divisor rounds to nearest. // Displacement syntax uses a 120/128 discount during mode search; adding half the divisor rounds to nearest.
return ((rate * DisplacementVectorCostWeight) + (1 << (WeightShift - 1))) >> WeightShift; return ((rate * DisplacementVectorCostWeight) + (1 << (WeightShift - 1))) >> WeightShift;
@ -720,7 +787,133 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// <param name="reference">The spatially derived reference vector.</param> /// <param name="reference">The spatially derived reference vector.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns> /// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetDisplacementVectorSearchCost(Av1MotionVector value, Av1MotionVector reference) public int GetDisplacementVectorSearchCost(Av1MotionVector value, Av1MotionVector reference)
=> this.displacementVector.GetCost(this.writer, value, reference); => this.displacementVector.GetCost(
this.writer,
value,
reference,
Av1MotionVectorPrecision.Integer);
/// <summary>
/// Measures one switchable interpolation filter against its live tile distribution.
/// </summary>
/// <param name="filter">The regular, smooth, or sharp filter.</param>
/// <param name="context">The spatial filter context for the selected direction.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetSwitchableInterpolationFilterCost(Av1InterpolationFilter filter, int context)
=> Av1ProbabilityCost.GetSymbolCost(this.entropyContext.SwitchableInterpolation[context], (int)filter);
/// <summary>
/// Writes one switchable interpolation filter and updates its live tile distribution.
/// </summary>
/// <param name="filter">The regular, smooth, or sharp filter.</param>
/// <param name="context">The spatial filter context for the selected direction.</param>
public void WriteSwitchableInterpolationFilter(Av1InterpolationFilter filter, int context)
=> this.writer.WriteSymbol((int)filter, this.entropyContext.SwitchableInterpolation[context]);
/// <summary>
/// Measures a single-reference inter mode against the live branch distributions.
/// </summary>
/// <param name="mode">The new, global, nearest, or near motion-vector mode.</param>
/// <param name="modeContext">The packed context derived from the reference-vector stack.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetInterModeCost(Av1PredictionMode mode, int modeContext)
{
bool isNotNew = mode != Av1PredictionMode.NewMotionVector;
int rate = Av1ProbabilityCost.GetSymbolCost(
this.newMotionVector[Av1SymbolContextHelper.GetNewMvContext(modeContext)],
isNotNew ? 1 : 0);
if (!isNotNew)
{
return rate;
}
bool isNotGlobal = mode != Av1PredictionMode.GlobalMotionVector;
rate += Av1ProbabilityCost.GetSymbolCost(
this.zeroMotionVector[Av1SymbolContextHelper.GetZeroMvContext(modeContext)],
isNotGlobal ? 1 : 0);
if (!isNotGlobal)
{
return rate;
}
return rate + Av1ProbabilityCost.GetSymbolCost(
this.referenceMotionVector[Av1SymbolContextHelper.GetRefMvContext(modeContext)],
mode == Av1PredictionMode.NearMotionVector ? 1 : 0);
}
/// <summary>
/// Writes a single-reference inter mode through the NEWMV, GLOBALMV, and NEARESTMV branch tree.
/// </summary>
/// <param name="mode">The new, global, nearest, or near motion-vector mode.</param>
/// <param name="modeContext">The packed context derived from the reference-vector stack.</param>
public void WriteInterMode(Av1PredictionMode mode, int modeContext)
{
ref Av1SymbolWriter w = ref this.writer;
bool isNotNew = mode != Av1PredictionMode.NewMotionVector;
w.WriteSymbol(isNotNew, this.newMotionVector[Av1SymbolContextHelper.GetNewMvContext(modeContext)]);
if (!isNotNew)
{
return;
}
bool isNotGlobal = mode != Av1PredictionMode.GlobalMotionVector;
w.WriteSymbol(isNotGlobal, this.zeroMotionVector[Av1SymbolContextHelper.GetZeroMvContext(modeContext)]);
if (!isNotGlobal)
{
return;
}
w.WriteSymbol(
mode == Av1PredictionMode.NearMotionVector,
this.referenceMotionVector[Av1SymbolContextHelper.GetRefMvContext(modeContext)]);
}
/// <summary>
/// Measures one dynamic-reference-list advance decision.
/// </summary>
/// <param name="advance">Whether selection advances to the next candidate.</param>
/// <param name="context">The candidate-weight context.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetDynamicReferenceListCost(bool advance, int context)
=> Av1ProbabilityCost.GetSymbolCost(this.dynamicReferenceList[context], advance ? 1 : 0);
/// <summary>
/// Writes one dynamic-reference-list advance decision.
/// </summary>
/// <param name="advance">Whether selection advances to the next candidate.</param>
/// <param name="context">The candidate-weight context.</param>
public void WriteDynamicReferenceList(bool advance, int context)
{
ref Av1SymbolWriter w = ref this.writer;
w.WriteSymbol(advance, this.dynamicReferenceList[context]);
}
/// <summary>
/// Measures an inter motion vector relative to its selected stack reference.
/// </summary>
/// <param name="value">The selected motion vector.</param>
/// <param name="reference">The differential reference from the candidate stack.</param>
/// <param name="precision">The fractional precision selected by the frame header.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetMotionVectorCost(
Av1MotionVector value,
Av1MotionVector reference,
Av1MotionVectorPrecision precision)
=> this.motionVector.GetCost(this.writer, value, reference, precision);
/// <summary>
/// Writes an inter motion vector relative to its selected stack reference.
/// </summary>
/// <param name="value">The selected motion vector.</param>
/// <param name="reference">The differential reference from the candidate stack.</param>
/// <param name="precision">The fractional precision selected by the frame header.</param>
public void WriteMotionVector(
Av1MotionVector value,
Av1MotionVector reference,
Av1MotionVectorPrecision precision)
=> this.motionVector.Write(this.writer, value, reference, precision);
/// <summary> /// <summary>
/// Gets the current fixed-point cost of a complete block partition symbol. /// Gets the current fixed-point cost of a complete block partition symbol.
@ -1136,15 +1329,11 @@ internal sealed class Av1SymbolEncoder : IDisposable
bool clearLevels, bool clearLevels,
out Span<sbyte> coefficientContexts) out Span<sbyte> coefficientContexts)
{ {
Av1LevelBuffer levels = this.levels ??= new(this.configuration);
IMemoryOwner<sbyte> coefficientContextOwner = this.coefficientContexts ??=
this.configuration.MemoryAllocator.Allocate<sbyte>(MaximumCoefficientContextCount);
// AV1 omits high-frequency coefficients beyond 32 samples on every 64-point transform dimension. The tile // AV1 omits high-frequency coefficients beyond 32 samples on every 64-point transform dimension. The tile
// creates maximum-sized workspaces once, then changes only the active views for subsequent transform blocks. // creates maximum-sized workspaces once, then changes only the active views for subsequent transform blocks.
levels.Reset(new Size(width, height), clearLevels); this.levels.Reset(new Size(width, height), clearLevels);
coefficientContexts = coefficientContextOwner.Memory.Span[..(width * height)]; coefficientContexts = this.coefficientContexts.Memory.Span[..(width * height)];
return levels; return this.levels;
} }
/// <summary> /// <summary>
@ -1314,21 +1503,23 @@ internal sealed class Av1SymbolEncoder : IDisposable
/// </summary> /// </summary>
/// <returns>The memory owner containing the encoded tile bytes.</returns> /// <returns>The memory owner containing the encoded tile bytes.</returns>
public IMemoryOwner<byte> Exit() public IMemoryOwner<byte> Exit()
{ => this.writer.Exit();
ref Av1SymbolWriter w = ref this.writer;
return w.Exit();
}
/// <summary> /// <summary>
/// Finalizes the range-coded tile payload and exposes its encoded prefix without copying. /// Finalizes the range-coded tile payload and exposes its encoded prefix without copying.
/// </summary> /// </summary>
/// <param name="length">The number of encoded bytes in the returned memory.</param> /// <param name="length">The number of encoded bytes in the returned memory.</param>
/// <returns>The encoded prefix, valid until this encoder is disposed.</returns> /// <returns>The encoded prefix, valid until this encoder is reset or disposed.</returns>
public ReadOnlyMemory<byte> Exit(out int length) public ReadOnlyMemory<byte> Exit(out int length)
{ => this.writer.Exit(out length);
ref Av1SymbolWriter w = ref this.writer;
return w.Exit(out length); /// <summary>
} /// Exposes a prefix containing every consecutively encoded tile without copying their bytes.
/// </summary>
/// <param name="length">The number of bytes in the prefix.</param>
/// <returns>The encoded prefix, valid until this encoder is reset to offset zero or disposed.</returns>
public ReadOnlyMemory<byte> GetOutput(int length)
=> this.writer.GetOutput(length);
/// <summary> /// <summary>
/// Releases the range-coder output buffer and coefficient scratch memory. /// Releases the range-coder output buffer and coefficient scratch memory.
@ -1337,8 +1528,8 @@ internal sealed class Av1SymbolEncoder : IDisposable
{ {
if (!this.isDisposed) if (!this.isDisposed)
{ {
this.coefficientContexts?.Dispose(); this.coefficientContexts.Dispose();
this.levels?.Dispose(); this.levels.Dispose();
this.writer.Dispose(); this.writer.Dispose();
this.isDisposed = true; this.isDisposed = true;
} }
@ -1657,6 +1848,142 @@ internal sealed class Av1SymbolEncoder : IDisposable
w.WriteSymbol((int)lumaMode, this.keyFrameYMode[topContext][leftContext]); w.WriteSymbol((int)lumaMode, this.keyFrameYMode[topContext][leftContext]);
} }
/// <summary>
/// Gets the cost of an intra luma mode coded inside an inter frame.
/// </summary>
/// <param name="lumaMode">The intra luma mode.</param>
/// <param name="blockSize">The coding block size selecting the size group.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetInterFrameLumaModeCost(Av1PredictionMode lumaMode, Av1BlockSize blockSize)
=> Av1ProbabilityCost.GetSymbolCost(this.frameYMode[blockSize.GetSizeGroup()], (int)lumaMode);
/// <summary>
/// Writes an intra luma mode coded inside an inter frame.
/// </summary>
/// <param name="lumaMode">The intra luma mode.</param>
/// <param name="blockSize">The coding block size selecting the size group.</param>
public void WriteInterFrameLumaMode(Av1PredictionMode lumaMode, Av1BlockSize blockSize)
{
ref Av1SymbolWriter w = ref this.writer;
w.WriteSymbol((int)lumaMode, this.frameYMode[blockSize.GetSizeGroup()]);
}
/// <summary>
/// Gets the cost of the prediction-domain decision for an inter-frame block.
/// </summary>
/// <param name="isInter">Whether the block uses a retained reference frame.</param>
/// <param name="context">The neighboring prediction-domain context.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetIsInterCost(bool isInter, int context)
=> Av1ProbabilityCost.GetSymbolCost(this.intraInter[context], isInter ? 1 : 0);
/// <summary>
/// Writes the prediction-domain decision for an inter-frame block.
/// </summary>
/// <param name="isInter">Whether the block uses a retained reference frame.</param>
/// <param name="context">The neighboring prediction-domain context.</param>
public void WriteIsInter(bool isInter, int context)
{
ref Av1SymbolWriter w = ref this.writer;
w.WriteSymbol(isInter, this.intraInter[context]);
}
/// <summary>
/// Gets the cost of selecting one reference from the single-reference branch tree.
/// </summary>
/// <param name="referenceFrame">The selected reference-frame label.</param>
/// <param name="referenceCounts">The neighboring reference counts indexed by reference-frame label.</param>
/// <returns>The syntax cost in 1/512-bit units.</returns>
public int GetSingleReferenceCost(
Av1ReferenceFrameType referenceFrame,
ReadOnlySpan<byte> referenceCounts)
{
bool isBackward = referenceFrame >= Av1ReferenceFrameType.Backward;
int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts);
int rate = Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][0], isBackward ? 1 : 0);
if (isBackward)
{
bool isAlternate = referenceFrame == Av1ReferenceFrameType.Alternate;
context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts);
rate += Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][1], isAlternate ? 1 : 0);
if (isAlternate)
{
return rate;
}
context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts);
return rate + Av1ProbabilityCost.GetSymbolCost(
this.singleReference[context][5],
referenceFrame == Av1ReferenceFrameType.Alternate2 ? 1 : 0);
}
bool isLast3OrGolden = referenceFrame is Av1ReferenceFrameType.Last3 or Av1ReferenceFrameType.Golden;
context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts);
rate += Av1ProbabilityCost.GetSymbolCost(this.singleReference[context][2], isLast3OrGolden ? 1 : 0);
if (isLast3OrGolden)
{
context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts);
return rate + Av1ProbabilityCost.GetSymbolCost(
this.singleReference[context][4],
referenceFrame == Av1ReferenceFrameType.Golden ? 1 : 0);
}
context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts);
return rate + Av1ProbabilityCost.GetSymbolCost(
this.singleReference[context][3],
referenceFrame == Av1ReferenceFrameType.Last2 ? 1 : 0);
}
/// <summary>
/// Writes one reference through the single-reference branch tree.
/// </summary>
/// <param name="referenceFrame">The selected reference-frame label.</param>
/// <param name="referenceCounts">The neighboring reference counts indexed by reference-frame label.</param>
public void WriteSingleReference(
Av1ReferenceFrameType referenceFrame,
ReadOnlySpan<byte> referenceCounts)
{
ref Av1SymbolWriter w = ref this.writer;
bool isBackward = referenceFrame >= Av1ReferenceFrameType.Backward;
int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts);
w.WriteSymbol(isBackward, this.singleReference[context][0]);
if (isBackward)
{
bool isAlternate = referenceFrame == Av1ReferenceFrameType.Alternate;
context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts);
w.WriteSymbol(isAlternate, this.singleReference[context][1]);
if (isAlternate)
{
return;
}
context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts);
w.WriteSymbol(
referenceFrame == Av1ReferenceFrameType.Alternate2,
this.singleReference[context][5]);
return;
}
bool isLast3OrGolden = referenceFrame is Av1ReferenceFrameType.Last3 or Av1ReferenceFrameType.Golden;
context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts);
w.WriteSymbol(isLast3OrGolden, this.singleReference[context][2]);
if (isLast3OrGolden)
{
context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts);
w.WriteSymbol(
referenceFrame == Av1ReferenceFrameType.Golden,
this.singleReference[context][4]);
return;
}
context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts);
w.WriteSymbol(
referenceFrame == Av1ReferenceFrameType.Last2,
this.singleReference[context][3]);
}
/// <summary> /// <summary>
/// Gets the current fixed-point cost of a directional angle-delta symbol. /// Gets the current fixed-point cost of a directional angle-delta symbol.
/// </summary> /// </summary>
@ -2087,40 +2414,4 @@ internal sealed class Av1SymbolEncoder : IDisposable
colorContext, colorContext,
planeType); planeType);
} }
/// <summary>
/// Owns the adaptive distributions used only by AV1 palette syntax.
/// </summary>
private sealed class PaletteEntropyContext
{
/// <summary>
/// Gets the luma palette-mode distributions.
/// </summary>
public Av1Distribution[][] YMode { get; } = Av1DefaultDistributions.PaletteYMode;
/// <summary>
/// Gets the chroma palette-mode distributions.
/// </summary>
public Av1Distribution[] UvMode { get; } = Av1DefaultDistributions.PaletteUvMode;
/// <summary>
/// Gets the luma palette-size distributions.
/// </summary>
public Av1Distribution[] YSize { get; } = Av1DefaultDistributions.PaletteYSize;
/// <summary>
/// Gets the chroma palette-size distributions.
/// </summary>
public Av1Distribution[] UvSize { get; } = Av1DefaultDistributions.PaletteUvSize;
/// <summary>
/// Gets the luma palette color-index distributions.
/// </summary>
public Av1Distribution[][] YColorIndex { get; } = Av1DefaultDistributions.PaletteYColorIndex;
/// <summary>
/// Gets the chroma palette color-index distributions.
/// </summary>
public Av1Distribution[][] UvColorIndex { get; } = Av1DefaultDistributions.PaletteUvColorIndex;
}
} }

53
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolWriter.cs

@ -12,6 +12,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
/// </summary> /// </summary>
internal sealed class Av1SymbolWriter : IDisposable internal sealed class Av1SymbolWriter : IDisposable
{ {
/// <summary>
/// The normalized range before the first symbol narrows the coding interval.
/// </summary>
private const uint InitialRange = 0x8000U;
/// <summary>
/// The initial bit count that crosses the first byte-and-carry flush boundary after one output byte.
/// </summary>
private const int InitialCount = -9;
/// <summary> /// <summary>
/// The lower endpoint of the current coding interval. /// The lower endpoint of the current coding interval.
/// </summary> /// </summary>
@ -20,7 +30,7 @@ internal sealed class Av1SymbolWriter : IDisposable
/// <summary> /// <summary>
/// The width of the current normalized coding interval. /// The width of the current normalized coding interval.
/// </summary> /// </summary>
private uint rng = 0x8000U; private uint rng = InitialRange;
/// <summary> /// <summary>
/// The number of accumulated bits relative to the next byte-and-carry flush boundary. /// The number of accumulated bits relative to the next byte-and-carry flush boundary.
@ -28,7 +38,7 @@ internal sealed class Av1SymbolWriter : IDisposable
/// <remarks> /// <remarks>
/// The initial value of -9 crosses zero after one output byte and its carry bit have accumulated. /// The initial value of -9 crosses zero after one output byte and its carry bit have accumulated.
/// </remarks> /// </remarks>
private int cnt = -9; private int cnt = InitialCount;
/// <summary> /// <summary>
/// The configuration that supplies output allocation. /// The configuration that supplies output allocation.
@ -36,14 +46,19 @@ internal sealed class Av1SymbolWriter : IDisposable
private readonly Configuration configuration; private readonly Configuration configuration;
/// <summary> /// <summary>
/// The owner of the fixed output buffer supplied for this tile. /// The owner of the fixed output buffer shared by consecutively encoded tiles.
/// </summary> /// </summary>
private readonly IMemoryOwner<byte> bufferOwner; private readonly IMemoryOwner<byte> bufferOwner;
/// <summary>
/// The complete requested output allocation, including every consecutively encoded tile.
/// </summary>
private readonly Memory<byte> outputBuffer;
/// <summary> /// <summary>
/// The requested output range, excluding any excess capacity returned by a pooling allocator. /// The requested output range, excluding any excess capacity returned by a pooling allocator.
/// </summary> /// </summary>
private readonly Memory<byte> buffer; private Memory<byte> buffer;
/// <summary> /// <summary>
/// Indicates whether encoded symbols adapt their distributions. /// Indicates whether encoded symbols adapt their distributions.
@ -65,10 +80,29 @@ internal sealed class Av1SymbolWriter : IDisposable
{ {
this.configuration = configuration; this.configuration = configuration;
this.bufferOwner = configuration.MemoryAllocator.Allocate<byte>(bufferLength); this.bufferOwner = configuration.MemoryAllocator.Allocate<byte>(bufferLength);
this.buffer = this.bufferOwner.Memory[..bufferLength]; this.outputBuffer = this.bufferOwner.Memory[..bufferLength];
this.buffer = this.outputBuffer;
this.updateCdf = updateCdf; this.updateCdf = updateCdf;
} }
/// <summary>
/// Restores the initial range-coder state while retaining the bounded output allocation.
/// </summary>
public void Reset() => this.Reset(0);
/// <summary>
/// Restores the initial range-coder state and begins writing at an offset in the retained output allocation.
/// </summary>
/// <param name="outputOffset">The first byte available to the next range-coded tile.</param>
public void Reset(int outputOffset)
{
this.buffer = this.outputBuffer[outputOffset..];
this.low = 0;
this.rng = InitialRange;
this.cnt = InitialCount;
this.position = 0;
}
/// <summary> /// <summary>
/// Releases the tile output buffer. /// Releases the tile output buffer.
/// </summary> /// </summary>
@ -147,13 +181,20 @@ internal sealed class Av1SymbolWriter : IDisposable
/// Finalizes the range-coded sequence and exposes its encoded prefix without copying. /// Finalizes the range-coded sequence and exposes its encoded prefix without copying.
/// </summary> /// </summary>
/// <param name="length">The number of encoded bytes in the returned memory.</param> /// <param name="length">The number of encoded bytes in the returned memory.</param>
/// <returns>The encoded prefix, valid until this writer is disposed.</returns> /// <returns>The encoded prefix, valid until this writer is reset or disposed.</returns>
public ReadOnlyMemory<byte> Exit(out int length) public ReadOnlyMemory<byte> Exit(out int length)
{ {
length = this.FinalizeRange(); length = this.FinalizeRange();
return this.buffer[..length]; return this.buffer[..length];
} }
/// <summary>
/// Exposes a prefix containing consecutively encoded tiles without copying their bytes.
/// </summary>
/// <param name="length">The number of bytes in the prefix.</param>
/// <returns>The encoded prefix, valid until this writer is reset to offset zero or disposed.</returns>
public ReadOnlyMemory<byte> GetOutput(int length) => this.outputBuffer[..length];
/// <summary> /// <summary>
/// Terminates the range-coded sequence in the current output allocation. /// Terminates the range-coded sequence in the current output allocation.
/// </summary> /// </summary>

45
src/ImageSharp/Formats/Heif/Av1/Motion/Av1GlobalMotionParameters.cs

@ -20,6 +20,51 @@ internal struct Av1GlobalMotionParameters
/// </summary> /// </summary>
public const int ModelScale = 1 << ModelPrecisionBits; public const int ModelScale = 1 << ModelPrecisionBits;
/// <summary>
/// The initial finite-subexponential group width used by every global-motion parameter.
/// </summary>
public const int SubexponentialGroupBitCount = 3;
/// <summary>
/// The finite signed-domain size parameter for coded affine coefficients.
/// </summary>
public const int AlphaValueMagnitude = (1 << 12) + 1;
/// <summary>
/// The number of fractional bits carried by coded affine coefficients.
/// </summary>
public const int AlphaPrecisionBits = 15;
/// <summary>
/// The precision increase from a coded affine coefficient to the stored matrix.
/// </summary>
public const int AlphaPrecisionDifference = ModelPrecisionBits - AlphaPrecisionBits;
/// <summary>
/// The scale factor that restores a coded affine coefficient to the stored matrix precision.
/// </summary>
public const int AlphaDecodeFactor = 1 << AlphaPrecisionDifference;
/// <summary>
/// The signed magnitude bit count of a general affine model's translation components.
/// </summary>
public const int AbsoluteTranslationBits = 12;
/// <summary>
/// The signed magnitude bit count of a translation-only model before precision adjustment.
/// </summary>
public const int AbsoluteTranslationOnlyBits = 9;
/// <summary>
/// The number of fractional bits carried by general affine translation components.
/// </summary>
public const int TranslationPrecisionBits = 6;
/// <summary>
/// The number of fractional bits carried by translation-only components.
/// </summary>
public const int TranslationOnlyPrecisionBits = 3;
/// <summary> /// <summary>
/// The number of low-order bits removed from the derived shear parameters. /// The number of low-order bits removed from the derived shear parameters.
/// </summary> /// </summary>

10
src/ImageSharp/Formats/Heif/Av1/Motion/Av1MotionVector.cs

@ -13,6 +13,16 @@ internal readonly struct Av1MotionVector : IEquatable<Av1MotionVector>
/// </summary> /// </summary>
public const int MaximumTemporalDistance = 31; public const int MaximumTemporalDistance = 31;
/// <summary>
/// The number of fractional bits used by AV1 motion-vector components.
/// </summary>
public const int SubpixelBits = 3;
/// <summary>
/// The number of motion-vector units in one full pixel.
/// </summary>
public const int SubpixelScale = 1 << SubpixelBits;
/// <summary> /// <summary>
/// The reserved lower endpoint of the signed AV1 motion-vector domain. /// The reserved lower endpoint of the signed AV1 motion-vector domain.
/// </summary> /// </summary>

459
src/ImageSharp/Formats/Heif/Av1/Motion/Av1ReferenceMotionVectors.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Diagnostics.CodeAnalysis;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -10,7 +11,7 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
/// <summary> /// <summary>
/// Derives the weighted AV1 reference-motion-vector candidates for one inter block. /// Derives the weighted AV1 reference-motion-vector candidates for one inter block.
/// </summary> /// </summary>
internal sealed class Av1ReferenceMotionVectors internal struct Av1ReferenceMotionVectors
{ {
/// <summary> /// <summary>
/// The number of surrounding mode-information rows and columns examined by the spatial search. /// The number of surrounding mode-information rows and columns examined by the spatial search.
@ -80,16 +81,19 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Gets the derived candidates in normative nearest-region then outer-region order. /// Gets the derived candidates in normative nearest-region then outer-region order.
/// </summary> /// </summary>
[UnscopedRef]
public ReadOnlySpan<Av1MotionVector> Candidates => this.candidates[..this.Count]; public ReadOnlySpan<Av1MotionVector> Candidates => this.candidates[..this.Count];
/// <summary> /// <summary>
/// Gets the secondary vectors corresponding to <see cref="Candidates"/> for a compound block. /// Gets the secondary vectors corresponding to <see cref="Candidates"/> for a compound block.
/// </summary> /// </summary>
[UnscopedRef]
public ReadOnlySpan<Av1MotionVector> CompoundCandidates => this.compoundCandidates[..this.Count]; public ReadOnlySpan<Av1MotionVector> CompoundCandidates => this.compoundCandidates[..this.Count];
/// <summary> /// <summary>
/// Gets the accumulated weight corresponding to each entry in <see cref="Candidates"/>. /// Gets the accumulated weight corresponding to each entry in <see cref="Candidates"/>.
/// </summary> /// </summary>
[UnscopedRef]
public ReadOnlySpan<ushort> Weights => this.weights[..this.Count]; public ReadOnlySpan<ushort> Weights => this.weights[..this.Count];
/// <summary> /// <summary>
@ -116,11 +120,67 @@ internal sealed class Av1ReferenceMotionVectors
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame = Av1ReferenceFrameType.None) Av1ReferenceFrameType secondaryReferenceFrame = Av1ReferenceFrameType.None)
{ {
Av1BlockSize blockSize = partitionInfo.ModeInfo.BlockSize; ReferenceContext context = new(ref partitionInfo, sequenceHeader.SuperblockModeInfoSize, frameInfo);
this.Build(
in context,
tileInfo,
sequenceHeader,
frameHeader,
referenceFrame,
secondaryReferenceFrame);
}
/// <summary>
/// Derives the spatial single-reference motion-vector candidates for an encoder block.
/// </summary>
/// <param name="picture">The encoder picture state containing previously coded neighbors.</param>
/// <param name="macroBlock">The current block geometry and tile availability.</param>
/// <param name="modeInfoPosition">The current block origin in 4x4 mode-information units.</param>
/// <param name="blockSize">The current coding-block size.</param>
/// <param name="partitionType">The partition that produced the current block.</param>
/// <param name="sequenceHeader">The sequence-level superblock configuration.</param>
/// <param name="frameHeader">The frame-level global-motion and motion-vector precision configuration.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
public void Build(
Av1PictureControlSet picture,
Av1MacroBlockD macroBlock,
Point modeInfoPosition,
Av1BlockSize blockSize,
Av1PartitionType partitionType,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader,
Av1ReferenceFrameType referenceFrame)
{
ReferenceContext context = new(
picture,
macroBlock,
modeInfoPosition,
blockSize,
partitionType,
sequenceHeader.SuperblockModeInfoSize);
this.Build(
in context,
macroBlock.Tile,
sequenceHeader,
frameHeader,
referenceFrame,
Av1ReferenceFrameType.None);
}
private void Build(
in ReferenceContext context,
Av1TileInfo tileInfo,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader,
Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame)
{
Av1BlockSize blockSize = context.BlockSize;
int width = blockSize.Get4x4WideCount(); int width = blockSize.Get4x4WideCount();
int height = blockSize.Get4x4HighCount(); int height = blockSize.Get4x4HighCount();
int row = partitionInfo.RowIndex; int row = context.RowIndex;
int column = partitionInfo.ColumnIndex; int column = context.ColumnIndex;
int rowAdjustment = height < 2 && (row & 1) != 0 ? 1 : 0; int rowAdjustment = height < 2 && (row & 1) != 0 ? 1 : 0;
int columnAdjustment = width < 2 && (column & 1) != 0 ? 1 : 0; int columnAdjustment = width < 2 && (column & 1) != 0 ? 1 : 0;
int maximumRowOffset = 0; int maximumRowOffset = 0;
@ -129,13 +189,13 @@ internal sealed class Av1ReferenceMotionVectors
this.Count = 0; this.Count = 0;
this.ModeContext = 0; this.ModeContext = 0;
if (partitionInfo.AvailableAbove) if (context.AvailableAbove)
{ {
maximumRowOffset = height < 2 ? -4 + rowAdjustment : -(ReferenceSearchDistance << 1) + rowAdjustment; maximumRowOffset = height < 2 ? -4 + rowAdjustment : -(ReferenceSearchDistance << 1) + rowAdjustment;
maximumRowOffset = Math.Clamp(maximumRowOffset, tileInfo.ModeInfoRowStart - row, tileInfo.ModeInfoRowEnd - row - 1); maximumRowOffset = Math.Clamp(maximumRowOffset, tileInfo.ModeInfoRowStart - row, tileInfo.ModeInfoRowEnd - row - 1);
} }
if (partitionInfo.AvailableLeft) if (context.AvailableLeft)
{ {
maximumColumnOffset = width < 2 ? -4 + columnAdjustment : -(ReferenceSearchDistance << 1) + columnAdjustment; maximumColumnOffset = width < 2 ? -4 + columnAdjustment : -(ReferenceSearchDistance << 1) + columnAdjustment;
maximumColumnOffset = Math.Clamp(maximumColumnOffset, tileInfo.ModeInfoColumnStart - column, tileInfo.ModeInfoColumnEnd - column - 1); maximumColumnOffset = Math.Clamp(maximumColumnOffset, tileInfo.ModeInfoColumnStart - column, tileInfo.ModeInfoColumnEnd - column - 1);
@ -171,7 +231,7 @@ internal sealed class Av1ReferenceMotionVectors
if (Math.Abs(maximumRowOffset) >= 1) if (Math.Abs(maximumRowOffset) >= 1)
{ {
this.ScanRow( this.ScanRow(
ref partitionInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
in globalMotion, in globalMotion,
@ -188,7 +248,7 @@ internal sealed class Av1ReferenceMotionVectors
if (Math.Abs(maximumColumnOffset) >= 1) if (Math.Abs(maximumColumnOffset) >= 1)
{ {
this.ScanColumn( this.ScanColumn(
ref partitionInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
in globalMotion, in globalMotion,
@ -202,10 +262,10 @@ internal sealed class Av1ReferenceMotionVectors
ref processedColumns); ref processedColumns);
} }
if (partitionInfo.HasTopRight(sequenceHeader.SuperblockModeInfoSize)) if (context.HasTopRight)
{ {
this.AddSpatialBlock( this.AddSpatialBlock(
ref partitionInfo, in context,
tileInfo, tileInfo,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
@ -229,9 +289,8 @@ internal sealed class Av1ReferenceMotionVectors
if (frameHeader.UseReferenceFrameMotionVectors) if (frameHeader.UseReferenceFrameMotionVectors)
{ {
this.AddTemporalCandidates( this.AddTemporalCandidates(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
sequenceHeader.OrderHintInfo, sequenceHeader.OrderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -245,7 +304,7 @@ internal sealed class Av1ReferenceMotionVectors
// The top-left block begins the lower-priority outer region. Candidate deduplication still spans both // The top-left block begins the lower-priority outer region. Candidate deduplication still spans both
// regions, while the two independent stable sorts below preserve the normative nearest-before-outer order. // regions, while the two independent stable sorts below preserve the normative nearest-before-outer order.
this.AddSpatialBlock( this.AddSpatialBlock(
ref partitionInfo, in context,
tileInfo, tileInfo,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
@ -265,7 +324,7 @@ internal sealed class Av1ReferenceMotionVectors
if (Math.Abs(rowOffset) <= Math.Abs(maximumRowOffset) && Math.Abs(rowOffset) > processedRows) if (Math.Abs(rowOffset) <= Math.Abs(maximumRowOffset) && Math.Abs(rowOffset) > processedRows)
{ {
this.ScanRow( this.ScanRow(
ref partitionInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
in globalMotion, in globalMotion,
@ -282,7 +341,7 @@ internal sealed class Av1ReferenceMotionVectors
if (Math.Abs(columnOffset) <= Math.Abs(maximumColumnOffset) && Math.Abs(columnOffset) > processedColumns) if (Math.Abs(columnOffset) <= Math.Abs(maximumColumnOffset) && Math.Abs(columnOffset) > processedColumns)
{ {
this.ScanColumn( this.ScanColumn(
ref partitionInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
in globalMotion, in globalMotion,
@ -323,8 +382,7 @@ internal sealed class Av1ReferenceMotionVectors
if (this.Count < 2) if (this.Count < 2)
{ {
this.ExtendCompoundStack( this.ExtendCompoundStack(
ref partitionInfo, in context,
frameInfo,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
globalMotionVector, globalMotionVector,
@ -340,15 +398,15 @@ internal sealed class Av1ReferenceMotionVectors
// Differing reference sign biases are reversed before either candidate enters that pair. // Differing reference sign biases are reversed before either candidate enters that pair.
for (int index = 0; Math.Abs(maximumRowOffset) >= 1 && index < extensionLength && this.Count < 2;) for (int index = 0; Math.Abs(maximumRowOffset) >= 1 && index < extensionLength && this.Count < 2;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column + index, row - 1)); ReferenceBlock candidate = context.GetModeInfoAt(new Point(column + index, row - 1));
this.AddExtensionCandidate(candidate, frameInfo, referenceFrame); this.AddExtensionCandidate(candidate, in context, referenceFrame);
index += candidate.BlockSize.Get4x4WideCount(); index += candidate.BlockSize.Get4x4WideCount();
} }
for (int index = 0; Math.Abs(maximumColumnOffset) >= 1 && index < extensionLength && this.Count < 2;) for (int index = 0; Math.Abs(maximumColumnOffset) >= 1 && index < extensionLength && this.Count < 2;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column - 1, row + index)); ReferenceBlock candidate = context.GetModeInfoAt(new Point(column - 1, row + index));
this.AddExtensionCandidate(candidate, frameInfo, referenceFrame); this.AddExtensionCandidate(candidate, in context, referenceFrame);
index += candidate.BlockSize.Get4x4HighCount(); index += candidate.BlockSize.Get4x4HighCount();
} }
} }
@ -358,20 +416,20 @@ internal sealed class Av1ReferenceMotionVectors
this.candidates[index] = this.candidates[index].ClampReference( this.candidates[index] = this.candidates[index].ClampReference(
blockSize.GetWidth(), blockSize.GetWidth(),
blockSize.GetHeight(), blockSize.GetHeight(),
partitionInfo.ModeBlockToLeftEdge, context.ModeBlockToLeftEdge,
partitionInfo.ModeBlockToRightEdge, context.ModeBlockToRightEdge,
partitionInfo.ModeBlockToTopEdge, context.ModeBlockToTopEdge,
partitionInfo.ModeBlockToBottomEdge); context.ModeBlockToBottomEdge);
if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra) if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra)
{ {
this.compoundCandidates[index] = this.compoundCandidates[index].ClampReference( this.compoundCandidates[index] = this.compoundCandidates[index].ClampReference(
blockSize.GetWidth(), blockSize.GetWidth(),
blockSize.GetHeight(), blockSize.GetHeight(),
partitionInfo.ModeBlockToLeftEdge, context.ModeBlockToLeftEdge,
partitionInfo.ModeBlockToRightEdge, context.ModeBlockToRightEdge,
partitionInfo.ModeBlockToTopEdge, context.ModeBlockToTopEdge,
partitionInfo.ModeBlockToBottomEdge); context.ModeBlockToBottomEdge);
} }
} }
@ -452,7 +510,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Scans one spatial row using AV1's block-size-dependent steps and weights. /// Scans one spatial row using AV1's block-size-dependent steps and weights.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry and frame-wide mode map.</param> /// <param name="context">The current block geometry and frame-wide mode map.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
/// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param> /// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param>
/// <param name="globalMotion">The selected reference's global-motion model.</param> /// <param name="globalMotion">The selected reference's global-motion model.</param>
@ -465,7 +523,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param> /// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param>
/// <param name="processedRows">Receives the spatial depth covered by block-height weighting.</param> /// <param name="processedRows">Receives the spatial depth covered by block-height weighting.</param>
private void ScanRow( private void ScanRow(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
in Av1GlobalMotionParameters globalMotion, in Av1GlobalMotionParameters globalMotion,
@ -478,13 +536,13 @@ internal sealed class Av1ReferenceMotionVectors
ref int newMotionVectorCount, ref int newMotionVectorCount,
ref int processedRows) ref int processedRows)
{ {
int width = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); int width = context.BlockSize.Get4x4WideCount();
int end = Math.Min(partitionInfo.GetMaxBlockWide(partitionInfo.ModeInfo.BlockSize, false), MaximumSearchBlockSize); int end = Math.Min(context.GetMaxBlockWide(), MaximumSearchBlockSize);
int columnOffset = 0; int columnOffset = 0;
if (Math.Abs(rowOffset) > 1) if (Math.Abs(rowOffset) > 1)
{ {
columnOffset = 1; columnOffset = 1;
if ((partitionInfo.ColumnIndex & 1) != 0 && width < 2) if ((context.ColumnIndex & 1) != 0 && width < 2)
{ {
columnOffset--; columnOffset--;
} }
@ -495,8 +553,8 @@ internal sealed class Av1ReferenceMotionVectors
bool useFourUnitStep = width >= MaximumSearchBlockSize; bool useFourUnitStep = width >= MaximumSearchBlockSize;
for (int index = 0; index < end;) for (int index = 0; index < end;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt( ReferenceBlock candidate = context.GetModeInfoAt(
new Point(partitionInfo.ColumnIndex + columnOffset + index, partitionInfo.RowIndex + rowOffset)); new Point(context.ColumnIndex + columnOffset + index, context.RowIndex + rowOffset));
int candidateWidth = candidate.BlockSize.Get4x4WideCount(); int candidateWidth = candidate.BlockSize.Get4x4WideCount();
int length = Math.Min(width, candidateWidth); int length = Math.Min(width, candidateWidth);
@ -536,7 +594,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Scans one spatial column using AV1's block-size-dependent steps and weights. /// Scans one spatial column using AV1's block-size-dependent steps and weights.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry and frame-wide mode map.</param> /// <param name="context">The current block geometry and frame-wide mode map.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
/// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param> /// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param>
/// <param name="globalMotion">The selected reference's global-motion model.</param> /// <param name="globalMotion">The selected reference's global-motion model.</param>
@ -549,7 +607,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param> /// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param>
/// <param name="processedColumns">Receives the spatial depth covered by block-width weighting.</param> /// <param name="processedColumns">Receives the spatial depth covered by block-width weighting.</param>
private void ScanColumn( private void ScanColumn(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
in Av1GlobalMotionParameters globalMotion, in Av1GlobalMotionParameters globalMotion,
@ -562,13 +620,13 @@ internal sealed class Av1ReferenceMotionVectors
ref int newMotionVectorCount, ref int newMotionVectorCount,
ref int processedColumns) ref int processedColumns)
{ {
int height = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); int height = context.BlockSize.Get4x4HighCount();
int end = Math.Min(partitionInfo.GetMaxBlockHigh(partitionInfo.ModeInfo.BlockSize, false), MaximumSearchBlockSize); int end = Math.Min(context.GetMaxBlockHigh(), MaximumSearchBlockSize);
int rowOffset = 0; int rowOffset = 0;
if (Math.Abs(columnOffset) > 1) if (Math.Abs(columnOffset) > 1)
{ {
rowOffset = 1; rowOffset = 1;
if ((partitionInfo.RowIndex & 1) != 0 && height < 2) if ((context.RowIndex & 1) != 0 && height < 2)
{ {
rowOffset--; rowOffset--;
} }
@ -579,8 +637,8 @@ internal sealed class Av1ReferenceMotionVectors
bool useFourUnitStep = height >= MaximumSearchBlockSize; bool useFourUnitStep = height >= MaximumSearchBlockSize;
for (int index = 0; index < end;) for (int index = 0; index < end;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt( ReferenceBlock candidate = context.GetModeInfoAt(
new Point(partitionInfo.ColumnIndex + columnOffset, partitionInfo.RowIndex + rowOffset + index)); new Point(context.ColumnIndex + columnOffset, context.RowIndex + rowOffset + index));
int candidateHeight = candidate.BlockSize.Get4x4HighCount(); int candidateHeight = candidate.BlockSize.Get4x4HighCount();
int length = Math.Min(height, candidateHeight); int length = Math.Min(height, candidateHeight);
@ -620,7 +678,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Adds the candidate at one tile-relative spatial search position. /// Adds the candidate at one tile-relative spatial search position.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry and frame-wide mode map.</param> /// <param name="context">The current block geometry and frame-wide mode map.</param>
/// <param name="tileInfo">The active tile boundaries.</param> /// <param name="tileInfo">The active tile boundaries.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
/// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param> /// <param name="secondaryReferenceFrame">The secondary compound reference, or <see cref="Av1ReferenceFrameType.None"/>.</param>
@ -633,7 +691,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="referenceMatchCount">Accumulates matching reference labels at the search position.</param> /// <param name="referenceMatchCount">Accumulates matching reference labels at the search position.</param>
/// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param> /// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param>
private void AddSpatialBlock( private void AddSpatialBlock(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1TileInfo tileInfo, Av1TileInfo tileInfo,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
@ -646,15 +704,15 @@ internal sealed class Av1ReferenceMotionVectors
ref int referenceMatchCount, ref int referenceMatchCount,
ref int newMotionVectorCount) ref int newMotionVectorCount)
{ {
int row = partitionInfo.RowIndex + rowOffset; int row = context.RowIndex + rowOffset;
int column = partitionInfo.ColumnIndex + columnOffset; int column = context.ColumnIndex + columnOffset;
if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd || if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd ||
column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd) column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd)
{ {
return; return;
} }
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column, row)); ReferenceBlock candidate = context.GetModeInfoAt(new Point(column, row));
this.AddCandidate( this.AddCandidate(
candidate, candidate,
referenceFrame, referenceFrame,
@ -682,7 +740,7 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="referenceMatchCount">Accumulates matching reference labels in the active scan direction.</param> /// <param name="referenceMatchCount">Accumulates matching reference labels in the active scan direction.</param>
/// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param> /// <param name="newMotionVectorCount">Accumulates matching neighbors whose inter mode contains a new vector.</param>
private void AddCandidate( private void AddCandidate(
Av1BlockModeInfo candidate, ReferenceBlock candidate,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
in Av1GlobalMotionParameters globalMotion, in Av1GlobalMotionParameters globalMotion,
@ -693,16 +751,15 @@ internal sealed class Av1ReferenceMotionVectors
ref int referenceMatchCount, ref int referenceMatchCount,
ref int newMotionVectorCount) ref int newMotionVectorCount)
{ {
Span<Av1ReferenceFrameType> candidateReferences = candidate.ReferenceFrames; if (candidate.GetReferenceFrame(0) <= Av1ReferenceFrameType.Intra)
if (candidateReferences[0] <= Av1ReferenceFrameType.Intra)
{ {
return; return;
} }
Span<Av1MotionVector> candidateMotionVectors = candidate.MotionVectors;
if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra) if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra)
{ {
if (candidateReferences[0] != referenceFrame || candidateReferences[1] != secondaryReferenceFrame) if (candidate.GetReferenceFrame(0) != referenceFrame ||
candidate.GetReferenceFrame(1) != secondaryReferenceFrame)
{ {
return; return;
} }
@ -719,11 +776,11 @@ internal sealed class Av1ReferenceMotionVectors
Av1MotionVector primaryMotionVector = usePrimaryGlobalMotion Av1MotionVector primaryMotionVector = usePrimaryGlobalMotion
? globalMotionVector ? globalMotionVector
: candidateMotionVectors[0]; : candidate.GetMotionVector(0);
Av1MotionVector secondaryMotionVector = useSecondaryGlobalMotion Av1MotionVector secondaryMotionVector = useSecondaryGlobalMotion
? secondaryGlobalMotionVector ? secondaryGlobalMotionVector
: candidateMotionVectors[1]; : candidate.GetMotionVector(1);
this.AddUnique(primaryMotionVector, secondaryMotionVector, weight); this.AddUnique(primaryMotionVector, secondaryMotionVector, weight);
if (UsesNewMotionVector(candidate.YMode)) if (UsesNewMotionVector(candidate.YMode))
@ -737,7 +794,7 @@ internal sealed class Av1ReferenceMotionVectors
for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++) for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++)
{ {
if (candidateReferences[referenceIndex] != referenceFrame) if (candidate.GetReferenceFrame(referenceIndex) != referenceFrame)
{ {
continue; continue;
} }
@ -752,7 +809,7 @@ internal sealed class Av1ReferenceMotionVectors
Av1MotionVector motionVector = useGlobalMotion Av1MotionVector motionVector = useGlobalMotion
? globalMotionVector ? globalMotionVector
: candidateMotionVectors[referenceIndex]; : candidate.GetMotionVector(referenceIndex);
this.AddUnique(motionVector, weight); this.AddUnique(motionVector, weight);
@ -770,9 +827,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Adds projected temporal candidates over the current block and its permitted extension positions. /// Adds projected temporal candidates over the current block and its permitted extension positions.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry.</param> /// <param name="context">The current block geometry and decoder temporal state.</param>
/// <param name="tileInfo">The active tile boundaries.</param> /// <param name="tileInfo">The active tile boundaries.</param>
/// <param name="frameInfo">The projected temporal motion field.</param>
/// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param> /// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param>
/// <param name="frameHeader">The frame-level motion-vector precision configuration.</param> /// <param name="frameHeader">The frame-level motion-vector precision configuration.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
@ -780,9 +836,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="globalMotionVector">The selected reference's global-motion vector at the current block.</param> /// <param name="globalMotionVector">The selected reference's global-motion vector at the current block.</param>
/// <param name="secondaryGlobalMotionVector">The secondary reference's global-motion vector at the current block.</param> /// <param name="secondaryGlobalMotionVector">The secondary reference's global-motion vector at the current block.</param>
private void AddTemporalCandidates( private void AddTemporalCandidates(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1TileInfo tileInfo, Av1TileInfo tileInfo,
Av1FrameInfo frameInfo,
ObuOrderHintInfo orderHintInfo, ObuOrderHintInfo orderHintInfo,
ObuFrameHeader frameHeader, ObuFrameHeader frameHeader,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
@ -790,8 +845,8 @@ internal sealed class Av1ReferenceMotionVectors
Av1MotionVector globalMotionVector, Av1MotionVector globalMotionVector,
Av1MotionVector secondaryGlobalMotionVector) Av1MotionVector secondaryGlobalMotionVector)
{ {
int width = partitionInfo.ModeInfo.BlockSize.Get4x4WideCount(); int width = context.BlockSize.Get4x4WideCount();
int height = partitionInfo.ModeInfo.BlockSize.Get4x4HighCount(); int height = context.BlockSize.Get4x4HighCount();
int verticalOffset = Math.Max(2, height); int verticalOffset = Math.Max(2, height);
int horizontalOffset = Math.Max(2, width); int horizontalOffset = Math.Max(2, width);
int blockRowEnd = Math.Min(height, MaximumSearchBlockSize); int blockRowEnd = Math.Min(height, MaximumSearchBlockSize);
@ -805,9 +860,8 @@ internal sealed class Av1ReferenceMotionVectors
for (int blockColumn = 0; blockColumn < blockColumnEnd; blockColumn += columnStep) for (int blockColumn = 0; blockColumn < blockColumnEnd; blockColumn += columnStep)
{ {
bool available = this.AddTemporalCandidate( bool available = this.AddTemporalCandidate(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
orderHintInfo, orderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -838,9 +892,8 @@ internal sealed class Av1ReferenceMotionVectors
// These three positions extend the temporal search below-left, below-right, and above-right. The 64x64 // These three positions extend the temporal search below-left, below-right, and above-right. The 64x64
// boundary test is normative even when the sequence uses 128x128 superblocks. // boundary test is normative even when the sequence uses 128x128 superblocks.
this.AddTemporalExtension( this.AddTemporalExtension(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
orderHintInfo, orderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -851,9 +904,8 @@ internal sealed class Av1ReferenceMotionVectors
-2); -2);
this.AddTemporalExtension( this.AddTemporalExtension(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
orderHintInfo, orderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -864,9 +916,8 @@ internal sealed class Av1ReferenceMotionVectors
horizontalOffset); horizontalOffset);
this.AddTemporalExtension( this.AddTemporalExtension(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
orderHintInfo, orderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -880,9 +931,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Adds one optional temporal extension candidate after applying the normative 64x64 boundary rule. /// Adds one optional temporal extension candidate after applying the normative 64x64 boundary rule.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry.</param> /// <param name="context">The current block geometry and decoder temporal state.</param>
/// <param name="tileInfo">The active tile boundaries.</param> /// <param name="tileInfo">The active tile boundaries.</param>
/// <param name="frameInfo">The projected temporal motion field.</param>
/// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param> /// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param>
/// <param name="frameHeader">The frame-level motion-vector precision configuration.</param> /// <param name="frameHeader">The frame-level motion-vector precision configuration.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
@ -892,9 +942,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="blockRow">The temporal sample row relative to the current block in 4x4 units.</param> /// <param name="blockRow">The temporal sample row relative to the current block in 4x4 units.</param>
/// <param name="blockColumn">The temporal sample column relative to the current block in 4x4 units.</param> /// <param name="blockColumn">The temporal sample column relative to the current block in 4x4 units.</param>
private void AddTemporalExtension( private void AddTemporalExtension(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1TileInfo tileInfo, Av1TileInfo tileInfo,
Av1FrameInfo frameInfo,
ObuOrderHintInfo orderHintInfo, ObuOrderHintInfo orderHintInfo,
ObuFrameHeader frameHeader, ObuFrameHeader frameHeader,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
@ -904,8 +953,8 @@ internal sealed class Av1ReferenceMotionVectors
int blockRow, int blockRow,
int blockColumn) int blockColumn)
{ {
int rowWithinBlock64 = partitionInfo.RowIndex & (MaximumSearchBlockSize - 1); int rowWithinBlock64 = context.RowIndex & (MaximumSearchBlockSize - 1);
int columnWithinBlock64 = partitionInfo.ColumnIndex & (MaximumSearchBlockSize - 1); int columnWithinBlock64 = context.ColumnIndex & (MaximumSearchBlockSize - 1);
if (rowWithinBlock64 + blockRow < 0 || rowWithinBlock64 + blockRow >= MaximumSearchBlockSize || if (rowWithinBlock64 + blockRow < 0 || rowWithinBlock64 + blockRow >= MaximumSearchBlockSize ||
columnWithinBlock64 + blockColumn < 0 || columnWithinBlock64 + blockColumn >= MaximumSearchBlockSize) columnWithinBlock64 + blockColumn < 0 || columnWithinBlock64 + blockColumn >= MaximumSearchBlockSize)
{ {
@ -913,9 +962,8 @@ internal sealed class Av1ReferenceMotionVectors
} }
_ = this.AddTemporalCandidate( _ = this.AddTemporalCandidate(
ref partitionInfo, in context,
tileInfo, tileInfo,
frameInfo,
orderHintInfo, orderHintInfo,
frameHeader, frameHeader,
referenceFrame, referenceFrame,
@ -929,9 +977,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <summary> /// <summary>
/// Projects and accumulates one temporal motion-field sample. /// Projects and accumulates one temporal motion-field sample.
/// </summary> /// </summary>
/// <param name="partitionInfo">The current block geometry.</param> /// <param name="context">The current block geometry and decoder temporal state.</param>
/// <param name="tileInfo">The active tile boundaries.</param> /// <param name="tileInfo">The active tile boundaries.</param>
/// <param name="frameInfo">The projected temporal motion field.</param>
/// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param> /// <param name="orderHintInfo">The sequence modulo order-hint configuration.</param>
/// <param name="frameHeader">The frame-level motion-vector precision configuration.</param> /// <param name="frameHeader">The frame-level motion-vector precision configuration.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
@ -942,9 +989,8 @@ internal sealed class Av1ReferenceMotionVectors
/// <param name="blockColumn">The temporal sample column relative to the current block in 4x4 units.</param> /// <param name="blockColumn">The temporal sample column relative to the current block in 4x4 units.</param>
/// <returns><see langword="true"/> when the projected motion field covers the requested position.</returns> /// <returns><see langword="true"/> when the projected motion field covers the requested position.</returns>
private bool AddTemporalCandidate( private bool AddTemporalCandidate(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1TileInfo tileInfo, Av1TileInfo tileInfo,
Av1FrameInfo frameInfo,
ObuOrderHintInfo orderHintInfo, ObuOrderHintInfo orderHintInfo,
ObuFrameHeader frameHeader, ObuFrameHeader frameHeader,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
@ -954,17 +1000,17 @@ internal sealed class Av1ReferenceMotionVectors
int blockRow, int blockRow,
int blockColumn) int blockColumn)
{ {
int rowOffset = (partitionInfo.RowIndex & 1) != 0 ? blockRow : blockRow + 1; int rowOffset = (context.RowIndex & 1) != 0 ? blockRow : blockRow + 1;
int columnOffset = (partitionInfo.ColumnIndex & 1) != 0 ? blockColumn : blockColumn + 1; int columnOffset = (context.ColumnIndex & 1) != 0 ? blockColumn : blockColumn + 1;
int row = partitionInfo.RowIndex + rowOffset; int row = context.RowIndex + rowOffset;
int column = partitionInfo.ColumnIndex + columnOffset; int column = context.ColumnIndex + columnOffset;
if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd || if (row < tileInfo.ModeInfoRowStart || row >= tileInfo.ModeInfoRowEnd ||
column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd) column < tileInfo.ModeInfoColumnStart || column >= tileInfo.ModeInfoColumnEnd)
{ {
return false; return false;
} }
if (!frameInfo.TryGetProjectedTemporalMotionVector( if (!context.TryGetProjectedTemporalMotionVector(
row, row,
column, column,
referenceFrame, referenceFrame,
@ -978,7 +1024,7 @@ internal sealed class Av1ReferenceMotionVectors
Av1MotionVector secondaryMotionVector = default; Av1MotionVector secondaryMotionVector = default;
if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra && if (secondaryReferenceFrame > Av1ReferenceFrameType.Intra &&
!frameInfo.TryGetProjectedTemporalMotionVector( !context.TryGetProjectedTemporalMotionVector(
row, row,
column, column,
secondaryReferenceFrame, secondaryReferenceFrame,
@ -1018,27 +1064,25 @@ internal sealed class Av1ReferenceMotionVectors
/// Extends a short stack with inter vectors from a neighboring block, correcting their temporal direction. /// Extends a short stack with inter vectors from a neighboring block, correcting their temporal direction.
/// </summary> /// </summary>
/// <param name="candidate">The decoded neighboring block.</param> /// <param name="candidate">The decoded neighboring block.</param>
/// <param name="frameInfo">The reference-side classification for the current frame.</param> /// <param name="context">The current block geometry and decoder reference classification.</param>
/// <param name="referenceFrame">The canonical inter reference selected for the current block.</param> /// <param name="referenceFrame">The canonical inter reference selected for the current block.</param>
private void AddExtensionCandidate( private void AddExtensionCandidate(
Av1BlockModeInfo candidate, ReferenceBlock candidate,
Av1FrameInfo frameInfo, in ReferenceContext context,
Av1ReferenceFrameType referenceFrame) Av1ReferenceFrameType referenceFrame)
{ {
Span<Av1ReferenceFrameType> candidateReferences = candidate.ReferenceFrames; bool targetSignBias = context.IsReferenceSignBiased(referenceFrame);
Span<Av1MotionVector> candidateMotionVectors = candidate.MotionVectors;
bool targetSignBias = frameInfo.IsReferenceSignBiased(referenceFrame);
for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++) for (int referenceIndex = 0; referenceIndex < 2; referenceIndex++)
{ {
Av1ReferenceFrameType candidateReference = candidateReferences[referenceIndex]; Av1ReferenceFrameType candidateReference = candidate.GetReferenceFrame(referenceIndex);
if (candidateReference <= Av1ReferenceFrameType.Intra) if (candidateReference <= Av1ReferenceFrameType.Intra)
{ {
continue; continue;
} }
Av1MotionVector motionVector = candidateMotionVectors[referenceIndex]; Av1MotionVector motionVector = candidate.GetMotionVector(referenceIndex);
if (frameInfo.IsReferenceSignBiased(candidateReference) != targetSignBias) if (context.IsReferenceSignBiased(candidateReference) != targetSignBias)
{ {
motionVector = new Av1MotionVector(-motionVector.Row, -motionVector.Column); motionVector = new Av1MotionVector(-motionVector.Row, -motionVector.Column);
} }
@ -1067,8 +1111,7 @@ internal sealed class Av1ReferenceMotionVectors
/// Extends a short compound stack from the immediate above and left blocks. /// Extends a short compound stack from the immediate above and left blocks.
/// </summary> /// </summary>
private void ExtendCompoundStack( private void ExtendCompoundStack(
ref Av1PartitionInfo partitionInfo, in ReferenceContext context,
Av1FrameInfo frameInfo,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
Av1MotionVector globalMotionVector, Av1MotionVector globalMotionVector,
@ -1085,15 +1128,15 @@ internal sealed class Av1ReferenceMotionVectors
int secondaryExactCount = 0; int secondaryExactCount = 0;
int primaryDifferentCount = 0; int primaryDifferentCount = 0;
int secondaryDifferentCount = 0; int secondaryDifferentCount = 0;
int row = partitionInfo.RowIndex; int row = context.RowIndex;
int column = partitionInfo.ColumnIndex; int column = context.ColumnIndex;
for (int index = 0; hasAbove && index < extensionLength;) for (int index = 0; hasAbove && index < extensionLength;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column + index, row - 1)); ReferenceBlock candidate = context.GetModeInfoAt(new Point(column + index, row - 1));
CollectCompoundExtensionCandidate( CollectCompoundExtensionCandidate(
candidate, candidate,
frameInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
ref primaryExact, ref primaryExact,
@ -1110,10 +1153,10 @@ internal sealed class Av1ReferenceMotionVectors
for (int index = 0; hasLeft && index < extensionLength;) for (int index = 0; hasLeft && index < extensionLength;)
{ {
Av1BlockModeInfo candidate = partitionInfo.SuperblockInfo.GetModeInfoAt(new Point(column - 1, row + index)); ReferenceBlock candidate = context.GetModeInfoAt(new Point(column - 1, row + index));
CollectCompoundExtensionCandidate( CollectCompoundExtensionCandidate(
candidate, candidate,
frameInfo, in context,
referenceFrame, referenceFrame,
secondaryReferenceFrame, secondaryReferenceFrame,
ref primaryExact, ref primaryExact,
@ -1168,8 +1211,8 @@ internal sealed class Av1ReferenceMotionVectors
/// Collects exact-reference and temporal-direction-corrected fallback vectors from one neighboring block. /// Collects exact-reference and temporal-direction-corrected fallback vectors from one neighboring block.
/// </summary> /// </summary>
private static void CollectCompoundExtensionCandidate( private static void CollectCompoundExtensionCandidate(
Av1BlockModeInfo candidate, ReferenceBlock candidate,
Av1FrameInfo frameInfo, in ReferenceContext context,
Av1ReferenceFrameType referenceFrame, Av1ReferenceFrameType referenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame, Av1ReferenceFrameType secondaryReferenceFrame,
ref InlineArray2<Av1MotionVector> primaryExact, ref InlineArray2<Av1MotionVector> primaryExact,
@ -1181,13 +1224,10 @@ internal sealed class Av1ReferenceMotionVectors
ref InlineArray2<Av1MotionVector> secondaryDifferent, ref InlineArray2<Av1MotionVector> secondaryDifferent,
ref int secondaryDifferentCount) ref int secondaryDifferentCount)
{ {
Span<Av1ReferenceFrameType> candidateReferences = candidate.ReferenceFrames;
Span<Av1MotionVector> candidateMotionVectors = candidate.MotionVectors;
for (int candidateIndex = 0; candidateIndex < 2; candidateIndex++) for (int candidateIndex = 0; candidateIndex < 2; candidateIndex++)
{ {
Av1ReferenceFrameType candidateReference = candidateReferences[candidateIndex]; Av1ReferenceFrameType candidateReference = candidate.GetReferenceFrame(candidateIndex);
Av1MotionVector candidateMotionVector = candidateMotionVectors[candidateIndex]; Av1MotionVector candidateMotionVector = candidate.GetMotionVector(candidateIndex);
for (int targetIndex = 0; targetIndex < 2; targetIndex++) for (int targetIndex = 0; targetIndex < 2; targetIndex++)
{ {
@ -1223,7 +1263,7 @@ internal sealed class Av1ReferenceMotionVectors
} }
Av1MotionVector differentMotionVector = candidateMotionVector; Av1MotionVector differentMotionVector = candidateMotionVector;
if (frameInfo.IsReferenceSignBiased(candidateReference) != frameInfo.IsReferenceSignBiased(targetReference)) if (context.IsReferenceSignBiased(candidateReference) != context.IsReferenceSignBiased(targetReference))
{ {
differentMotionVector = new Av1MotionVector(-differentMotionVector.Row, -differentMotionVector.Column); differentMotionVector = new Av1MotionVector(-differentMotionVector.Row, -differentMotionVector.Column);
} }
@ -1364,4 +1404,195 @@ internal sealed class Av1ReferenceMotionVectors
Av1PredictionMode.NewNearestMotionVector or Av1PredictionMode.NewNearestMotionVector or
Av1PredictionMode.NearNewMotionVector or Av1PredictionMode.NearNewMotionVector or
Av1PredictionMode.NewNearMotionVector; Av1PredictionMode.NewNearMotionVector;
/// <summary>
/// Provides one allocation-free view over decoder or encoder mode-information storage.
/// </summary>
private readonly struct ReferenceContext
{
private readonly Av1SuperblockInfo decodedSuperblock;
private readonly Av1PictureControlSet? encodedPicture;
private readonly Av1FrameInfo? decodedFrame;
public ReferenceContext(ref Av1PartitionInfo partitionInfo, int superblockModeInfoSize, Av1FrameInfo frameInfo)
{
this.decodedSuperblock = partitionInfo.SuperblockInfo;
this.encodedPicture = null;
this.decodedFrame = frameInfo;
this.BlockSize = partitionInfo.ModeInfo.BlockSize;
this.RowIndex = partitionInfo.RowIndex;
this.ColumnIndex = partitionInfo.ColumnIndex;
this.AvailableAbove = partitionInfo.AvailableAbove;
this.AvailableLeft = partitionInfo.AvailableLeft;
this.ModeBlockToLeftEdge = partitionInfo.ModeBlockToLeftEdge;
this.ModeBlockToRightEdge = partitionInfo.ModeBlockToRightEdge;
this.ModeBlockToTopEdge = partitionInfo.ModeBlockToTopEdge;
this.ModeBlockToBottomEdge = partitionInfo.ModeBlockToBottomEdge;
this.HasTopRight = partitionInfo.HasTopRight(superblockModeInfoSize);
}
public ReferenceContext(
Av1PictureControlSet picture,
Av1MacroBlockD macroBlock,
Point modeInfoPosition,
Av1BlockSize blockSize,
Av1PartitionType partitionType,
int superblockModeInfoSize)
{
this.decodedSuperblock = default;
this.encodedPicture = picture;
this.decodedFrame = null;
this.BlockSize = blockSize;
this.RowIndex = modeInfoPosition.Y;
this.ColumnIndex = modeInfoPosition.X;
this.AvailableAbove = macroBlock.IsUpAvailable;
this.AvailableLeft = macroBlock.IsLeftAvailable;
this.ModeBlockToLeftEdge = macroBlock.ToLeftEdge;
this.ModeBlockToRightEdge = macroBlock.ToRightEdge;
this.ModeBlockToTopEdge = macroBlock.ToTopEdge;
this.ModeBlockToBottomEdge = macroBlock.ToBottomEdge;
this.HasTopRight = Av1PartitionInfo.HasTopRight(
blockSize,
partitionType,
modeInfoPosition.Y,
modeInfoPosition.X,
superblockModeInfoSize);
}
public Av1BlockSize BlockSize { get; }
public int RowIndex { get; }
public int ColumnIndex { get; }
public bool AvailableAbove { get; }
public bool AvailableLeft { get; }
public int ModeBlockToLeftEdge { get; }
public int ModeBlockToRightEdge { get; }
public int ModeBlockToTopEdge { get; }
public int ModeBlockToBottomEdge { get; }
public bool HasTopRight { get; }
public int GetMaxBlockWide()
{
int width = this.BlockSize.GetWidth();
if (this.ModeBlockToRightEdge < 0)
{
width += this.ModeBlockToRightEdge >> 3;
}
return width >> Av1Constants.ModeInfoSizeLog2;
}
public int GetMaxBlockHigh()
{
int height = this.BlockSize.GetHeight();
if (this.ModeBlockToBottomEdge < 0)
{
height += this.ModeBlockToBottomEdge >> 3;
}
return height >> Av1Constants.ModeInfoSizeLog2;
}
public ReferenceBlock GetModeInfoAt(Point position)
{
Av1PictureControlSet? picture = this.encodedPicture;
if (picture is not null)
{
Av1MacroBlockModeInfo encodedModeInfo = picture.GetFromModeInfoGrid(position);
return new ReferenceBlock(
encodedModeInfo.Block.BlockSize,
encodedModeInfo.Block.Mode,
encodedModeInfo.Block.ReferenceFrame,
Av1ReferenceFrameType.None,
picture.GetDisplacementVector(position),
default);
}
Av1BlockModeInfo decodedModeInfo = this.decodedSuperblock.GetModeInfoAt(position);
return new ReferenceBlock(
decodedModeInfo.BlockSize,
decodedModeInfo.YMode,
decodedModeInfo.ReferenceFrames[0],
decodedModeInfo.ReferenceFrames[1],
decodedModeInfo.MotionVectors[0],
decodedModeInfo.MotionVectors[1]);
}
public bool IsReferenceSignBiased(Av1ReferenceFrameType referenceFrame)
{
Av1FrameInfo? frameInfo = this.decodedFrame;
return frameInfo is not null && frameInfo.IsReferenceSignBiased(referenceFrame);
}
public bool TryGetProjectedTemporalMotionVector(
int row,
int column,
Av1ReferenceFrameType referenceFrame,
ObuOrderHintInfo orderHintInfo,
bool allowHighPrecisionMotionVector,
bool forceIntegerMotionVector,
out Av1MotionVector motionVector)
{
Av1FrameInfo? frameInfo = this.decodedFrame;
if (frameInfo is null)
{
motionVector = default;
return false;
}
return frameInfo.TryGetProjectedTemporalMotionVector(
row,
column,
referenceFrame,
orderHintInfo,
allowHighPrecisionMotionVector,
forceIntegerMotionVector,
out motionVector);
}
}
/// <summary>
/// Carries the neighboring mode fields consumed by reference-vector ranking.
/// </summary>
private readonly struct ReferenceBlock
{
private readonly Av1ReferenceFrameType primaryReferenceFrame;
private readonly Av1ReferenceFrameType secondaryReferenceFrame;
private readonly Av1MotionVector primaryMotionVector;
private readonly Av1MotionVector secondaryMotionVector;
public ReferenceBlock(
Av1BlockSize blockSize,
Av1PredictionMode mode,
Av1ReferenceFrameType primaryReferenceFrame,
Av1ReferenceFrameType secondaryReferenceFrame,
Av1MotionVector primaryMotionVector,
Av1MotionVector secondaryMotionVector)
{
this.BlockSize = blockSize;
this.YMode = mode;
this.primaryReferenceFrame = primaryReferenceFrame;
this.secondaryReferenceFrame = secondaryReferenceFrame;
this.primaryMotionVector = primaryMotionVector;
this.secondaryMotionVector = secondaryMotionVector;
}
public Av1BlockSize BlockSize { get; }
public Av1PredictionMode YMode { get; }
public Av1ReferenceFrameType GetReferenceFrame(int index)
=> index == 0 ? this.primaryReferenceFrame : this.secondaryReferenceFrame;
public Av1MotionVector GetMotionVector(int index)
=> index == 0 ? this.primaryMotionVector : this.secondaryMotionVector;
}
} }

9
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuFrameHeader.cs

@ -72,6 +72,15 @@ internal sealed class ObuFrameHeader
/// </summary> /// </summary>
public bool AllowHighPrecisionMotionVector { get; set; } public bool AllowHighPrecisionMotionVector { get; set; }
/// <summary>
/// Gets the component precision selected by the integer and high-precision frame flags.
/// </summary>
public Av1MotionVectorPrecision MotionVectorPrecision => this.ForceIntegerMotionVector
? Av1MotionVectorPrecision.Integer
: this.AllowHighPrecisionMotionVector
? Av1MotionVectorPrecision.EighthSample
: Av1MotionVectorPrecision.QuarterSample;
/// <summary> /// <summary>
/// Gets or sets the frame-level interpolation filter used for inter prediction. /// Gets or sets the frame-level interpolation filter used for inter prediction.
/// </summary> /// </summary>

15
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuOperatingPoint.cs

@ -28,6 +28,21 @@ internal sealed class ObuOperatingPoint
/// </summary> /// </summary>
public bool IsDecoderModelInfoPresent { get; set; } public bool IsDecoderModelInfoPresent { get; set; }
/// <summary>
/// Gets or sets the decoder-buffer delay measured in decoding ticks.
/// </summary>
public uint DecoderBufferDelay { get; set; }
/// <summary>
/// Gets or sets the encoder-buffer delay measured in decoding ticks.
/// </summary>
public uint EncoderBufferDelay { get; set; }
/// <summary>
/// Gets or sets a value indicating whether the operating point uses the low-delay decoding model.
/// </summary>
public bool LowDelayMode { get; set; }
/// <summary> /// <summary>
/// Gets or sets a value indicating whether an initial display delay is present for this operating point. /// Gets or sets a value indicating whether an initial display delay is present for this operating point.
/// </summary> /// </summary>

153
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuReader.cs

@ -14,57 +14,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
/// </summary> /// </summary>
internal sealed class ObuReader internal sealed class ObuReader
{ {
/// <summary>
/// The number of bits used to address one of AV1's eight reference-map slots.
/// </summary>
private const int ReferenceFrameIndexBits = 3;
/// <summary>
/// The initial finite-subexponential group width used by every global-motion parameter.
/// </summary>
private const int GlobalMotionSubexponentialGroupBitCount = 3;
/// <summary>
/// The finite signed-domain size parameter for coded global-motion affine coefficients.
/// </summary>
private const int GlobalMotionAlphaValueMagnitude = (1 << 12) + 1;
/// <summary>
/// The number of fractional bits carried by coded global-motion affine coefficients.
/// </summary>
private const int GlobalMotionAlphaPrecisionBits = 15;
/// <summary>
/// The precision increase from a coded affine coefficient to the stored global-motion matrix.
/// </summary>
private const int GlobalMotionAlphaPrecisionDifference =
Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionAlphaPrecisionBits;
/// <summary>
/// The scale factor that restores a coded affine coefficient to the global-motion matrix precision.
/// </summary>
private const int GlobalMotionAlphaDecodeFactor = 1 << GlobalMotionAlphaPrecisionDifference;
/// <summary>
/// The signed magnitude bit count of a general affine model's translation components.
/// </summary>
private const int GlobalMotionAbsoluteTranslationBits = 12;
/// <summary>
/// The signed magnitude bit count of a translation-only model before precision adjustment.
/// </summary>
private const int GlobalMotionAbsoluteTranslationOnlyBits = 9;
/// <summary>
/// The number of fractional bits carried by general affine translation components.
/// </summary>
private const int GlobalMotionTranslationPrecisionBits = 6;
/// <summary>
/// The number of fractional bits carried by translation-only components.
/// </summary>
private const int GlobalMotionTranslationOnlyPrecisionBits = 3;
/// <summary> /// <summary>
/// The zero-based sequence-header operating-point index selected by the container. /// The zero-based sequence-header operating-point index selected by the container.
/// </summary> /// </summary>
@ -951,7 +900,7 @@ internal sealed class ObuReader
} }
sequenceHeader.InitialDisplayDelayPresentFlag = reader.ReadBoolean(); sequenceHeader.InitialDisplayDelayPresentFlag = reader.ReadBoolean();
int operatingPointsCnt = (int)reader.ReadLiteral(5) + 1; int operatingPointsCnt = (int)reader.ReadLiteral(Av1Constants.OperatingPointCountBits) + 1;
if (sequenceHeader.OperatingPoint.Length != operatingPointsCnt) if (sequenceHeader.OperatingPoint.Length != operatingPointsCnt)
{ {
sequenceHeader.OperatingPoint = new ObuOperatingPoint[operatingPointsCnt]; sequenceHeader.OperatingPoint = new ObuOperatingPoint[operatingPointsCnt];
@ -961,15 +910,15 @@ internal sealed class ObuReader
{ {
sequenceHeader.OperatingPoint[i] = new ObuOperatingPoint sequenceHeader.OperatingPoint[i] = new ObuOperatingPoint
{ {
Idc = reader.ReadLiteral(12), Idc = reader.ReadLiteral(Av1Constants.OperatingPointIdcBits),
SequenceLevelIndex = (int)reader.ReadLiteral(5) SequenceLevelIndex = (int)reader.ReadLiteral(Av1Constants.LevelBits)
}; };
if (!IsValidSequenceLevel(sequenceHeader.OperatingPoint[i].SequenceLevelIndex)) if (!IsValidSequenceLevel(sequenceHeader.OperatingPoint[i].SequenceLevelIndex))
{ {
throw new InvalidImageContentException("The AV1 sequence header contains an undefined sequence-level index."); throw new InvalidImageContentException("The AV1 sequence header contains an undefined sequence-level index.");
} }
if (sequenceHeader.OperatingPoint[i].SequenceLevelIndex > 7) if (sequenceHeader.OperatingPoint[i].SequenceLevelIndex >= Av1Constants.SequenceTierMinimumLevelIndex)
{ {
sequenceHeader.OperatingPoint[i].SequenceTier = (int)reader.ReadLiteral(1); sequenceHeader.OperatingPoint[i].SequenceTier = (int)reader.ReadLiteral(1);
} }
@ -983,10 +932,13 @@ internal sealed class ObuReader
sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent = reader.ReadBoolean(); sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent = reader.ReadBoolean();
if (sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent) if (sequenceHeader.OperatingPoint[i].IsDecoderModelInfoPresent)
{ {
// Operating-point delays affect scheduling rather than still-image reconstruction, but their // Retain the scheduling values so the parsed sequence header can be written again without
// syntax must be consumed so the following image dimensions remain bit aligned. // losing decoder-model state that is independent from pixel reconstruction.
ObuDecoderModelInfo decoderModelInfo = sequenceHeader.GetDecoderModelInfo(); ObuDecoderModelInfo decoderModelInfo = sequenceHeader.GetDecoderModelInfo();
ReadOperatingParametersInfo(ref reader, (int)decoderModelInfo.BufferDelayLength); ReadOperatingParametersInfo(
ref reader,
(int)decoderModelInfo.BufferDelayLength,
sequenceHeader.OperatingPoint[i]);
} }
} }
else else
@ -1047,8 +999,8 @@ internal sealed class ObuReader
sequenceHeader.EnableDualFilter = false; sequenceHeader.EnableDualFilter = false;
sequenceHeader.OrderHintInfo.EnableJointCompound = false; sequenceHeader.OrderHintInfo.EnableJointCompound = false;
sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors = false; sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors = false;
sequenceHeader.ForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS sequenceHeader.ForceScreenContentTools = Av1Constants.SelectScreenContentTools;
sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector;
sequenceHeader.OrderHintInfo.OrderHintBits = 0; sequenceHeader.OrderHintInfo.OrderHintBits = 0;
} }
else else
@ -1072,7 +1024,7 @@ internal sealed class ObuReader
bool seqChooseScreenContentTools = reader.ReadBoolean(); bool seqChooseScreenContentTools = reader.ReadBoolean();
if (seqChooseScreenContentTools) if (seqChooseScreenContentTools)
{ {
sequenceHeader.ForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS sequenceHeader.ForceScreenContentTools = Av1Constants.SelectScreenContentTools;
} }
else else
{ {
@ -1084,7 +1036,7 @@ internal sealed class ObuReader
bool seqChooseIntegerMv = reader.ReadBoolean(); bool seqChooseIntegerMv = reader.ReadBoolean();
if (seqChooseIntegerMv) if (seqChooseIntegerMv)
{ {
sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector;
} }
else else
{ {
@ -1093,7 +1045,7 @@ internal sealed class ObuReader
} }
else else
{ {
sequenceHeader.ForceIntegerMotionVector = 2; // SELECT_INTEGER_MV sequenceHeader.ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector;
} }
if (sequenceHeader.EnableOrderHint) if (sequenceHeader.EnableOrderHint)
@ -1233,15 +1185,19 @@ internal sealed class ObuReader
}; };
/// <summary> /// <summary>
/// Consumes operating-point buffer parameters that do not affect still-image reconstruction. /// Reads the decoder-model parameters for one operating point.
/// </summary> /// </summary>
/// <param name="reader">The reader positioned at the operating-point parameters.</param> /// <param name="reader">The reader positioned at the operating-point parameters.</param>
/// <param name="bufferDelayLength">The bit width of each encoded buffer delay.</param> /// <param name="bufferDelayLength">The bit width of each encoded buffer delay.</param>
private static void ReadOperatingParametersInfo(ref Av1BitStreamReader reader, int bufferDelayLength) /// <param name="operatingPoint">The operating point that receives the decoded parameters.</param>
private static void ReadOperatingParametersInfo(
ref Av1BitStreamReader reader,
int bufferDelayLength,
ObuOperatingPoint operatingPoint)
{ {
_ = reader.ReadLiteral(bufferDelayLength); operatingPoint.DecoderBufferDelay = reader.ReadLiteral(bufferDelayLength);
_ = reader.ReadLiteral(bufferDelayLength); operatingPoint.EncoderBufferDelay = reader.ReadLiteral(bufferDelayLength);
_ = reader.ReadBoolean(); operatingPoint.LowDelayMode = reader.ReadBoolean();
} }
/// <summary> /// <summary>
@ -1716,7 +1672,7 @@ internal sealed class ObuReader
throw new InvalidImageContentException("An AV1 still picture cannot display a previously decoded frame."); throw new InvalidImageContentException("An AV1 still picture cannot display a previously decoded frame.");
} }
frameHeader.FrameToShowMapIdx = reader.ReadLiteral(3); frameHeader.FrameToShowMapIdx = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits);
if (sequenceHeader.DecoderModelInfoPresentFlag && sequenceHeader.TimingInfo?.EqualPictureInterval == false) if (sequenceHeader.DecoderModelInfoPresentFlag && sequenceHeader.TimingInfo?.EqualPictureInterval == false)
{ {
@ -1781,7 +1737,7 @@ internal sealed class ObuReader
return; return;
} }
frameHeader.FrameType = (ObuFrameType)reader.ReadLiteral(2); frameHeader.FrameType = (ObuFrameType)reader.ReadLiteral(Av1Constants.FrameTypeBits);
frameHeader.ShowFrame = reader.ReadBoolean(); frameHeader.ShowFrame = reader.ReadBoolean();
if (sequenceHeader.IsStillPicture && (frameHeader.FrameType != ObuFrameType.KeyFrame || !frameHeader.ShowFrame)) if (sequenceHeader.IsStillPicture && (frameHeader.FrameType != ObuFrameType.KeyFrame || !frameHeader.ShowFrame))
{ {
@ -2109,8 +2065,8 @@ internal sealed class ObuReader
if (usesShortSignaling) if (usesShortSignaling)
{ {
uint lastFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); uint lastFrameIndex = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits);
uint goldenFrameIndex = reader.ReadLiteral(ReferenceFrameIndexBits); uint goldenFrameIndex = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits);
InlineArray8<bool> slotOccupancyStorage = default; InlineArray8<bool> slotOccupancyStorage = default;
Span<bool> slotOccupancy = slotOccupancyStorage; Span<bool> slotOccupancy = slotOccupancyStorage;
@ -2136,7 +2092,7 @@ internal sealed class ObuReader
uint slot = referenceFrameIndices[reference]; uint slot = referenceFrameIndices[reference];
if (!usesShortSignaling) if (!usesShortSignaling)
{ {
slot = reader.ReadLiteral(ReferenceFrameIndexBits); slot = reader.ReadLiteral(Av1Constants.ReferenceFrameIndexBits);
referenceFrameIndices[reference] = slot; referenceFrameIndices[reference] = slot;
} }
@ -2778,31 +2734,43 @@ internal sealed class ObuReader
{ {
// Diagonal terms are coded as a delta from the identity scale, whereas off-diagonal terms are centered // Diagonal terms are coded as a delta from the identity scale, whereas off-diagonal terms are centered
// directly around zero. Both are restored to the common sixteen-bit matrix precision after decoding. // directly around zero. Both are restored to the common sixteen-bit matrix precision after decoding.
int referenceHorizontalScale =
(referenceParameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
parameters[2] = parameters[2] =
(reader.ReadSignedReferenceSubexponential( (reader.ReadSignedReferenceSubexponential(
GlobalMotionAlphaValueMagnitude, Av1GlobalMotionParameters.AlphaValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
(referenceParameters[2] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + referenceHorizontalScale) *
Av1GlobalMotionParameters.AlphaDecodeFactor) +
Av1GlobalMotionParameters.ModelScale; Av1GlobalMotionParameters.ModelScale;
parameters[3] = reader.ReadSignedReferenceSubexponential( parameters[3] = reader.ReadSignedReferenceSubexponential(
GlobalMotionAlphaValueMagnitude, Av1GlobalMotionParameters.AlphaValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[3] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; referenceParameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) *
Av1GlobalMotionParameters.AlphaDecodeFactor;
} }
if (type >= Av1GlobalMotionType.Affine) if (type >= Av1GlobalMotionType.Affine)
{ {
int referenceVerticalScale =
(referenceParameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
parameters[4] = reader.ReadSignedReferenceSubexponential( parameters[4] = reader.ReadSignedReferenceSubexponential(
GlobalMotionAlphaValueMagnitude, Av1GlobalMotionParameters.AlphaValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[4] >> GlobalMotionAlphaPrecisionDifference) * GlobalMotionAlphaDecodeFactor; referenceParameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) *
Av1GlobalMotionParameters.AlphaDecodeFactor;
parameters[5] = parameters[5] =
(reader.ReadSignedReferenceSubexponential( (reader.ReadSignedReferenceSubexponential(
GlobalMotionAlphaValueMagnitude, Av1GlobalMotionParameters.AlphaValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
(referenceParameters[5] >> GlobalMotionAlphaPrecisionDifference) - (1 << GlobalMotionAlphaPrecisionBits)) * GlobalMotionAlphaDecodeFactor) + referenceVerticalScale) *
Av1GlobalMotionParameters.AlphaDecodeFactor) +
Av1GlobalMotionParameters.ModelScale; Av1GlobalMotionParameters.ModelScale;
} }
else else
@ -2820,23 +2788,26 @@ internal sealed class ObuReader
// remains in quarter-sample units. Affine translation retains the fixed model-to-translation precision gap. // remains in quarter-sample units. Affine translation retains the fixed model-to-translation precision gap.
int precisionAdjustment = type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0; int precisionAdjustment = type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0;
int translationBits = type == Av1GlobalMotionType.Translation int translationBits = type == Av1GlobalMotionType.Translation
? GlobalMotionAbsoluteTranslationOnlyBits - precisionAdjustment ? Av1GlobalMotionParameters.AbsoluteTranslationOnlyBits - precisionAdjustment
: GlobalMotionAbsoluteTranslationBits; : Av1GlobalMotionParameters.AbsoluteTranslationBits;
int translationPrecisionDifference = type == Av1GlobalMotionType.Translation int translationPrecisionDifference = type == Av1GlobalMotionType.Translation
? Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationOnlyPrecisionBits + precisionAdjustment ? Av1GlobalMotionParameters.ModelPrecisionBits -
: Av1GlobalMotionParameters.ModelPrecisionBits - GlobalMotionTranslationPrecisionBits; Av1GlobalMotionParameters.TranslationOnlyPrecisionBits +
precisionAdjustment
: Av1GlobalMotionParameters.ModelPrecisionBits -
Av1GlobalMotionParameters.TranslationPrecisionBits;
int translationDecodeFactor = 1 << translationPrecisionDifference; int translationDecodeFactor = 1 << translationPrecisionDifference;
int translationValueMagnitude = (1 << translationBits) + 1; int translationValueMagnitude = (1 << translationBits) + 1;
parameters[0] = reader.ReadSignedReferenceSubexponential( parameters[0] = reader.ReadSignedReferenceSubexponential(
translationValueMagnitude, translationValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[0] >> translationPrecisionDifference) * translationDecodeFactor; referenceParameters[0] >> translationPrecisionDifference) * translationDecodeFactor;
parameters[1] = reader.ReadSignedReferenceSubexponential( parameters[1] = reader.ReadSignedReferenceSubexponential(
translationValueMagnitude, translationValueMagnitude,
GlobalMotionSubexponentialGroupBitCount, Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[1] >> translationPrecisionDifference) * translationDecodeFactor; referenceParameters[1] >> translationPrecisionDifference) * translationDecodeFactor;
} }

8
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuSequenceHeader.cs

@ -219,6 +219,14 @@ internal sealed class ObuSequenceHeader
/// </summary> /// </summary>
public uint AdditionalFrameIdLength { get; set; } public uint AdditionalFrameIdLength { get; set; }
/// <summary>
/// Gets the timing information required by syntax whose presence flag is set.
/// </summary>
/// <returns>The sequence timing information.</returns>
public ObuTimingInfo GetTimingInfo() =>
this.TimingInfo
?? throw new InvalidOperationException("The AV1 sequence has no timing information.");
/// <summary> /// <summary>
/// Gets the decoder-buffer model information required by syntax whose presence flag is set. /// Gets the decoder-buffer model information required by syntax whose presence flag is set.
/// </summary> /// </summary>

654
src/ImageSharp/Formats/Heif/Av1/OpenBitstreamUnit/ObuWriter.cs

@ -3,82 +3,142 @@
using System.Buffers; using System.Buffers;
using System.Buffers.Binary; using System.Buffers.Binary;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; namespace SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
/// <summary> /// <summary>
/// Writes the AV1 open bitstream units required for a single still-image frame. /// Writes the AV1 open bitstream units for one coded frame.
/// </summary> /// </summary>
internal sealed class ObuWriter internal sealed class ObuWriter : IDisposable
{ {
// Sequence and uncompressed-frame syntax have fixed field and array limits. A 512-byte owner covers their // Sequence and uncompressed-frame syntax have fixed field and array limits. A 512-byte owner covers their
// maximum supported representation without retaining any entropy-coded tile bytes in the header scratch. // maximum supported representation without retaining any entropy-coded tile bytes in the header scratch.
private const int MaximumHeaderLength = 512; private const int MaximumHeaderLength = 512;
private readonly IMemoryOwner<byte> headerOwner;
/// <summary>
/// Initializes a new instance of the <see cref="ObuWriter"/> class.
/// </summary>
/// <param name="configuration">The configuration providing reusable header memory.</param>
public ObuWriter(Configuration configuration)
=> this.headerOwner = configuration.MemoryAllocator.Allocate<byte>(MaximumHeaderLength);
/// <summary>
/// Writes a temporal delimiter, sequence header, and coded frame for the first sample in a sequence.
/// </summary>
/// <typeparam name="TTileWriter">The non-boxed tile source type.</typeparam>
/// <param name="stream">The destination stream.</param>
/// <param name="sequenceHeader">The sequence header.</param>
/// <param name="frameHeader">The frame header.</param>
/// <param name="tileWriter">The encoded tile source.</param>
public void WriteSequenceFrame<TTileWriter>(
Stream stream,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader,
TTileWriter tileWriter)
where TTileWriter : IAv1TileWriter
{
Span<byte> headerBuffer = this.headerOwner.Memory.Span[..MaximumHeaderLength];
Av1BitStreamWriter writer = new(headerBuffer);
WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []);
WriteSequenceHeader(ref writer, sequenceHeader);
int bytesWritten = (writer.BitPosition + 7) >> 3;
writer.Flush();
WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]);
WriteFrameObu(stream, sequenceHeader, frameHeader, tileWriter, headerBuffer, ref writer);
}
/// <summary> /// <summary>
/// Writes a temporal delimiter and the supplied sequence and frame OBUs. /// Writes an empty temporal-delimiter OBU.
/// </summary>
/// <param name="stream">The destination stream.</param>
public static void WriteTemporalDelimiter(Stream stream)
=> WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []);
/// <summary>
/// Writes a temporal delimiter followed by one sequence-header OBU.
/// </summary> /// </summary>
/// <param name="configuration">The configuration used to allocate temporary encoding memory.</param> /// <param name="configuration">The configuration used to allocate temporary encoding memory.</param>
/// <param name="stream">The destination stream.</param> /// <param name="stream">The destination stream.</param>
/// <param name="sequenceHeader">The optional still-picture sequence header.</param> /// <param name="sequenceHeader">The sequence header.</param>
/// <param name="frameHeader">The optional intra-frame header.</param> public static void WriteSequenceHeader(Configuration configuration, Stream stream, ObuSequenceHeader sequenceHeader)
/// <param name="tileWriter">The tile writer used when a frame header is supplied.</param> {
[System.Diagnostics.CodeAnalysis.SuppressMessage(
"Performance",
"CA1822:Mark members as static",
Justification = "Preserves the existing writer instance contract.")]
public void WriteAll(Configuration configuration, Stream stream, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, IAv1TileWriter tileWriter)
{
// The reusable scratch only contains headers. Entropy-coded tiles remain in their owning buffers and are
// streamed directly so the complete compressed frame is never duplicated.
using IMemoryOwner<byte> headerOwner = configuration.MemoryAllocator.Allocate<byte>(MaximumHeaderLength); using IMemoryOwner<byte> headerOwner = configuration.MemoryAllocator.Allocate<byte>(MaximumHeaderLength);
Span<byte> headerBuffer = headerOwner.Memory.Span[..MaximumHeaderLength]; Span<byte> headerBuffer = headerOwner.Memory.Span[..MaximumHeaderLength];
Av1BitStreamWriter writer = new(headerBuffer); Av1BitStreamWriter writer = new(headerBuffer);
WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []); WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []);
WriteSequenceHeader(ref writer, sequenceHeader);
int bytesWritten = (writer.BitPosition + 7) >> 3;
writer.Flush();
WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]);
}
if (sequenceHeader != null) /// <summary>
{ /// Writes a temporal delimiter and coded frame that continues an established sequence.
WriteSequenceHeader(ref writer, sequenceHeader); /// </summary>
int bytesWritten = (writer.BitPosition + 7) >> 3; /// <typeparam name="TTileWriter">The non-boxed tile source type.</typeparam>
writer.Flush(); /// <param name="stream">The destination stream.</param>
WriteObuHeaderAndSize(stream, ObuType.SequenceHeader, headerBuffer[..bytesWritten]); /// <param name="sequenceHeader">The sequence header established by an earlier sample.</param>
} /// <param name="frameHeader">The frame header.</param>
/// <param name="tileWriter">The encoded tile source.</param>
if (frameHeader != null && sequenceHeader != null) public void WriteFrame<TTileWriter>(
{ Stream stream,
WriteFrameHeader(ref writer, sequenceHeader, frameHeader); ObuSequenceHeader sequenceHeader,
ObuTileGroupHeader tileInfo = frameHeader.TilesInfo; ObuFrameHeader frameHeader,
if (tileInfo != null) TTileWriter tileWriter)
{ where TTileWriter : IAv1TileWriter
WriteTileGroupHeader(ref writer, tileInfo); {
} Span<byte> headerBuffer = this.headerOwner.Memory.Span[..MaximumHeaderLength];
Av1BitStreamWriter writer = new(headerBuffer);
int frameHeaderBytes = (writer.BitPosition + 7) >> 3; WriteObuHeaderAndSize(stream, ObuType.TemporalDelimiter, []);
writer.Flush(); WriteFrameObu(stream, sequenceHeader, frameHeader, tileWriter, headerBuffer, ref writer);
}
uint framePayloadSize = (uint)frameHeaderBytes; /// <inheritdoc/>
if (tileInfo != null) public void Dispose() => this.headerOwner.Dispose();
{
int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount;
framePayloadSize += (uint)((tileCount - 1) * tileInfo.TileSizeBytes);
for (int tileNum = 0; tileNum < tileCount; tileNum++) /// <summary>
{ /// Writes the combined frame OBU header followed by each retained tile payload.
framePayloadSize += (uint)tileWriter.GetTileData(tileNum).Length; /// </summary>
} /// <typeparam name="TTileWriter">The non-boxed tile source type.</typeparam>
} /// <param name="stream">The destination stream.</param>
/// <param name="sequenceHeader">The sequence header governing frame syntax.</param>
/// <param name="frameHeader">The uncompressed frame header.</param>
/// <param name="tileWriter">The encoded tile source.</param>
/// <param name="headerBuffer">The reusable OBU header scratch.</param>
/// <param name="writer">The bit writer over <paramref name="headerBuffer"/>.</param>
private static void WriteFrameObu<TTileWriter>(
Stream stream,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader,
TTileWriter tileWriter,
Span<byte> headerBuffer,
ref Av1BitStreamWriter writer)
where TTileWriter : IAv1TileWriter
{
WriteFrameHeader(ref writer, sequenceHeader, frameHeader);
ObuTileGroupHeader tileInfo = frameHeader.TilesInfo;
WriteTileGroupHeader(ref writer, tileInfo);
WriteObuHeaderAndSize(stream, ObuType.Frame, framePayloadSize); int frameHeaderBytes = (writer.BitPosition + 7) >> 3;
stream.Write(headerBuffer[..frameHeaderBytes]); writer.Flush();
if (tileInfo != null) int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount;
{ uint framePayloadSize = (uint)(frameHeaderBytes + ((tileCount - 1) * tileInfo.TileSizeBytes));
WriteTileData(stream, tileInfo, tileWriter); for (int tileNum = 0; tileNum < tileCount; tileNum++)
} {
framePayloadSize += (uint)tileWriter.GetTileData(tileNum).Length;
} }
WriteObuHeaderAndSize(stream, ObuType.Frame, framePayloadSize);
stream.Write(headerBuffer[..frameHeaderBytes]);
WriteTileData(stream, tileInfo, tileWriter);
} }
/// <summary> /// <summary>
@ -145,29 +205,123 @@ internal sealed class ObuWriter
} }
/// <summary> /// <summary>
/// Writes a reduced still-picture sequence header. /// Writes an AV1 sequence header.
/// </summary> /// </summary>
/// <param name="writer">The bit writer receiving the sequence header.</param> /// <param name="writer">The bit writer receiving the sequence header.</param>
/// <param name="sequenceHeader">The sequence header to encode.</param> /// <param name="sequenceHeader">The sequence header to encode.</param>
private static void WriteSequenceHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader) private static void WriteSequenceHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader)
{ {
writer.WriteLiteral((uint)sequenceHeader.SequenceProfile, 3); writer.WriteLiteral((uint)sequenceHeader.SequenceProfile, 3);
writer.WriteBoolean(true); // IsStillPicture writer.WriteBoolean(sequenceHeader.IsStillPicture);
writer.WriteBoolean(true); // IsReducedStillPicture writer.WriteBoolean(sequenceHeader.IsReducedStillPictureHeader);
writer.WriteLiteral((uint)sequenceHeader.OperatingPoint[0].SequenceLevelIndex, Av1Constants.LevelBits); if (sequenceHeader.IsReducedStillPictureHeader)
{
writer.WriteLiteral((uint)sequenceHeader.OperatingPoint[0].SequenceLevelIndex, Av1Constants.LevelBits);
}
else
{
writer.WriteBoolean(sequenceHeader.TimingInfoPresentFlag);
if (sequenceHeader.TimingInfoPresentFlag)
{
WriteTimingInfo(ref writer, sequenceHeader.GetTimingInfo());
writer.WriteBoolean(sequenceHeader.DecoderModelInfoPresentFlag);
if (sequenceHeader.DecoderModelInfoPresentFlag)
{
WriteDecoderModelInfo(ref writer, sequenceHeader.GetDecoderModelInfo());
}
}
writer.WriteBoolean(sequenceHeader.InitialDisplayDelayPresentFlag);
writer.WriteLiteral(
(uint)(sequenceHeader.OperatingPoint.Length - 1),
Av1Constants.OperatingPointCountBits);
foreach (ObuOperatingPoint operatingPoint in sequenceHeader.OperatingPoint)
{
writer.WriteLiteral(operatingPoint.Idc, Av1Constants.OperatingPointIdcBits);
writer.WriteLiteral((uint)operatingPoint.SequenceLevelIndex, Av1Constants.LevelBits);
if (operatingPoint.SequenceLevelIndex >= Av1Constants.SequenceTierMinimumLevelIndex)
{
writer.WriteBoolean(operatingPoint.SequenceTier != 0);
}
if (sequenceHeader.DecoderModelInfoPresentFlag)
{
writer.WriteBoolean(operatingPoint.IsDecoderModelInfoPresent);
if (operatingPoint.IsDecoderModelInfoPresent)
{
WriteOperatingParametersInfo(
ref writer,
sequenceHeader.GetDecoderModelInfo(),
operatingPoint);
}
}
if (sequenceHeader.InitialDisplayDelayPresentFlag)
{
writer.WriteBoolean(operatingPoint.IsInitialDisplayDelayPresent);
if (operatingPoint.IsInitialDisplayDelayPresent)
{
writer.WriteLiteral(operatingPoint.InitialDisplayDelay - 1, 4);
}
}
}
}
// Frame width and Height // The maximum dimensions determine the fixed-width fields used by every frame in the sequence.
writer.WriteLiteral((uint)sequenceHeader.FrameWidthBits - 1, 4); writer.WriteLiteral((uint)sequenceHeader.FrameWidthBits - 1, 4);
writer.WriteLiteral((uint)sequenceHeader.FrameHeightBits - 1, 4); writer.WriteLiteral((uint)sequenceHeader.FrameHeightBits - 1, 4);
writer.WriteLiteral((uint)sequenceHeader.MaxFrameWidth - 1, sequenceHeader.FrameWidthBits); writer.WriteLiteral((uint)sequenceHeader.MaxFrameWidth - 1, sequenceHeader.FrameWidthBits);
writer.WriteLiteral((uint)sequenceHeader.MaxFrameHeight - 1, sequenceHeader.FrameHeightBits); writer.WriteLiteral((uint)sequenceHeader.MaxFrameHeight - 1, sequenceHeader.FrameHeightBits);
if (!sequenceHeader.IsReducedStillPictureHeader)
{
writer.WriteBoolean(sequenceHeader.IsFrameIdNumbersPresent);
if (sequenceHeader.IsFrameIdNumbersPresent)
{
writer.WriteLiteral((uint)sequenceHeader.DeltaFrameIdLength - 2, 4);
writer.WriteLiteral(sequenceHeader.AdditionalFrameIdLength - 1, 3);
}
}
// Video related flags removed
writer.WriteBoolean(sequenceHeader.Use128x128Superblock); writer.WriteBoolean(sequenceHeader.Use128x128Superblock);
writer.WriteBoolean(sequenceHeader.EnableFilterIntra); writer.WriteBoolean(sequenceHeader.EnableFilterIntra);
writer.WriteBoolean(sequenceHeader.EnableIntraEdgeFilter); writer.WriteBoolean(sequenceHeader.EnableIntraEdgeFilter);
if (!sequenceHeader.IsReducedStillPictureHeader)
{
writer.WriteBoolean(sequenceHeader.EnableInterIntraCompound);
writer.WriteBoolean(sequenceHeader.EnableMaskedCompound);
writer.WriteBoolean(sequenceHeader.EnableWarpedMotion);
writer.WriteBoolean(sequenceHeader.EnableDualFilter);
writer.WriteBoolean(sequenceHeader.EnableOrderHint);
if (sequenceHeader.EnableOrderHint)
{
writer.WriteBoolean(sequenceHeader.OrderHintInfo.EnableJointCompound);
writer.WriteBoolean(sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors);
}
bool chooseScreenContentTools = sequenceHeader.ForceScreenContentTools == Av1Constants.SelectScreenContentTools;
writer.WriteBoolean(chooseScreenContentTools);
if (!chooseScreenContentTools)
{
writer.WriteBoolean(sequenceHeader.ForceScreenContentTools != 0);
}
if (sequenceHeader.ForceScreenContentTools > 0)
{
bool chooseIntegerMotionVector = sequenceHeader.ForceIntegerMotionVector == Av1Constants.SelectIntegerMotionVector;
writer.WriteBoolean(chooseIntegerMotionVector);
if (!chooseIntegerMotionVector)
{
writer.WriteBoolean(sequenceHeader.ForceIntegerMotionVector != 0);
}
}
if (sequenceHeader.EnableOrderHint)
{
writer.WriteLiteral((uint)sequenceHeader.OrderHintInfo.OrderHintBits - 1, 3);
}
}
// Video related flags removed
writer.WriteBoolean(sequenceHeader.EnableSuperResolution); writer.WriteBoolean(sequenceHeader.EnableSuperResolution);
writer.WriteBoolean(sequenceHeader.EnableCdef); writer.WriteBoolean(sequenceHeader.EnableCdef);
writer.WriteBoolean(sequenceHeader.EnableRestoration); writer.WriteBoolean(sequenceHeader.EnableRestoration);
@ -176,6 +330,65 @@ internal sealed class ObuWriter
WriteTrailingBits(ref writer); WriteTrailingBits(ref writer);
} }
/// <summary>
/// Writes sequence timing in the fixed-width and unsigned-variable-length forms required by AV1.
/// </summary>
/// <param name="writer">The bit writer receiving the timing information.</param>
/// <param name="timingInfo">The timing values to encode.</param>
private static void WriteTimingInfo(ref Av1BitStreamWriter writer, ObuTimingInfo timingInfo)
{
writer.WriteLiteral(timingInfo.NumUnitsInDisplayTick, 32);
writer.WriteLiteral(timingInfo.TimeScale, 32);
writer.WriteBoolean(timingInfo.EqualPictureInterval);
if (timingInfo.EqualPictureInterval)
{
WriteUnsignedVariableLength(ref writer, timingInfo.NumTicksPerPicture - 1);
}
}
/// <summary>
/// Writes decoder-buffer field widths and decoding-clock units.
/// </summary>
/// <param name="writer">The bit writer receiving the decoder-model information.</param>
/// <param name="decoderModelInfo">The decoder-model values to encode.</param>
private static void WriteDecoderModelInfo(ref Av1BitStreamWriter writer, ObuDecoderModelInfo decoderModelInfo)
{
writer.WriteLiteral(decoderModelInfo.BufferDelayLength - 1, 5);
writer.WriteLiteral(decoderModelInfo.NumUnitsInDecodingTick, 32);
writer.WriteLiteral(decoderModelInfo.BufferRemovalTimeLength - 1, 5);
writer.WriteLiteral(decoderModelInfo.FramePresentationTimeLength - 1, 5);
}
/// <summary>
/// Writes the decoder-model parameters for one operating point.
/// </summary>
/// <param name="writer">The bit writer receiving the operating-point parameters.</param>
/// <param name="decoderModelInfo">The decoder model defining the delay field width.</param>
/// <param name="operatingPoint">The operating-point values to encode.</param>
private static void WriteOperatingParametersInfo(
ref Av1BitStreamWriter writer,
ObuDecoderModelInfo decoderModelInfo,
ObuOperatingPoint operatingPoint)
{
int bufferDelayLength = (int)decoderModelInfo.BufferDelayLength;
writer.WriteLiteral(operatingPoint.DecoderBufferDelay, bufferDelayLength);
writer.WriteLiteral(operatingPoint.EncoderBufferDelay, bufferDelayLength);
writer.WriteBoolean(operatingPoint.LowDelayMode);
}
/// <summary>
/// Writes an AV1 unsigned variable-length value.
/// </summary>
/// <param name="writer">The bit writer receiving the value.</param>
/// <param name="value">The value to encode.</param>
private static void WriteUnsignedVariableLength(ref Av1BitStreamWriter writer, uint value)
{
uint encodedValue = value + 1;
int leadingZeroCount = Av1Math.MostSignificantBit(encodedValue);
writer.WriteLiteral(0, leadingZeroCount);
writer.WriteLiteral(encodedValue, leadingZeroCount + 1);
}
/// <summary> /// <summary>
/// Writes the sequence color configuration. /// Writes the sequence color configuration.
/// </summary> /// </summary>
@ -418,15 +631,60 @@ internal sealed class ObuWriter
} }
/// <summary> /// <summary>
/// Writes the reduced uncompressed header for an intra still-image frame. /// Writes the uncompressed header for an AV1 frame.
/// </summary> /// </summary>
/// <param name="writer">The bit writer receiving the uncompressed frame header.</param> /// <param name="writer">The bit writer receiving the uncompressed frame header.</param>
/// <param name="sequenceHeader">The sequence header controlling available coding tools.</param> /// <param name="sequenceHeader">The sequence header controlling available coding tools.</param>
/// <param name="frameHeader">The frame header to encode.</param> /// <param name="frameHeader">The frame header to encode.</param>
private static void WriteUncompressedFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) private static void WriteUncompressedFrameHeader(ref Av1BitStreamWriter writer, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader)
{ {
bool frameSizeOverrideFlag = false;
if (!sequenceHeader.IsReducedStillPictureHeader)
{
writer.WriteBoolean(frameHeader.ShowExistingFrame);
if (frameHeader.ShowExistingFrame)
{
writer.WriteLiteral(frameHeader.FrameToShowMapIdx, Av1Constants.ReferenceFrameIndexBits);
if (sequenceHeader.DecoderModelInfoPresentFlag && !sequenceHeader.GetTimingInfo().EqualPictureInterval)
{
writer.WriteLiteral(
frameHeader.FramePresentationTime,
(int)sequenceHeader.GetDecoderModelInfo().FramePresentationTimeLength);
}
if (sequenceHeader.IsFrameIdNumbersPresent)
{
writer.WriteLiteral(frameHeader.DisplayFrameId, sequenceHeader.FrameIdLength);
}
return;
}
writer.WriteLiteral((uint)frameHeader.FrameType, Av1Constants.FrameTypeBits);
writer.WriteBoolean(frameHeader.ShowFrame);
if (frameHeader.ShowFrame &&
sequenceHeader.DecoderModelInfoPresentFlag &&
!sequenceHeader.GetTimingInfo().EqualPictureInterval)
{
writer.WriteLiteral(
frameHeader.FramePresentationTime,
(int)sequenceHeader.GetDecoderModelInfo().FramePresentationTimeLength);
}
if (!frameHeader.ShowFrame)
{
writer.WriteBoolean(frameHeader.ShowableFrame);
}
if (frameHeader.FrameType != ObuFrameType.SwitchFrame &&
(frameHeader.FrameType != ObuFrameType.KeyFrame || !frameHeader.ShowFrame))
{
writer.WriteBoolean(frameHeader.ErrorResilientMode);
}
}
writer.WriteBoolean(frameHeader.DisableCdfUpdate); writer.WriteBoolean(frameHeader.DisableCdfUpdate);
if (sequenceHeader.ForceScreenContentTools == 2) if (sequenceHeader.ForceScreenContentTools == Av1Constants.SelectScreenContentTools)
{ {
writer.WriteBoolean(frameHeader.AllowScreenContentTools); writer.WriteBoolean(frameHeader.AllowScreenContentTools);
} }
@ -437,7 +695,7 @@ internal sealed class ObuWriter
if (frameHeader.AllowScreenContentTools) if (frameHeader.AllowScreenContentTools)
{ {
if (sequenceHeader.ForceIntegerMotionVector == 2) if (sequenceHeader.ForceIntegerMotionVector == Av1Constants.SelectIntegerMotionVector)
{ {
writer.WriteBoolean(frameHeader.ForceIntegerMotionVector); writer.WriteBoolean(frameHeader.ForceIntegerMotionVector);
} }
@ -447,6 +705,41 @@ internal sealed class ObuWriter
} }
} }
if (!sequenceHeader.IsReducedStillPictureHeader)
{
if (sequenceHeader.IsFrameIdNumbersPresent)
{
writer.WriteLiteral(frameHeader.CurrentFrameId, sequenceHeader.FrameIdLength);
}
frameSizeOverrideFlag = frameHeader.FrameType == ObuFrameType.SwitchFrame ||
frameHeader.FrameSize.SuperResolutionUpscaledWidth != sequenceHeader.MaxFrameWidth ||
frameHeader.FrameSize.FrameHeight != sequenceHeader.MaxFrameHeight;
if (frameHeader.FrameType != ObuFrameType.SwitchFrame)
{
writer.WriteBoolean(frameSizeOverrideFlag);
}
writer.WriteLiteral(frameHeader.OrderHint, sequenceHeader.OrderHintInfo.OrderHintBits);
if (!frameHeader.ErrorResilientMode && !frameHeader.IsIntra)
{
writer.WriteLiteral(frameHeader.PrimaryReferenceFrame, Av1Constants.PrimaryReferenceBits);
}
}
if (sequenceHeader.DecoderModelInfoPresentFlag)
{
// Image-sequence timing is carried by the container track, so encoded samples do not signal decoder-buffer removal times.
writer.WriteBoolean(false);
}
if ((frameHeader.FrameType == ObuFrameType.KeyFrame && !frameHeader.ShowFrame) ||
frameHeader.FrameType is ObuFrameType.InterFrame or ObuFrameType.IntraOnlyFrame)
{
writer.WriteLiteral(frameHeader.RefreshFrameFlags, Av1Constants.ReferenceFrameCount);
}
if (frameHeader.FrameType == ObuFrameType.KeyFrame) if (frameHeader.FrameType == ObuFrameType.KeyFrame)
{ {
if (!frameHeader.ShowFrame) if (!frameHeader.ShowFrame)
@ -461,7 +754,7 @@ internal sealed class ObuWriter
if (frameHeader.FrameType == ObuFrameType.KeyFrame) if (frameHeader.FrameType == ObuFrameType.KeyFrame)
{ {
WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag);
WriteRenderSize(ref writer, frameHeader); WriteRenderSize(ref writer, frameHeader);
if (frameHeader.AllowScreenContentTools) if (frameHeader.AllowScreenContentTools)
{ {
@ -470,7 +763,7 @@ internal sealed class ObuWriter
} }
else if (frameHeader.FrameType == ObuFrameType.IntraOnlyFrame) else if (frameHeader.FrameType == ObuFrameType.IntraOnlyFrame)
{ {
WriteFrameSize(ref writer, sequenceHeader, frameHeader, false); WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag);
WriteRenderSize(ref writer, frameHeader); WriteRenderSize(ref writer, frameHeader);
if (frameHeader.AllowScreenContentTools) if (frameHeader.AllowScreenContentTools)
{ {
@ -479,7 +772,32 @@ internal sealed class ObuWriter
} }
else else
{ {
throw new NotImplementedException("Inter frames not applicable for AVIF."); WriteReferenceFrameIndices(ref writer, sequenceHeader, frameHeader);
WriteFrameSize(ref writer, sequenceHeader, frameHeader, frameSizeOverrideFlag);
WriteRenderSize(ref writer, frameHeader);
if (!frameHeader.ForceIntegerMotionVector)
{
writer.WriteBoolean(frameHeader.AllowHighPrecisionMotionVector);
}
WriteFrameInterpolationFilter(ref writer, frameHeader.InterpolationFilter);
writer.WriteBoolean(frameHeader.IsMotionModeSwitchable);
}
bool mightAllowReferenceFrameMotionVectors =
!frameHeader.ErrorResilientMode &&
sequenceHeader.OrderHintInfo.EnableReferenceFrameMotionVectors &&
sequenceHeader.EnableOrderHint &&
!frameHeader.IsIntra;
if (mightAllowReferenceFrameMotionVectors)
{
writer.WriteBoolean(frameHeader.UseReferenceFrameMotionVectors);
}
if (!sequenceHeader.IsReducedStillPictureHeader && !frameHeader.DisableCdfUpdate)
{
writer.WriteBoolean(frameHeader.DisableFrameEndUpdateCdf);
} }
WriteTileInfo(ref writer, sequenceHeader, frameHeader); WriteTileInfo(ref writer, sequenceHeader, frameHeader);
@ -531,20 +849,68 @@ internal sealed class ObuWriter
} }
} }
// No Frame Reference mode selection for AVIF
WriteTransformMode(ref writer, frameHeader); WriteTransformMode(ref writer, frameHeader);
// No compound INTER-INTER for AVIF.
WriteFrameReferenceMode(ref writer, frameHeader); WriteFrameReferenceMode(ref writer, frameHeader);
WriteSkipModeParameters(ref writer, frameHeader); WriteSkipModeParameters(ref writer, frameHeader);
if (!frameHeader.IsIntra && !frameHeader.ErrorResilientMode && sequenceHeader.EnableWarpedMotion)
{
writer.WriteBoolean(frameHeader.AllowWarpedMotion);
}
// No warp motion for AVIF.
writer.WriteBoolean(frameHeader.UseReducedTransformSet); writer.WriteBoolean(frameHeader.UseReducedTransformSet);
WriteGlobalMotionParameters(ref writer, frameHeader); WriteGlobalMotionParameters(ref writer, frameHeader);
WriteFilmGrainFilterParameters(ref writer, sequenceHeader, frameHeader); WriteFilmGrainFilterParameters(ref writer, sequenceHeader, frameHeader);
} }
/// <summary>
/// Writes the seven reference-map slots selected by an inter frame.
/// </summary>
/// <param name="writer">The bit writer receiving the reference indices.</param>
/// <param name="sequenceHeader">The sequence header defining frame-ID and order-hint syntax.</param>
/// <param name="frameHeader">The frame header containing the selected reference slots.</param>
private static void WriteReferenceFrameIndices(
ref Av1BitStreamWriter writer,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader)
{
// Long signaling is deterministic and permits every reference role to select the same retained slot.
if (sequenceHeader.EnableOrderHint)
{
writer.WriteBoolean(false);
}
Span<uint> referenceFrameIndices = frameHeader.GetReferenceFrameIndices();
Span<uint> referenceFrameIds = frameHeader.GetReferenceFrameIds();
uint frameIdModulus = 1U << sequenceHeader.FrameIdLength;
for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++)
{
uint slot = referenceFrameIndices[reference];
writer.WriteLiteral(slot, Av1Constants.ReferenceFrameIndexBits);
if (sequenceHeader.IsFrameIdNumbersPresent)
{
uint deltaFrameId = (frameHeader.CurrentFrameId + frameIdModulus - referenceFrameIds[(int)slot]) % frameIdModulus;
writer.WriteLiteral(deltaFrameId - 1, sequenceHeader.DeltaFrameIdLength);
}
}
}
/// <summary>
/// Writes the frame-level interpolation-filter selection.
/// </summary>
/// <param name="writer">The bit writer receiving the filter selection.</param>
/// <param name="filter">The fixed filter family or per-block selection.</param>
private static void WriteFrameInterpolationFilter(ref Av1BitStreamWriter writer, Av1InterpolationFilter filter)
{
bool isSwitchable = filter == Av1InterpolationFilter.Switchable;
writer.WriteBoolean(isSwitchable);
if (!isSwitchable)
{
writer.WriteLiteral((uint)filter, 2);
}
}
/// <summary> /// <summary>
/// Writes the frame-header portion of a combined frame OBU. /// Writes the frame-header portion of a combined frame OBU.
/// </summary> /// </summary>
@ -571,8 +937,8 @@ internal sealed class ObuWriter
if (tileCount > 1) if (tileCount > 1)
{ {
// A combined OBU_FRAME has implicit complete-frame tile bounds. The reference decoder still // A combined frame always carries the complete raster tile group. The zero bit selects those implicit
// writes the presence bit for a multi-tile frame, but requires that bit to remain zero. // full-frame bounds instead of adding explicit start and end tile indices.
writer.WriteBoolean(false); writer.WriteBoolean(false);
} }
@ -582,10 +948,15 @@ internal sealed class ObuWriter
/// <summary> /// <summary>
/// Writes the size-prefixed tile payloads in raster order. /// Writes the size-prefixed tile payloads in raster order.
/// </summary> /// </summary>
/// <typeparam name="TTileWriter">The non-boxed tile source type.</typeparam>
/// <param name="stream">The destination stream receiving tile data.</param> /// <param name="stream">The destination stream receiving tile data.</param>
/// <param name="tileInfo">The frame tile layout and tile-size field width.</param> /// <param name="tileInfo">The frame tile layout and tile-size field width.</param>
/// <param name="tileWriter">The writer that produces each tile payload.</param> /// <param name="tileWriter">The writer that produces each tile payload.</param>
private static void WriteTileData(Stream stream, ObuTileGroupHeader tileInfo, IAv1TileWriter tileWriter) private static void WriteTileData<TTileWriter>(
Stream stream,
ObuTileGroupHeader tileInfo,
TTileWriter tileWriter)
where TTileWriter : IAv1TileWriter
{ {
int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount; int tileCount = tileInfo.TileColumnCount * tileInfo.TileRowCount;
Span<byte> tileSizeBuffer = stackalloc byte[sizeof(uint)]; Span<byte> tileSizeBuffer = stackalloc byte[sizeof(uint)];
@ -659,7 +1030,7 @@ internal sealed class ObuWriter
} }
/// <summary> /// <summary>
/// Writes segmentation feature data for an independently decoded still-image frame. /// Writes segmentation feature data for one coded frame.
/// </summary> /// </summary>
/// <param name="writer">The bit writer receiving the segmentation parameters.</param> /// <param name="writer">The bit writer receiving the segmentation parameters.</param>
/// <param name="frameHeader">The frame header containing segmentation feature data.</param> /// <param name="frameHeader">The frame header containing segmentation feature data.</param>
@ -672,9 +1043,8 @@ internal sealed class ObuWriter
return; return;
} }
// The still-image writer emits independent intra frames with no primary reference. // A frame with no primary reference starts a new segmentation domain. AV1 therefore infers
// AV1 therefore infers update-map and update-data as enabled and carries feature data // update-map and update-data as enabled and carries the complete feature state directly.
// directly, without the inter-frame update flags.
for (int segmentId = 0; segmentId < Av1Constants.MaxSegmentCount; segmentId++) for (int segmentId = 0; segmentId < Av1Constants.MaxSegmentCount; segmentId++)
{ {
for (int featureId = 0; featureId < Av1Constants.SegmentationLevelMax; featureId++) for (int featureId = 0; featureId < Av1Constants.SegmentationLevelMax; featureId++)
@ -839,15 +1209,140 @@ internal sealed class ObuWriter
/// <param name="frameHeader">The current frame header.</param> /// <param name="frameHeader">The current frame header.</param>
private static void WriteGlobalMotionParameters(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader) private static void WriteGlobalMotionParameters(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader)
{ {
_ = writer;
if (frameHeader.IsIntra) if (frameHeader.IsIntra)
{ {
// Nothing to be written for INTRA frames.
return; return;
} }
throw new InvalidImageContentException("AVIF files can only contain INTRA frames."); ReadOnlySpan<Av1GlobalMotionParameters> parameters = frameHeader.GetGlobalMotionParameters();
Av1GlobalMotionParameters referenceParameters = Av1GlobalMotionParameters.Identity;
for (int reference = 0; reference < Av1Constants.ReferencesPerFrame; reference++)
{
WriteGlobalMotionModel(
ref writer,
parameters[reference],
referenceParameters,
frameHeader.AllowHighPrecisionMotionVector);
}
}
/// <summary>
/// Writes one global-motion model relative to the same-role model in the primary reference frame.
/// </summary>
private static void WriteGlobalMotionModel(
ref Av1BitStreamWriter writer,
Av1GlobalMotionParameters parameters,
Av1GlobalMotionParameters referenceParameters,
bool allowHighPrecisionMotionVector)
{
Av1GlobalMotionType type = parameters.Type;
writer.WriteBoolean(type != Av1GlobalMotionType.Identity);
if (type != Av1GlobalMotionType.Identity)
{
writer.WriteBoolean(type == Av1GlobalMotionType.RotationZoom);
if (type != Av1GlobalMotionType.RotationZoom)
{
writer.WriteBoolean(type == Av1GlobalMotionType.Translation);
}
}
if (type >= Av1GlobalMotionType.RotationZoom)
{
int horizontalScale =
(parameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
int referenceHorizontalScale =
(referenceParameters[2] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
writer.WriteSignedReferenceSubexponential(
horizontalScale,
Av1GlobalMotionParameters.AlphaValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceHorizontalScale);
writer.WriteSignedReferenceSubexponential(
parameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference,
Av1GlobalMotionParameters.AlphaValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[3] >> Av1GlobalMotionParameters.AlphaPrecisionDifference);
}
if (type >= Av1GlobalMotionType.Affine)
{
int verticalScale =
(parameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
int referenceVerticalScale =
(referenceParameters[5] >> Av1GlobalMotionParameters.AlphaPrecisionDifference) -
(1 << Av1GlobalMotionParameters.AlphaPrecisionBits);
writer.WriteSignedReferenceSubexponential(
parameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference,
Av1GlobalMotionParameters.AlphaValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[4] >> Av1GlobalMotionParameters.AlphaPrecisionDifference);
writer.WriteSignedReferenceSubexponential(
verticalScale,
Av1GlobalMotionParameters.AlphaValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceVerticalScale);
}
if (type >= Av1GlobalMotionType.Translation)
{
int precisionAdjustment =
type == Av1GlobalMotionType.Translation && !allowHighPrecisionMotionVector ? 1 : 0;
int translationBits = type == Av1GlobalMotionType.Translation
? Av1GlobalMotionParameters.AbsoluteTranslationOnlyBits - precisionAdjustment
: Av1GlobalMotionParameters.AbsoluteTranslationBits;
int translationPrecisionDifference = type == Av1GlobalMotionType.Translation
? Av1GlobalMotionParameters.ModelPrecisionBits -
Av1GlobalMotionParameters.TranslationOnlyPrecisionBits +
precisionAdjustment
: Av1GlobalMotionParameters.ModelPrecisionBits -
Av1GlobalMotionParameters.TranslationPrecisionBits;
int translationValueMagnitude = (1 << translationBits) + 1;
writer.WriteSignedReferenceSubexponential(
parameters[0] >> translationPrecisionDifference,
translationValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[0] >> translationPrecisionDifference);
writer.WriteSignedReferenceSubexponential(
parameters[1] >> translationPrecisionDifference,
translationValueMagnitude,
Av1GlobalMotionParameters.SubexponentialGroupBitCount,
referenceParameters[1] >> translationPrecisionDifference);
}
}
/// <summary>
/// Gets the exact number of uncompressed-header bits required by one global-motion model.
/// </summary>
/// <param name="parameters">The model to measure.</param>
/// <param name="allowHighPrecisionMotionVector">Whether translation may retain one-eighth-sample precision.</param>
/// <returns>The encoded model length in bits.</returns>
internal static int GetGlobalMotionModelBitCount(
Av1GlobalMotionParameters parameters,
bool allowHighPrecisionMotionVector)
{
InlineArray16<byte> storage = default;
Span<byte> buffer = storage;
Av1BitStreamWriter writer = new(buffer);
WriteGlobalMotionModel(
ref writer,
parameters,
Av1GlobalMotionParameters.Identity,
allowHighPrecisionMotionVector);
return writer.BitPosition;
} }
/// <summary> /// <summary>
@ -857,15 +1352,12 @@ internal sealed class ObuWriter
/// <param name="frameHeader">The current frame header.</param> /// <param name="frameHeader">The current frame header.</param>
private static void WriteFrameReferenceMode(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader) private static void WriteFrameReferenceMode(ref Av1BitStreamWriter writer, ObuFrameHeader frameHeader)
{ {
_ = writer;
if (frameHeader.IsIntra) if (frameHeader.IsIntra)
{ {
// Nothing to be written for INTRA frames.
return; return;
} }
throw new InvalidImageContentException("AVIF files can only contain INTRA frames."); writer.WriteBoolean(frameHeader.ReferenceMode == ObuReferenceMode.ReferenceModeSelect);
} }
/// <summary> /// <summary>
@ -882,7 +1374,7 @@ internal sealed class ObuWriter
} }
/// <summary> /// <summary>
/// Writes film-grain synthesis parameters for a displayed still-image frame. /// Writes film-grain synthesis parameters for a displayed frame.
/// </summary> /// </summary>
/// <param name="writer">The bit writer receiving the film-grain parameters.</param> /// <param name="writer">The bit writer receiving the film-grain parameters.</param>
/// <param name="sequenceHeader">The sequence header defining film-grain availability and color sampling.</param> /// <param name="sequenceHeader">The sequence header defining film-grain availability and color sampling.</param>

92
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs

@ -3,6 +3,7 @@
using System.Buffers; using System.Buffers;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
@ -39,40 +40,49 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
private const int TransformCoefficientOffset = ResidualStorageLength; private const int TransformCoefficientOffset = ResidualStorageLength;
private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount; private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount;
private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount; private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount;
private const int IntraBlockCopySampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength; private const int InterPredictionSampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength;
private const int IntraBlockCopySampleStorageLength = private const int InterPredictionSampleStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.SampleBufferCount * Av1EncoderInterPredictionWorkspace<ushort>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount * Av1EncoderInterPredictionWorkspace<ushort>.MaximumSampleCount *
sizeof(ushort) / sizeof(ushort) /
sizeof(int); sizeof(int);
private const int IntraBlockCopyResidualStorageOffset = private const int InterPredictionResidualStorageOffset =
IntraBlockCopySampleStorageOffset + IntraBlockCopySampleStorageLength; InterPredictionSampleStorageOffset + InterPredictionSampleStorageLength;
private const int IntraBlockCopyResidualStorageLength = private const int InterPredictionResidualStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount * Av1EncoderInterPredictionWorkspace<ushort>.MaximumSampleCount *
sizeof(short) / sizeof(short) /
sizeof(int); sizeof(int);
private const int IntraBlockCopyCoefficientStorageOffset = private const int InterPredictionScratchStorageOffset =
IntraBlockCopyResidualStorageOffset + IntraBlockCopyResidualStorageLength; InterPredictionResidualStorageOffset + InterPredictionResidualStorageLength;
private const int IntraBlockCopyCoefficientStorageLength = private const int InterPredictionScratchStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.CoefficientBufferCount * Av1EncoderInterPredictionWorkspace<ushort>.PredictionScratchCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount; sizeof(short) /
sizeof(int);
private const int InterPredictionCoefficientStorageOffset =
InterPredictionScratchStorageOffset + InterPredictionScratchStorageLength;
private const int IntraBlockCopyStorageLength = private const int InterPredictionCoefficientStorageLength =
IntraBlockCopySampleStorageLength + Av1EncoderInterPredictionWorkspace<ushort>.CoefficientBufferCount *
IntraBlockCopyResidualStorageLength + Av1EncoderInterPredictionWorkspace<ushort>.MaximumSampleCount;
IntraBlockCopyCoefficientStorageLength;
private const int InterPredictionStorageLength =
InterPredictionSampleStorageLength +
InterPredictionResidualStorageLength +
InterPredictionScratchStorageLength +
InterPredictionCoefficientStorageLength;
private const int ModeDecisionStorageLength = Av1EncoderModeDecisionWorkspace<ushort>.StorageLength; private const int ModeDecisionStorageLength = Av1EncoderModeDecisionWorkspace<ushort>.StorageLength;
private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > IntraBlockCopyStorageLength private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > InterPredictionStorageLength
? ModeDecisionStorageLength ? ModeDecisionStorageLength
: IntraBlockCopyStorageLength; : InterPredictionStorageLength;
private const int PartitionContextStorageOffset = private const int PartitionContextStorageOffset =
IntraBlockCopySampleStorageOffset + SharedModeDecisionStorageLength; InterPredictionSampleStorageOffset + SharedModeDecisionStorageLength;
private const int MaximumPartitionEdgeUnitCount = private const int MaximumPartitionEdgeUnitCount =
2 * (1 << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2)); 2 * (1 << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2));
@ -97,6 +107,11 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
/// </summary> /// </summary>
private readonly IMemoryOwner<int> owner; private readonly IMemoryOwner<int> owner;
/// <summary>
/// Reuses the fixed-capacity reference-vector stack for every inter block in the frame.
/// </summary>
private Av1ReferenceMotionVectors referenceMotionVectors;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderBlockWorkspace"/> class. /// Initializes a new instance of the <see cref="Av1EncoderBlockWorkspace"/> class.
/// </summary> /// </summary>
@ -128,6 +143,11 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
public Span<int> TransformWorkspace public Span<int> TransformWorkspace
=> this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength); => this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength);
/// <summary>
/// Gets the reusable reference-vector stack used by inter mode decision and syntax writing.
/// </summary>
public ref Av1ReferenceMotionVectors ReferenceMotionVectors => ref this.referenceMotionVectors;
/// <summary> /// <summary>
/// Gets the disjoint edge snapshot used to restore one square partition-search level. /// Gets the disjoint edge snapshot used to restore one square partition-search level.
/// </summary> /// </summary>
@ -152,43 +172,49 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
public Av1EncoderModeDecisionWorkspace<TSample> GetModeDecisionWorkspace<TSample>() public Av1EncoderModeDecisionWorkspace<TSample> GetModeDecisionWorkspace<TSample>()
where TSample : unmanaged where TSample : unmanaged
{ {
// Conventional intra search finishes before intra-block-copy search begins for the same block. // Conventional intra search finishes before reference prediction begins for the same block.
// Both phases can therefore reuse this aligned region without extending the owner or preserving stale scratch. // Both phases can therefore reuse this aligned region without extending the owner or preserving stale scratch.
Span<int> storage = this.owner.Memory.Span.Slice( Span<int> storage = this.owner.Memory.Span.Slice(
IntraBlockCopySampleStorageOffset, InterPredictionSampleStorageOffset,
SharedModeDecisionStorageLength); SharedModeDecisionStorageLength);
return new(storage[..Av1EncoderModeDecisionWorkspace<TSample>.StorageLength]); return new(storage[..Av1EncoderModeDecisionWorkspace<TSample>.StorageLength]);
} }
/// <summary> /// <summary>
/// Gets the reusable storage used while comparing intra-block-copy candidates. /// Gets the reusable storage used while comparing single-reference or intra-block-copy candidates.
/// </summary> /// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam> /// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
/// <returns>The typed intra-block-copy workspace.</returns> /// <returns>The typed inter-prediction workspace.</returns>
public Av1EncoderIntraBlockCopyWorkspace<TSample> GetIntraBlockCopyWorkspace<TSample>() public Av1EncoderInterPredictionWorkspace<TSample> GetInterPredictionWorkspace<TSample>()
where TSample : unmanaged where TSample : unmanaged
{ {
Span<int> storage = this.owner.Memory.Span; Span<int> storage = this.owner.Memory.Span;
Span<TSample> sampleStorage = MemoryMarshal Span<TSample> sampleStorage = MemoryMarshal
.Cast<int, TSample>(storage.Slice(IntraBlockCopySampleStorageOffset, IntraBlockCopySampleStorageLength)); .Cast<int, TSample>(storage.Slice(InterPredictionSampleStorageOffset, InterPredictionSampleStorageLength));
sampleStorage = sampleStorage[ sampleStorage = sampleStorage[
..(Av1EncoderIntraBlockCopyWorkspace<TSample>.SampleBufferCount * ..(Av1EncoderInterPredictionWorkspace<TSample>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount)]; Av1EncoderInterPredictionWorkspace<TSample>.MaximumSampleCount)];
Span<short> residualStorage = MemoryMarshal Span<short> residualStorage = MemoryMarshal
.Cast<int, short>(storage.Slice(IntraBlockCopyResidualStorageOffset, IntraBlockCopyResidualStorageLength)); .Cast<int, short>(storage.Slice(InterPredictionResidualStorageOffset, InterPredictionResidualStorageLength));
residualStorage = residualStorage[..Av1EncoderInterPredictionWorkspace<TSample>.MaximumSampleCount];
Span<short> predictionScratch = MemoryMarshal
.Cast<int, short>(storage.Slice(InterPredictionScratchStorageOffset, InterPredictionScratchStorageLength));
residualStorage = residualStorage[..Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount]; predictionScratch = predictionScratch[..Av1EncoderInterPredictionWorkspace<TSample>.PredictionScratchCount];
Span<int> coefficientStorage = storage.Slice( Span<int> coefficientStorage = storage.Slice(
IntraBlockCopyCoefficientStorageOffset, InterPredictionCoefficientStorageOffset,
IntraBlockCopyCoefficientStorageLength); InterPredictionCoefficientStorageLength);
return new Av1EncoderIntraBlockCopyWorkspace<TSample>( return new Av1EncoderInterPredictionWorkspace<TSample>(
sampleStorage, sampleStorage,
residualStorage, residualStorage,
predictionScratch,
coefficientStorage); coefficientStorage);
} }

38
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs

@ -345,6 +345,44 @@ internal readonly struct Av1EncoderFrame<TSample>
_ => this.chromaRed _ => this.chromaRed
}; };
/// <summary>
/// Gets a top-left view whose visible dimensions can be smaller than the backing coded planes.
/// </summary>
/// <param name="width">The visible luma width.</param>
/// <param name="height">The visible luma height.</param>
/// <returns>The requested non-owning planar view.</returns>
public PlanarView GetSubView(int width, int height)
{
Av1ColorFormat colorFormat = this.IsMonochrome
? Av1ColorFormat.Yuv400
: this.ChromaSubsamplingX == 0
? Av1ColorFormat.Yuv444
: this.ChromaSubsamplingY == 0
? Av1ColorFormat.Yuv422
: Av1ColorFormat.Yuv420;
int chromaWidth = (width + this.ChromaSubsamplingX) >> this.ChromaSubsamplingX;
int chromaHeight = (height + this.ChromaSubsamplingY) >> this.ChromaSubsamplingY;
Buffer2DRegion<TSample> blue = this.IsMonochrome
? default
: this.chromaBlue.GetSubRegion(0, 0, chromaWidth, chromaHeight);
Buffer2DRegion<TSample> red = this.IsMonochrome
? default
: this.chromaRed.GetSubRegion(0, 0, chromaWidth, chromaHeight);
return new PlanarView(
this.luma.GetSubRegion(0, 0, width, height),
blue,
red,
width,
height,
this.LumaBitDepth,
colorFormat,
this.ChromaPositionX,
this.ChromaPositionY);
}
/// <inheritdoc/> /// <inheritdoc/>
public Span<TSample> GetLumaRowSpan(int row) => this.luma.DangerousGetRowSpan(row); public Span<TSample> GetLumaRowSpan(int row) => this.luma.DangerousGetRowSpan(row);

7
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs

@ -14,6 +14,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary>
/// The byte boundary used by libaom for SIMD-accessible component planes.
/// </summary>
private const int PlaneAlignmentBytes = 32;
/// <summary> /// <summary>
/// The complete frame owner, or <see langword="null"/> after disposal. /// The complete frame owner, or <see langword="null"/> after disposal.
/// </summary> /// </summary>
@ -48,7 +53,7 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
: Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY); : Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY);
int chromaElementCount = checked(chromaSize.Width * chromaSize.Height); int chromaElementCount = checked(chromaSize.Width * chromaSize.Height);
int planeAlignment = Math.Max(32 / Unsafe.SizeOf<TSample>(), 1); int planeAlignment = Math.Max(PlaneAlignmentBytes / Unsafe.SizeOf<TSample>(), 1);
int chromaBlueOffset = Align(lumaElementCount, planeAlignment); int chromaBlueOffset = Align(lumaElementCount, planeAlignment);
int chromaRedOffset = Align(checked(chromaBlueOffset + chromaElementCount), planeAlignment); int chromaRedOffset = Align(checked(chromaBlueOffset + chromaElementCount), planeAlignment);
int storageLength = colorFormat == Av1ColorFormat.Yuv400 int storageLength = colorFormat == Av1ColorFormat.Yuv400

35
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs → src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderInterPredictionWorkspace.cs

@ -1,19 +1,33 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary> /// <summary>
/// Provides disjoint reusable buffers for intra-block-copy mode decisions. /// Provides disjoint reusable buffers for single-reference and intra-block-copy mode decisions.
/// </summary> /// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam> /// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace<TSample> internal readonly ref struct Av1EncoderInterPredictionWorkspace<TSample>
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary> /// <summary>
/// The number of samples in the fixed 8x8 intra-block-copy transform. /// The width and height of the fixed prediction block handled by the current inter search.
/// </summary>
private const int MaximumBlockDimension = 8;
/// <summary>
/// The number of samples in the fixed prediction block.
/// </summary>
public const int MaximumSampleCount = MaximumBlockDimension * MaximumBlockDimension;
/// <summary>
/// The signed intermediate capacity needed when both translational interpolation axes are filtered.
/// </summary> /// </summary>
public const int MaximumSampleCount = 8 * 8; public const int PredictionScratchCount =
Av1TranslationalInterPredictor.MinimumScratchStride *
(MaximumBlockDimension + Av1TranslationalInterPredictor.MaximumExtraRows);
/// <summary> /// <summary>
/// The number of sample buffers retained by one mode decision. /// The number of sample buffers retained by one mode decision.
@ -27,21 +41,25 @@ internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace<TSample>
private readonly Span<TSample> samples; private readonly Span<TSample> samples;
private readonly Span<short> residual; private readonly Span<short> residual;
private readonly Span<short> predictionScratch;
private readonly Span<int> coefficients; private readonly Span<int> coefficients;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderIntraBlockCopyWorkspace{TSample}"/> struct. /// Initializes a new instance of the <see cref="Av1EncoderInterPredictionWorkspace{TSample}"/> struct.
/// </summary> /// </summary>
/// <param name="samples">The sample storage.</param> /// <param name="samples">The sample storage.</param>
/// <param name="residual">The residual storage shared by sequential plane evaluations.</param> /// <param name="residual">The residual storage shared by sequential plane evaluations.</param>
/// <param name="predictionScratch">The intermediate storage used by two-dimensional interpolation.</param>
/// <param name="coefficients">The coefficient storage.</param> /// <param name="coefficients">The coefficient storage.</param>
public Av1EncoderIntraBlockCopyWorkspace( public Av1EncoderInterPredictionWorkspace(
Span<TSample> samples, Span<TSample> samples,
Span<short> residual, Span<short> residual,
Span<short> predictionScratch,
Span<int> coefficients) Span<int> coefficients)
{ {
this.samples = samples; this.samples = samples;
this.residual = residual; this.residual = residual;
this.predictionScratch = predictionScratch;
this.coefficients = coefficients; this.coefficients = coefficients;
} }
@ -100,6 +118,11 @@ internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace<TSample>
/// </summary> /// </summary>
public Span<short> Residual => this.residual; public Span<short> Residual => this.residual;
/// <summary>
/// Gets the intermediate scratch used when both translational interpolation axes are filtered.
/// </summary>
public Span<short> PredictionScratch => this.predictionScratch;
/// <summary> /// <summary>
/// Gets the selected luma coefficients. /// Gets the selected luma coefficients.
/// </summary> /// </summary>

1700
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs

File diff suppressed because it is too large

586
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs

@ -1,586 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <content>
/// Provides full rate-distortion selection for intra-block-copy candidates.
/// </content>
internal static partial class Av1IntraSuperblockEncoder
{
internal partial struct ModeDecision<TSample, TOperator>
where TSample : unmanaged
where TOperator : struct, IBlockEncodingOperator<TSample>
{
private long SelectIntraBlockCopy(
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,
Point blockOrigin,
ushort tileIndex,
long regularCost,
ref Av1MacroBlockModeInfo modeInfo,
ref Av1EncoderBlockStruct block,
ref Av1EncoderPaletteInfo paletteInfo)
{
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8;
Buffer2DRegion<TSample> lumaSource = this.source.GetPlane(Av1Plane.Y);
Buffer2DRegion<TSample> lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y);
Point modeInfoPosition = new(
blockOrigin.X >> Av1Constants.ModeInfoSizeLog2,
blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2);
Span<Av1MotionVector> referenceCandidates = stackalloc Av1MotionVector[8];
Span<int> referenceWeights = stackalloc int[8];
Av1MotionVector reference = Av1IntraBlockCopy.FindReference(
this.picture,
macroBlock,
modeInfoPosition,
BlockSize,
Av1PartitionType.None,
referenceCandidates,
referenceWeights);
Span<Av1MotionVector> candidates = stackalloc Av1MotionVector[4];
Av1IntraBlockCopySearchIndex search = this.picture.IntraBlockCopySearch;
int candidateCount = search.FindCandidates<TSample, TOperator>(
lumaSource,
lumaReconstruction,
blockOrigin,
macroBlock.Tile,
this.picture.Sequence.SequenceHeader,
writer,
reference,
this.rateMultiplier,
candidates);
candidateCount += search.FindPixelCandidates<TSample, TOperator>(
lumaSource,
lumaReconstruction,
blockOrigin,
macroBlock.Tile,
this.picture.Sequence.SequenceHeader,
writer,
reference,
this.quantization.QIndex[0],
this.rateMultiplier,
candidates[candidateCount..]);
// Hash and full-pixel searches can converge on the same vector. Preserve the first search-order
// occurrence so repeated vectors do not pay for duplicate transform searches or alter ties.
int uniqueCandidateCount = 0;
for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++)
{
Av1MotionVector candidate = candidates[candidateIndex];
bool duplicate = false;
for (int uniqueIndex = 0; uniqueIndex < uniqueCandidateCount; uniqueIndex++)
{
if (candidate == candidates[uniqueIndex])
{
duplicate = true;
break;
}
}
if (!duplicate)
{
candidates[uniqueCandidateCount++] = candidate;
}
}
if (uniqueCandidateCount == 0)
{
return regularCost;
}
int skipContext = Av1TileWriter.GetSkipContext(macroBlock);
long bestCost = regularCost;
bool hasSelectedCandidate = false;
bool selectedSkip = false;
Av1MotionVector selectedVector = default;
Av1EncoderTransformBlockState selectedLumaState = default;
Av1EncoderTransformBlockState selectedBlueState = default;
Av1EncoderTransformBlockState selectedRedState = default;
Av1EncoderIntraBlockCopyWorkspace<TSample> workspace =
this.blockWorkspace.GetIntraBlockCopyWorkspace<TSample>();
Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Luminance,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
blockOrigin,
BlockSize,
LumaTransformSize);
// A coded IBC residual uses the unsplit transform root at this fixed block size. A skipped block
// omits both the transform-partition bit and coefficient syntax, so this rate is added only below.
int transformPartitionRate = 0;
if (this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select)
{
Av1NeighborArrayUnit<byte> transformContexts = this.picture.TransformFunctionContexts[tileIndex];
int topIndex = transformContexts.GetTopIndex(blockOrigin);
int leftIndex = transformContexts.GetLeftIndex(blockOrigin);
int transformPartitionContext = Av1SymbolContextHelper.GetTransformPartitionContext(
transformContexts.Top[topIndex],
transformContexts.Left[leftIndex],
BlockSize,
LumaTransformSize);
transformPartitionRate = writer.GetTransformPartitionCost(
false,
transformPartitionContext);
}
ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig;
int subsamplingX = colorConfig.SubSamplingX ? 1 : 0;
int subsamplingY = colorConfig.SubSamplingY ? 1 : 0;
Point chromaOrigin = new(blockOrigin.X >> subsamplingX, blockOrigin.Y >> subsamplingY);
Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize(
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
Av1TransformBlockContext blueContext = default;
Av1TransformBlockContext redContext = default;
if (!this.source.IsMonochrome)
{
Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled(
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
blueContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Chroma,
this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex],
chromaOrigin,
chromaBlockSize,
chromaTransformSize);
redContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Chroma,
this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex],
chromaOrigin,
chromaBlockSize,
chromaTransformSize);
}
// Per-vector plane results reuse candidate scratch. Separate selected spans retain only a new
// global winner, allowing the complete search to finish before committed reconstruction changes.
for (int candidateIndex = 0; candidateIndex < uniqueCandidateCount; candidateIndex++)
{
Av1MotionVector candidate = candidates[candidateIndex];
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.Y,
Av1ComponentType.Luminance,
blockOrigin,
0,
0,
LumaTransformSize,
Av1TransformType.AllTransformTypes,
lumaContext,
workspace.LumaPrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.LumaCandidateReconstruction,
workspace.LumaCandidateCoefficients,
out Av1EncoderTransformBlockState lumaCandidateState,
out int lumaRate,
out long lumaDistortion,
out bool hasEmptyLuma,
out Av1EncoderTransformBlockState emptyLumaState,
out long emptyLumaDistortion);
int blueRate = 0;
int redRate = 0;
long blueDistortion = 0;
long redDistortion = 0;
long emptyBlueDistortion = 0;
long emptyRedDistortion = 0;
bool hasEmptyBlue = true;
bool hasEmptyRed = true;
Av1EncoderTransformBlockState blueCandidateState = default;
Av1EncoderTransformBlockState redCandidateState = default;
Av1EncoderTransformBlockState emptyBlueState = default;
Av1EncoderTransformBlockState emptyRedState = default;
if (!this.source.IsMonochrome)
{
Av1TransformType chromaTransformType = lumaCandidateState.TransformType;
Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType(
chromaTransformSize,
isInter: true,
this.picture.Parent.FrameHeader.UseReducedTransformSet);
// Inter prediction does not signal an independent chroma transform type. Chroma reuses the
// selected luma type when that type belongs to its transform set and otherwise falls back to DCT.
if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet))
{
chromaTransformType = Av1TransformType.DctDct;
}
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.U,
Av1ComponentType.Chroma,
blockOrigin,
subsamplingX,
subsamplingY,
chromaTransformSize,
chromaTransformType,
blueContext,
workspace.BluePrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.BlueCandidateReconstruction,
workspace.BlueCandidateCoefficients,
out blueCandidateState,
out blueRate,
out blueDistortion,
out hasEmptyBlue,
out emptyBlueState,
out emptyBlueDistortion);
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.V,
Av1ComponentType.Chroma,
blockOrigin,
subsamplingX,
subsamplingY,
chromaTransformSize,
chromaTransformType,
redContext,
workspace.RedPrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.RedCandidateReconstruction,
workspace.RedCandidateCoefficients,
out redCandidateState,
out redRate,
out redDistortion,
out hasEmptyRed,
out emptyRedState,
out emptyRedDistortion);
}
int displacementRate = writer.GetDisplacementVectorCost(candidate, reference);
int candidateRate = writer.GetUseIntraBlockCopyCost(true) +
displacementRate +
writer.GetSkipCost(false, skipContext) +
transformPartitionRate +
lumaRate +
blueRate +
redRate;
long candidateDistortion = lumaDistortion + blueDistortion + redDistortion;
long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, candidateDistortion);
bool candidateSkip = false;
// The skip alternative is available only when every coded plane has an empty transform. Its
// distortion comes from prediction alone and its rate excludes the transform tree and coefficients.
if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed)
{
int skipRate = writer.GetUseIntraBlockCopyCost(true) +
displacementRate +
writer.GetSkipCost(true, skipContext);
long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion;
long skipCost = Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion);
if (skipCost < candidateCost)
{
candidateCost = skipCost;
candidateSkip = true;
}
}
// Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD.
if (candidateCost >= bestCost)
{
continue;
}
bestCost = candidateCost;
hasSelectedCandidate = true;
selectedSkip = candidateSkip;
selectedVector = candidate;
if (candidateSkip)
{
workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction);
workspace.SelectedLumaCoefficients.Clear();
selectedLumaState = emptyLumaState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
workspace.BluePrediction[..chromaSampleCount].CopyTo(workspace.SelectedBlueReconstruction);
workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction);
workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear();
workspace.SelectedRedCoefficients[..chromaSampleCount].Clear();
selectedBlueState = emptyBlueState;
selectedRedState = emptyRedState;
}
}
else
{
workspace.LumaCandidateReconstruction.CopyTo(workspace.SelectedLumaReconstruction);
workspace.LumaCandidateCoefficients.CopyTo(workspace.SelectedLumaCoefficients);
selectedLumaState = lumaCandidateState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
workspace.BlueCandidateReconstruction[..chromaSampleCount]
.CopyTo(workspace.SelectedBlueReconstruction);
workspace.RedCandidateReconstruction[..chromaSampleCount]
.CopyTo(workspace.SelectedRedReconstruction);
workspace.BlueCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedBlueCoefficients);
workspace.RedCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedRedCoefficients);
selectedBlueState = blueCandidateState;
selectedRedState = redCandidateState;
}
}
}
if (!hasSelectedCandidate)
{
return bestCost;
}
// Only the winning vector is now visible to later coding blocks. This single publication keeps
// rejected motion vectors from contaminating intra references or entropy contexts.
Span<int> retainedLumaCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.Y);
Span<Av1EncoderTransformBlockState> retainedLumaTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.Y);
int lumaTransformIndex = this.codedAreaLuma /
Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex];
CopyCandidate(
workspace.SelectedLumaReconstruction,
workspace.SelectedLumaCoefficients,
lumaReconstruction,
blockOrigin,
retainedLumaCoefficients[this.codedAreaLuma..],
LumaTransformSize,
selectedLumaState,
ref retainedLumaState);
if (!this.source.IsMonochrome)
{
Span<int> retainedBlueCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.U);
Span<int> retainedRedCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.V);
Span<Av1EncoderTransformBlockState> retainedBlueTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.U);
Span<Av1EncoderTransformBlockState> retainedRedTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.V);
int chromaTransformIndex = this.codedAreaChroma /
Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex];
ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex];
CopyCandidate(
workspace.SelectedBlueReconstruction,
workspace.SelectedBlueCoefficients,
this.reconstruction.GetPlane(Av1Plane.U),
chromaOrigin,
retainedBlueCoefficients[this.codedAreaChroma..],
chromaTransformSize,
selectedBlueState,
ref retainedBlueState);
CopyCandidate(
workspace.SelectedRedReconstruction,
workspace.SelectedRedCoefficients,
this.reconstruction.GetPlane(Av1Plane.V),
chromaOrigin,
retainedRedCoefficients[this.codedAreaChroma..],
chromaTransformSize,
selectedRedState,
ref retainedRedState);
}
modeInfo.Block.Mode = Av1PredictionMode.DC;
modeInfo.Block.UvMode = Av1ChromaPredictionMode.DC;
modeInfo.Block.TransformSize = LumaTransformSize;
modeInfo.Block.Skip = selectedSkip;
modeInfo.Block.UseIntraBlockCopy = true;
block.FilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 0;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 0;
block.PredictionUnit.ChromaFromLumaIndex = 0;
block.PredictionUnit.ChromaFromLumaSigns = 0;
paletteInfo = default;
this.picture.SetDisplacementVector(modeInfoPosition, selectedVector);
return bestCost;
}
private void EvaluateIntraBlockCopyPlane(
Av1SymbolEncoder writer,
Av1MotionVector vector,
Av1Plane plane,
Av1ComponentType componentType,
Point lumaOrigin,
int subsamplingX,
int subsamplingY,
Av1TransformSize transformSize,
Av1TransformType transformTypeSelection,
Av1TransformBlockContext blockContext,
Span<TSample> prediction,
Span<short> residual,
Span<TSample> transformReconstruction,
Span<int> transformCoefficients,
Span<TSample> selectedReconstruction,
Span<int> selectedCoefficients,
out Av1EncoderTransformBlockState selectedState,
out int selectedRate,
out long selectedDistortion,
out bool hasEmptyTransform,
out Av1EncoderTransformBlockState emptyState,
out long emptyDistortion)
{
Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY);
int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX));
int sourceRowQ4 = (planeOrigin.Y << 4) + (vector.Row << (1 - subsamplingY));
Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4);
int sampleCount = transformSize.GetSize2d();
Buffer2DRegion<TSample> sourcePlane = this.source.GetPlane(plane);
Buffer2DRegion<TSample> reconstructionPlane = this.reconstruction.GetPlane(plane);
TOperator.PrepareIntraBlockCopy(
sourcePlane,
planeOrigin,
reconstructionPlane,
predictionOrigin,
(sourceColumnQ4 & 15) != 0,
(sourceRowQ4 & 15) != 0,
prediction[..sampleCount],
residual[..sampleCount],
transformSize);
// Motion compensation and subtraction do not depend on transform type. Keep them outside the
// transform loop so exhaustive luma search traverses the source and reference blocks only once.
Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType(
transformSize,
isInter: true,
this.picture.Parent.FrameHeader.UseReducedTransformSet);
Av1TransformType firstTransformType = transformTypeSelection == Av1TransformType.AllTransformTypes
? Av1TransformType.DctDct
: transformTypeSelection;
Av1TransformType transformTypeLimit = transformTypeSelection == Av1TransformType.AllTransformTypes
? Av1TransformType.AllTransformTypes
: (Av1TransformType)((int)transformTypeSelection + 1);
long bestCost = long.MaxValue;
selectedState = default;
selectedRate = 0;
selectedDistortion = 0;
hasEmptyTransform = false;
emptyState = default;
emptyDistortion = 0;
// The candidate and best spans alternate ownership whenever a transform improves the result.
// This mirrors the reference's buffer-pointer swap and replaces a copy on every improvement
// with at most one normalization copy after the transform search.
Span<TSample> candidateReconstruction = transformReconstruction[..sampleCount];
Span<int> candidateCoefficients = transformCoefficients[..sampleCount];
Span<TSample> bestReconstruction = selectedReconstruction[..sampleCount];
Span<int> bestCoefficients = selectedCoefficients[..sampleCount];
bool bestUsesSelectedStorage = true;
for (Av1TransformType transformType = firstTransformType;
transformType < transformTypeLimit;
transformType++)
{
if (!transformType.IsExtendedSetUsed(transformSetType))
{
continue;
}
Av1EncoderTransformBlockState candidateState = default;
long candidateDistortion = TOperator.EncodePredictionCandidate(
this.blockWorkspace,
sourcePlane,
planeOrigin,
prediction[..sampleCount],
residual[..sampleCount],
candidateReconstruction,
transformSize.GetWidth(),
candidateCoefficients,
transformSize,
transformType,
plane,
this.quantization.QIndex[0],
this.quantization.DeltaQDc[(int)plane],
this.quantization.DeltaQAc[(int)plane],
this.bitDepth,
ref candidateState);
int candidateRate = writer.GetCoefficientCost(
transformSize,
transformType,
Av1PredictionMode.DC,
candidateCoefficients,
componentType,
blockContext,
candidateState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: true);
long candidateCost = Av1RateDistortion.GetCost(
this.rateMultiplier,
candidateRate,
candidateDistortion);
if (candidateCost < bestCost)
{
Span<TSample> previousBestReconstruction = bestReconstruction;
bestReconstruction = candidateReconstruction;
candidateReconstruction = previousBestReconstruction;
Span<int> previousBestCoefficients = bestCoefficients;
bestCoefficients = candidateCoefficients;
candidateCoefficients = previousBestCoefficients;
bestUsesSelectedStorage = !bestUsesSelectedStorage;
bestCost = candidateCost;
selectedState = candidateState;
selectedRate = candidateRate;
selectedDistortion = candidateDistortion;
}
if (candidateState.EndOfBlock == 0 &&
(!hasEmptyTransform || candidateDistortion < emptyDistortion))
{
hasEmptyTransform = true;
emptyState = candidateState;
emptyDistortion = candidateDistortion;
}
}
// Callers retain the designated selected spans after this scratch workspace is reused by the
// next plane or motion vector, so normalize only when the final best result occupies scratch.
if (!bestUsesSelectedStorage)
{
bestReconstruction.CopyTo(selectedReconstruction);
bestCoefficients.CopyTo(selectedCoefficients);
}
}
}
}

98
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs

@ -127,6 +127,7 @@ internal static partial class Av1IntraSuperblockEncoder
where TOperator : struct, IBlockEncodingOperator<TSample> where TOperator : struct, IBlockEncodingOperator<TSample>
{ {
private readonly Av1EncoderFrame<TSample>.PlanarView source; private readonly Av1EncoderFrame<TSample>.PlanarView source;
private readonly Av1EncoderFrame<TSample>.PlanarView reference;
private readonly Av1EncoderFrame<TSample>.PlanarView reconstruction; private readonly Av1EncoderFrame<TSample>.PlanarView reconstruction;
private readonly Av1PictureControlSet picture; private readonly Av1PictureControlSet picture;
private readonly Av1Superblock superblock; private readonly Av1Superblock superblock;
@ -144,6 +145,7 @@ internal static partial class Av1IntraSuperblockEncoder
/// Initializes a new instance of the <see cref="ModeDecision{TSample, TOperator}"/> struct. /// Initializes a new instance of the <see cref="ModeDecision{TSample, TOperator}"/> struct.
/// </summary> /// </summary>
/// <param name="source">The coded source frame.</param> /// <param name="source">The coded source frame.</param>
/// <param name="reference">The reconstructed inter reference, or the current reconstruction for an intra frame.</param>
/// <param name="reconstruction">The reconstructed frame updated by winning candidates.</param> /// <param name="reconstruction">The reconstructed frame updated by winning candidates.</param>
/// <param name="picture">The frame coding and mode-information state.</param> /// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="superblock">The current superblock.</param> /// <param name="superblock">The current superblock.</param>
@ -152,6 +154,7 @@ internal static partial class Av1IntraSuperblockEncoder
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param> /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public ModeDecision( public ModeDecision(
Av1EncoderFrame<TSample> source, Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reference,
Av1EncoderFrame<TSample> reconstruction, Av1EncoderFrame<TSample> reconstruction,
Av1PictureControlSet picture, Av1PictureControlSet picture,
Av1Superblock superblock, Av1Superblock superblock,
@ -160,6 +163,7 @@ internal static partial class Av1IntraSuperblockEncoder
int effort) int effort)
{ {
this.source = source.CodedView; this.source = source.CodedView;
this.reference = reference.CodedView;
this.reconstruction = reconstruction.CodedView; this.reconstruction = reconstruction.CodedView;
this.picture = picture; this.picture = picture;
this.superblock = superblock; this.superblock = superblock;
@ -167,7 +171,10 @@ internal static partial class Av1IntraSuperblockEncoder
this.blockWorkspace = blockWorkspace; this.blockWorkspace = blockWorkspace;
this.quantization = picture.Parent.FrameHeader.QuantizationParameters; this.quantization = picture.Parent.FrameHeader.QuantizationParameters;
this.bitDepth = picture.Sequence.SequenceHeader.ColorConfig.BitDepth; this.bitDepth = picture.Sequence.SequenceHeader.ColorConfig.BitDepth;
this.rateMultiplier = Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth); this.rateMultiplier = picture.Parent.FrameHeader.IsIntra
? Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth)
: Av1RateDistortion.GetInterFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth);
this.effort = effort; this.effort = effort;
this.codedAreaLuma = 0; this.codedAreaLuma = 0;
this.codedAreaChroma = 0; this.codedAreaChroma = 0;
@ -183,6 +190,13 @@ internal static partial class Av1IntraSuperblockEncoder
Av1BlockSize blockSize, Av1BlockSize blockSize,
Av1PartitionType preparedPartition) Av1PartitionType preparedPartition)
{ {
if (!this.picture.Parent.FrameHeader.IsIntra)
{
// The first inter implementation retains the prepared 8x8 tree so every prediction and residual
// transform fits the single reusable block workspace while larger inter partitions remain unsearched.
return preparedPartition;
}
bool searchPartition = blockSize is Av1BlockSize.Block8x8 or Av1BlockSize.Block16x16 || bool searchPartition = blockSize is Av1BlockSize.Block8x8 or Av1BlockSize.Block16x16 ||
(this.effort == 10 && (this.effort == 10 &&
blockSize is Av1BlockSize.Block32x32 or Av1BlockSize.Block64x64 or Av1BlockSize.Block128x128); blockSize is Av1BlockSize.Block32x32 or Av1BlockSize.Block64x64 or Av1BlockSize.Block128x128);
@ -640,8 +654,9 @@ internal static partial class Av1IntraSuperblockEncoder
modeInfo.Block.Skip, modeInfo.Block.Skip,
allowIntraBlockCopy); allowIntraBlockCopy);
this.selectedBlockCost = allowIntraBlockCopy if (!this.picture.Parent.FrameHeader.IsIntra)
? this.SelectIntraBlockCopy( {
this.selectedBlockCost = this.SelectInterPrediction(
writer, writer,
macroBlock, macroBlock,
blockOrigin, blockOrigin,
@ -649,8 +664,22 @@ internal static partial class Av1IntraSuperblockEncoder
regularCost, regularCost,
ref modeInfo, ref modeInfo,
ref block, ref block,
ref paletteInfo) ref paletteInfo);
: regularCost; }
else
{
this.selectedBlockCost = allowIntraBlockCopy
? this.SelectIntraBlockCopy(
writer,
macroBlock,
blockOrigin,
tileIndex,
regularCost,
ref modeInfo,
ref block,
ref paletteInfo)
: regularCost;
}
this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight(); this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight();
return; return;
@ -787,8 +816,9 @@ internal static partial class Av1IntraSuperblockEncoder
modeInfo.Block.Skip, modeInfo.Block.Skip,
allowColorIntraBlockCopy); allowColorIntraBlockCopy);
this.selectedBlockCost = allowColorIntraBlockCopy if (!this.picture.Parent.FrameHeader.IsIntra)
? this.SelectIntraBlockCopy( {
this.selectedBlockCost = this.SelectInterPrediction(
writer, writer,
macroBlock, macroBlock,
blockOrigin, blockOrigin,
@ -796,8 +826,22 @@ internal static partial class Av1IntraSuperblockEncoder
regularColorCost, regularColorCost,
ref modeInfo, ref modeInfo,
ref block, ref block,
ref paletteInfo) ref paletteInfo);
: regularColorCost; }
else
{
this.selectedBlockCost = allowColorIntraBlockCopy
? this.SelectIntraBlockCopy(
writer,
macroBlock,
blockOrigin,
tileIndex,
regularColorCost,
ref modeInfo,
ref block,
ref paletteInfo)
: regularColorCost;
}
this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight(); this.codedAreaLuma += blockSize.GetWidth() * blockSize.GetHeight();
if (block.HasChroma) if (block.HasChroma)
@ -1237,6 +1281,12 @@ internal static partial class Av1IntraSuperblockEncoder
bool allowIntraBlockCopy) bool allowIntraBlockCopy)
{ {
int rateAdjustment = writer.GetSkipCost(skip, Av1TileWriter.GetSkipContext(macroBlock)); int rateAdjustment = writer.GetSkipCost(skip, Av1TileWriter.GetSkipContext(macroBlock));
if (!this.picture.Parent.FrameHeader.IsIntra)
{
int intraInterContext = Av1TileWriter.GetIntraInterContext(macroBlock);
rateAdjustment += writer.GetIsInterCost(false, intraInterContext);
}
if (allowIntraBlockCopy) if (allowIntraBlockCopy)
{ {
rateAdjustment += writer.GetUseIntraBlockCopyCost(false); rateAdjustment += writer.GetUseIntraBlockCopyCost(false);
@ -2082,7 +2132,14 @@ internal static partial class Av1IntraSuperblockEncoder
leftContexts, leftContexts,
out int coefficientRate); out int coefficientRate);
int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, blockSize, mode, angleDelta); int rate = Av1TileWriter.GetLumaModeCost(
writer,
macroBlock,
blockSize,
mode,
angleDelta,
this.picture.Parent.FrameHeader.IsIntra);
rate += transformSizeRate + coefficientRate; rate += transformSizeRate + coefficientRate;
if (mode == Av1PredictionMode.DC) if (mode == Av1PredictionMode.DC)
{ {
@ -2193,7 +2250,14 @@ internal static partial class Av1IntraSuperblockEncoder
} }
else else
{ {
rate += Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta); rate += Av1TileWriter.GetLumaModeCost(
writer,
macroBlock,
BlockSize,
mode,
angleDelta,
this.picture.Parent.FrameHeader.IsIntra);
if (mode == Av1PredictionMode.DC) if (mode == Av1PredictionMode.DC)
{ {
rate += paletteDisabledCost; rate += paletteDisabledCost;
@ -2643,7 +2707,14 @@ internal static partial class Av1IntraSuperblockEncoder
// Charge every block-level choice that distinguishes this spatial candidate before adding // Charge every block-level choice that distinguishes this spatial candidate before adding
// coefficient syntax derived from the live neighboring-transform context. // coefficient syntax derived from the live neighboring-transform context.
int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, blockSize, mode, angleDelta); int rate = Av1TileWriter.GetLumaModeCost(
writer,
macroBlock,
blockSize,
mode,
angleDelta,
this.picture.Parent.FrameHeader.IsIntra);
rate += transformSizeRate; rate += transformSizeRate;
if (mode == Av1PredictionMode.DC) if (mode == Av1PredictionMode.DC)
{ {
@ -2714,7 +2785,8 @@ internal static partial class Av1IntraSuperblockEncoder
macroBlock, macroBlock,
blockSize, blockSize,
Av1PredictionMode.DC, Av1PredictionMode.DC,
0); 0,
this.picture.Parent.FrameHeader.IsIntra);
rate += transformSizeRate; rate += transformSizeRate;
rate += paletteDisabledCost; rate += paletteDisabledCost;

570
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs

@ -6,6 +6,7 @@ using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
@ -18,6 +19,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// </content> /// </content>
internal static partial class Av1IntraSuperblockEncoder internal static partial class Av1IntraSuperblockEncoder
{ {
/// <summary>
/// The width and height of the fixed block currently used by inter motion search.
/// </summary>
private const int InterSearchBlockDimension = 8;
/// <summary> /// <summary>
/// Defines type-specific block encoding without coupling traversal to sample storage width. /// Defines type-specific block encoding without coupling traversal to sample storage width.
/// </summary> /// </summary>
@ -259,7 +265,7 @@ internal static partial class Av1IntraSuperblockEncoder
/// <param name="prediction">The contiguous prediction destination.</param> /// <param name="prediction">The contiguous prediction destination.</param>
/// <param name="residual">The contiguous source-minus-prediction destination.</param> /// <param name="residual">The contiguous source-minus-prediction destination.</param>
/// <param name="transformSize">The prediction dimensions.</param> /// <param name="transformSize">The prediction dimensions.</param>
public static abstract void PrepareIntraBlockCopy( public static abstract void PrepareIntraBlockCopyPrediction(
Buffer2DRegion<TSample> source, Buffer2DRegion<TSample> source,
Point blockOrigin, Point blockOrigin,
Buffer2DRegion<TSample> reconstruction, Buffer2DRegion<TSample> reconstruction,
@ -270,6 +276,68 @@ internal static partial class Av1IntraSuperblockEncoder
Span<short> residual, Span<short> residual,
Av1TransformSize transformSize); Av1TransformSize transformSize);
/// <summary>
/// Subtracts a retained prediction from its source without rebuilding the inter predictor.
/// </summary>
/// <param name="source">The source plane.</param>
/// <param name="blockOrigin">The block origin in plane samples.</param>
/// <param name="prediction">The tightly packed prediction samples.</param>
/// <param name="residual">The destination signed residual samples.</param>
/// <param name="transformSize">The plane block geometry.</param>
public static abstract void SubtractPrediction(
Buffer2DRegion<TSample> source,
Point blockOrigin,
ReadOnlySpan<TSample> prediction,
Span<short> residual,
Av1TransformSize transformSize);
/// <summary>
/// Builds a translational prediction from a retained reference frame and the matching source residual.
/// </summary>
/// <param name="source">The coded source plane.</param>
/// <param name="blockOrigin">The destination block origin in plane samples.</param>
/// <param name="reference">The padded retained reference plane.</param>
/// <param name="predictionOrigin">The integer reference origin preceding the subpixel phase.</param>
/// <param name="horizontalFilter">The horizontal interpolation filter.</param>
/// <param name="verticalFilter">The vertical interpolation filter.</param>
/// <param name="horizontalPhase">The horizontal phase in one-sixteenth-sample units.</param>
/// <param name="verticalPhase">The vertical phase in one-sixteenth-sample units.</param>
/// <param name="prediction">The contiguous prediction destination.</param>
/// <param name="residual">The contiguous source-minus-prediction destination.</param>
/// <param name="predictionScratch">The intermediate storage used by two-dimensional filtering.</param>
/// <param name="transformSize">The prediction dimensions.</param>
/// <param name="bitDepth">The coded sample bit depth.</param>
public static abstract void PrepareTranslationalInterPrediction(
Buffer2DRegion<TSample> source,
Point blockOrigin,
Buffer2DRegion<TSample> reference,
Point predictionOrigin,
Av1InterpolationFilter horizontalFilter,
Av1InterpolationFilter verticalFilter,
int horizontalPhase,
int verticalPhase,
Span<TSample> prediction,
Span<short> residual,
Span<short> predictionScratch,
Av1TransformSize transformSize,
Av1BitDepth bitDepth);
/// <summary>
/// Measures an 8x8 full-pixel reference candidate through the bordered plane storage.
/// </summary>
/// <param name="source">The coded source plane.</param>
/// <param name="sourceOrigin">The source block origin in visible-plane coordinates.</param>
/// <param name="reference">The padded retained reference plane.</param>
/// <param name="predictionOrigin">The candidate origin, which may lie inside the physical border.</param>
/// <param name="bitDepth">The coded sample precision.</param>
/// <returns>The squared error normalized to the eight-bit distortion domain.</returns>
public static abstract long GetInterPredictionError(
Buffer2DRegion<TSample> source,
Point sourceOrigin,
Buffer2DRegion<TSample> reference,
Point predictionOrigin,
Av1BitDepth bitDepth);
/// <summary> /// <summary>
/// Encodes one prepared prediction with the selected transform into decision scratch. /// Encodes one prepared prediction with the selected transform into decision scratch.
/// </summary> /// </summary>
@ -384,9 +452,8 @@ internal static partial class Av1IntraSuperblockEncoder
ReadOnlySpan<byte> firstRow = plane.DangerousGetRowSpan(first.Y + row)[first.X..]; ReadOnlySpan<byte> firstRow = plane.DangerousGetRowSpan(first.Y + row)[first.X..];
ReadOnlySpan<byte> secondRow = plane.DangerousGetRowSpan(second.Y + row)[second.X..]; ReadOnlySpan<byte> secondRow = plane.DangerousGetRowSpan(second.Y + row)[second.X..];
// An 8x8 search row occupies one machine word, so one unaligned load and comparison replaces // Compare the complete row as byte lanes so collision rejection remains independent of native endianness.
// eight dependent scalar branches while retaining exact collision rejection. if (Vector64.Create(firstRow) != Vector64.Create(secondRow))
if (MemoryMarshal.Read<ulong>(firstRow) != MemoryMarshal.Read<ulong>(secondRow))
{ {
return false; return false;
} }
@ -395,49 +462,47 @@ internal static partial class Av1IntraSuperblockEncoder
return true; return true;
} }
/// <inheritdoc/>
public static long GetInterPredictionError(
Buffer2DRegion<byte> source,
Point sourceOrigin,
Buffer2DRegion<byte> reference,
Point predictionOrigin,
Av1BitDepth bitDepth)
{
Rectangle sourceBounds = source.Bounds;
Rectangle referenceBounds = reference.Bounds;
int sourceIndex =
((sourceBounds.Y + sourceOrigin.Y) * source.Stride) +
sourceBounds.X +
sourceOrigin.X;
int referenceIndex =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
referenceBounds.X +
predictionOrigin.X;
// The shared residual kernel selects the widest available vector width and handles the scalar tail.
return Av1ResidualBuilder.SumSquaredError(
source.Buffer.DangerousGetSingleSpan()[sourceIndex..],
source.Stride,
reference.Buffer.DangerousGetSingleSpan()[referenceIndex..],
reference.Stride,
InterSearchBlockDimension,
InterSearchBlockDimension);
}
/// <inheritdoc/> /// <inheritdoc/>
public static int GetSumOfAbsoluteDifferences( public static int GetSumOfAbsoluteDifferences(
Buffer2DRegion<byte> source, Buffer2DRegion<byte> source,
Point sourceOrigin, Point sourceOrigin,
Buffer2DRegion<byte> reconstruction, Buffer2DRegion<byte> reconstruction,
Point predictionOrigin) Point predictionOrigin)
{ => Av1ResidualBuilder.SumAbsoluteDifferences8x8(
int sum = 0; Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
if (Vector128.IsHardwareAccelerated) source.Stride,
{ Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin),
for (int row = 0; row < 8; row++) reconstruction.Stride);
{
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
Vector128<short> difference =
(Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(sourceRow)).AsByte()) -
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow)).AsByte()))
.AsInt16();
// Widened signed differences retain both subtraction directions; absolute values then reduce
// the complete eight-sample row without scalar extraction or per-sample branches.
sum += Vector128.Sum(Vector128.Abs(difference));
}
}
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
sum += Math.Abs(sourceRow[column] - predictionRow[column]);
}
}
}
return sum;
}
/// <inheritdoc/> /// <inheritdoc/>
public static void GetFourSumsOfAbsoluteDifferences( public static void GetFourSumsOfAbsoluteDifferences(
@ -446,66 +511,12 @@ internal static partial class Av1IntraSuperblockEncoder
Buffer2DRegion<byte> reconstruction, Buffer2DRegion<byte> reconstruction,
Point firstPredictionOrigin, Point firstPredictionOrigin,
Span<int> sums) Span<int> sums)
{ => Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(
int sum0 = 0; Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
int sum1 = 0; source.Stride,
int sum2 = 0; Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, firstPredictionOrigin),
int sum3 = 0; reconstruction.Stride,
if (Vector128.IsHardwareAccelerated) sums);
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..];
// The four candidates reuse one widened source vector; only their overlapping predictor
// windows are loaded separately before the packed absolute-difference reductions.
Vector128<short> sourceSamples =
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(sourceRow)).AsByte()).AsInt16();
Vector128<short> prediction0 =
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow)).AsByte()).AsInt16();
Vector128<short> prediction1 =
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow[1..])).AsByte()).AsInt16();
Vector128<short> prediction2 =
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow[2..])).AsByte()).AsInt16();
Vector128<short> prediction3 =
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow[3..])).AsByte()).AsInt16();
sum0 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction0));
sum1 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction1));
sum2 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction2));
sum3 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction3));
}
}
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
int sourceSample = sourceRow[column];
sum0 += Math.Abs(sourceSample - predictionRow[column]);
sum1 += Math.Abs(sourceSample - predictionRow[column + 1]);
sum2 += Math.Abs(sourceSample - predictionRow[column + 2]);
sum3 += Math.Abs(sourceSample - predictionRow[column + 3]);
}
}
}
sums[0] = sum0;
sums[1] = sum1;
sums[2] = sum2;
sums[3] = sum3;
}
/// <inheritdoc/> /// <inheritdoc/>
public static int GetVariance( public static int GetVariance(
@ -515,44 +526,13 @@ internal static partial class Av1IntraSuperblockEncoder
Point predictionOrigin, Point predictionOrigin,
Av1BitDepth bitDepth) Av1BitDepth bitDepth)
{ {
int sum = 0; Av1ResidualBuilder.GetMoments8x8(
int sumOfSquares = 0; Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
if (Vector128.IsHardwareAccelerated) source.Stride,
{ Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin),
for (int row = 0; row < 8; row++) reconstruction.Stride,
{ out int sum,
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; out int sumOfSquares);
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
Vector128<short> difference =
(Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(sourceRow)).AsByte()) -
Vector128.WidenLower(Vector128.CreateScalarUnsafe(MemoryMarshal.Read<ulong>(predictionRow)).AsByte()))
.AsInt16();
// Widen before squaring so signed residuals cannot wrap in 16-bit lanes.
Vector128<int> lower = Vector128.WidenLower(difference);
Vector128<int> upper = Vector128.WidenUpper(difference);
sum += Vector128.Sum(difference);
sumOfSquares += Vector128.Sum(lower * lower) + Vector128.Sum(upper * upper);
}
}
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<byte> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<byte> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
int difference = sourceRow[column] - predictionRow[column];
sum += difference;
sumOfSquares += difference * difference;
}
}
}
return GetNormalizedVariance(sum, sumOfSquares, bitDepth); return GetNormalizedVariance(sum, sumOfSquares, bitDepth);
} }
@ -575,10 +555,7 @@ internal static partial class Av1IntraSuperblockEncoder
// A complete row widens in one vector; clipped edge rows retain scalar bounds. // A complete row widens in one vector; clipped edge rows retain scalar bounds.
if (columns == 8 && Vector128.IsHardwareAccelerated) if (columns == 8 && Vector128.IsHardwareAccelerated)
{ {
ulong packed = MemoryMarshal.Read<ulong>(sourceRow); Vector128.WidenLower(Vector128.Create(Vector64.Create(sourceRow), Vector64<byte>.Zero)).AsInt16().CopyTo(samples[sampleOffset..]);
Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte())
.AsInt16()
.CopyTo(samples[sampleOffset..]);
sampleOffset += columns; sampleOffset += columns;
continue; continue;
@ -803,7 +780,7 @@ internal static partial class Av1IntraSuperblockEncoder
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void PrepareIntraBlockCopy( public static void PrepareIntraBlockCopyPrediction(
Buffer2DRegion<byte> source, Buffer2DRegion<byte> source,
Point blockOrigin, Point blockOrigin,
Buffer2DRegion<byte> reconstruction, Buffer2DRegion<byte> reconstruction,
@ -837,6 +814,64 @@ internal static partial class Av1IntraSuperblockEncoder
height); height);
} }
/// <inheritdoc/>
public static void SubtractPrediction(
Buffer2DRegion<byte> source,
Point blockOrigin,
ReadOnlySpan<byte> prediction,
Span<short> residual,
Av1TransformSize transformSize)
=> Av1ResidualBuilder.Subtract(
Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin),
source.Stride,
prediction,
transformSize.GetWidth(),
residual,
transformSize.GetWidth(),
transformSize.GetWidth(),
transformSize.GetHeight());
/// <inheritdoc/>
public static void PrepareTranslationalInterPrediction(
Buffer2DRegion<byte> source,
Point blockOrigin,
Buffer2DRegion<byte> reference,
Point predictionOrigin,
Av1InterpolationFilter horizontalFilter,
Av1InterpolationFilter verticalFilter,
int horizontalPhase,
int verticalPhase,
Span<byte> prediction,
Span<short> residual,
Span<short> predictionScratch,
Av1TransformSize transformSize,
Av1BitDepth bitDepth)
{
int width = transformSize.GetWidth();
int height = transformSize.GetHeight();
Rectangle referenceBounds = reference.Bounds;
int referenceOrigin =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
referenceBounds.X +
predictionOrigin.X;
Av1TranslationalInterPredictor.Predict(
reference.Buffer.DangerousGetSingleSpan(),
reference.Stride,
referenceOrigin,
prediction,
width,
width,
height,
horizontalFilter,
verticalFilter,
horizontalPhase,
verticalPhase,
predictionScratch);
SubtractPrediction(source, blockOrigin, prediction, residual, transformSize);
}
/// <inheritdoc/> /// <inheritdoc/>
public static long EncodePredictionCandidate( public static long EncodePredictionCandidate(
Av1EncoderBlockWorkspace workspace, Av1EncoderBlockWorkspace workspace,
@ -947,49 +982,49 @@ internal static partial class Av1IntraSuperblockEncoder
} }
/// <inheritdoc/> /// <inheritdoc/>
public static int GetSumOfAbsoluteDifferences( public static long GetInterPredictionError(
Buffer2DRegion<ushort> source, Buffer2DRegion<ushort> source,
Point sourceOrigin, Point sourceOrigin,
Buffer2DRegion<ushort> reconstruction, Buffer2DRegion<ushort> reference,
Point predictionOrigin) Point predictionOrigin,
Av1BitDepth bitDepth)
{ {
int sum = 0; Rectangle sourceBounds = source.Bounds;
if (Vector128.IsHardwareAccelerated) Rectangle referenceBounds = reference.Bounds;
{ int sourceIndex =
for (int row = 0; row < 8; row++) ((sourceBounds.Y + sourceOrigin.Y) * source.Stride) +
{ sourceBounds.X +
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; sourceOrigin.X;
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..]; int referenceIndex =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
// Twelve-bit samples remain within signed 16-bit subtraction and absolute-value ranges, referenceBounds.X +
// allowing all eight row differences to stay packed until their horizontal reduction. predictionOrigin.X;
Vector128<short> difference =
(Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)) - long error = Av1ResidualBuilder.SumSquaredError(
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow))) source.Buffer.DangerousGetSingleSpan()[sourceIndex..],
.AsInt16(); source.Stride,
reference.Buffer.DangerousGetSingleSpan()[referenceIndex..],
sum += Vector128.Sum(Vector128.Abs(difference)); reference.Stride,
} InterSearchBlockDimension,
} InterSearchBlockDimension);
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
sum += Math.Abs(sourceRow[column] - predictionRow[column]);
}
}
}
return sum; int shift = (bitDepth.GetBitCount() - 8) * 2;
return shift == 0 ? error : (error + (1L << (shift - 1))) >> shift;
} }
/// <inheritdoc/>
public static int GetSumOfAbsoluteDifferences(
Buffer2DRegion<ushort> source,
Point sourceOrigin,
Buffer2DRegion<ushort> reconstruction,
Point predictionOrigin)
=> Av1ResidualBuilder.SumAbsoluteDifferences8x8(
Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
source.Stride,
Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin),
reconstruction.Stride);
/// <inheritdoc/> /// <inheritdoc/>
public static void GetFourSumsOfAbsoluteDifferences( public static void GetFourSumsOfAbsoluteDifferences(
Buffer2DRegion<ushort> source, Buffer2DRegion<ushort> source,
@ -997,66 +1032,12 @@ internal static partial class Av1IntraSuperblockEncoder
Buffer2DRegion<ushort> reconstruction, Buffer2DRegion<ushort> reconstruction,
Point firstPredictionOrigin, Point firstPredictionOrigin,
Span<int> sums) Span<int> sums)
{ => Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(
int sum0 = 0; Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
int sum1 = 0; source.Stride,
int sum2 = 0; Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, firstPredictionOrigin),
int sum3 = 0; reconstruction.Stride,
if (Vector128.IsHardwareAccelerated) sums);
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..];
// Signed 16-bit lanes preserve every AV1 sample difference while four horizontally adjacent
// candidates reuse the same source load and stay packed through horizontal reduction.
Vector128<short> sourceSamples =
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)).AsInt16();
Vector128<short> prediction0 =
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow)).AsInt16();
Vector128<short> prediction1 =
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[1..])).AsInt16();
Vector128<short> prediction2 =
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[2..])).AsInt16();
Vector128<short> prediction3 =
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow[3..])).AsInt16();
sum0 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction0));
sum1 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction1));
sum2 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction2));
sum3 += Vector128.Sum(Vector128.Abs(sourceSamples - prediction3));
}
}
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(firstPredictionOrigin.Y + row)[firstPredictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
int sourceSample = sourceRow[column];
sum0 += Math.Abs(sourceSample - predictionRow[column]);
sum1 += Math.Abs(sourceSample - predictionRow[column + 1]);
sum2 += Math.Abs(sourceSample - predictionRow[column + 2]);
sum3 += Math.Abs(sourceSample - predictionRow[column + 3]);
}
}
}
sums[0] = sum0;
sums[1] = sum1;
sums[2] = sum2;
sums[3] = sum3;
}
/// <inheritdoc/> /// <inheritdoc/>
public static int GetVariance( public static int GetVariance(
@ -1066,45 +1047,13 @@ internal static partial class Av1IntraSuperblockEncoder
Point predictionOrigin, Point predictionOrigin,
Av1BitDepth bitDepth) Av1BitDepth bitDepth)
{ {
int sum = 0; Av1ResidualBuilder.GetMoments8x8(
int sumOfSquares = 0; Av1TransformBlockEncoder.GetPlaneSpan(source, sourceOrigin),
if (Vector128.IsHardwareAccelerated) source.Stride,
{ Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin),
for (int row = 0; row < 8; row++) reconstruction.Stride,
{ out int sum,
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..]; out int sumOfSquares);
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
// AV1's high-bit-depth domain tops out at 4095, so signed 16-bit subtraction preserves
// every possible sample difference before the square is widened to 32-bit lanes.
Vector128<short> difference =
(Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(sourceRow)) -
Vector128.LoadUnsafe(ref MemoryMarshal.GetReference(predictionRow)))
.AsInt16();
Vector128<int> lower = Vector128.WidenLower(difference);
Vector128<int> upper = Vector128.WidenUpper(difference);
sum += Vector128.Sum(difference);
sumOfSquares += Vector128.Sum(lower * lower) + Vector128.Sum(upper * upper);
}
}
else
{
for (int row = 0; row < 8; row++)
{
ReadOnlySpan<ushort> sourceRow = source.DangerousGetRowSpan(sourceOrigin.Y + row)[sourceOrigin.X..];
ReadOnlySpan<ushort> predictionRow =
reconstruction.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X..];
for (int column = 0; column < 8; column++)
{
int difference = sourceRow[column] - predictionRow[column];
sum += difference;
sumOfSquares += difference * difference;
}
}
}
return GetNormalizedVariance(sum, sumOfSquares, bitDepth); return GetNormalizedVariance(sum, sumOfSquares, bitDepth);
} }
@ -1353,7 +1302,7 @@ internal static partial class Av1IntraSuperblockEncoder
} }
/// <inheritdoc/> /// <inheritdoc/>
public static void PrepareIntraBlockCopy( public static void PrepareIntraBlockCopyPrediction(
Buffer2DRegion<ushort> source, Buffer2DRegion<ushort> source,
Point blockOrigin, Point blockOrigin,
Buffer2DRegion<ushort> reconstruction, Buffer2DRegion<ushort> reconstruction,
@ -1387,6 +1336,65 @@ internal static partial class Av1IntraSuperblockEncoder
height); height);
} }
/// <inheritdoc/>
public static void SubtractPrediction(
Buffer2DRegion<ushort> source,
Point blockOrigin,
ReadOnlySpan<ushort> prediction,
Span<short> residual,
Av1TransformSize transformSize)
=> Av1ResidualBuilder.Subtract(
Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin),
source.Stride,
prediction,
transformSize.GetWidth(),
residual,
transformSize.GetWidth(),
transformSize.GetWidth(),
transformSize.GetHeight());
/// <inheritdoc/>
public static void PrepareTranslationalInterPrediction(
Buffer2DRegion<ushort> source,
Point blockOrigin,
Buffer2DRegion<ushort> reference,
Point predictionOrigin,
Av1InterpolationFilter horizontalFilter,
Av1InterpolationFilter verticalFilter,
int horizontalPhase,
int verticalPhase,
Span<ushort> prediction,
Span<short> residual,
Span<short> predictionScratch,
Av1TransformSize transformSize,
Av1BitDepth bitDepth)
{
int width = transformSize.GetWidth();
int height = transformSize.GetHeight();
Rectangle referenceBounds = reference.Bounds;
int referenceOrigin =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
referenceBounds.X +
predictionOrigin.X;
Av1TranslationalInterPredictor.Predict(
reference.Buffer.DangerousGetSingleSpan(),
reference.Stride,
referenceOrigin,
prediction,
width,
width,
height,
horizontalFilter,
verticalFilter,
horizontalPhase,
verticalPhase,
bitDepth.GetBitCount(),
predictionScratch);
SubtractPrediction(source, blockOrigin, prediction, residual, transformSize);
}
/// <inheritdoc/> /// <inheritdoc/>
public static long EncodePredictionCandidate( public static long EncodePredictionCandidate(
Av1EncoderBlockWorkspace workspace, Av1EncoderBlockWorkspace workspace,

3
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs

@ -372,7 +372,8 @@ internal static partial class Av1IntraSuperblockEncoder
macroBlock, macroBlock,
BlockSize, BlockSize,
Av1PredictionMode.DC, Av1PredictionMode.DC,
0); 0,
this.picture.Parent.FrameHeader.IsIntra);
rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext); rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext);
rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y); rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y);

1751
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs

File diff suppressed because it is too large

170
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs

@ -1,170 +0,0 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Encodes one range-coded all-intra tile payload.
/// </summary>
internal sealed partial class Av1IntraTileWriter : IAv1TileWriter
{
private readonly ReadOnlyMemory<byte> tileData;
private readonly int tileDataLength;
/// <summary>
/// Initializes a new instance of the <see cref="Av1IntraTileWriter"/> class for eight-bit samples.
/// </summary>
/// <param name="writer">The operation-owned symbol encoder that retains the tile output memory.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1IntraTileWriter(
Av1SymbolEncoder writer,
Av1EncoderFrame<byte> source,
Av1EncoderFrame<byte> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
writer,
source,
reconstruction,
picture,
coefficientBuffer,
superblockWorkspace,
blockWorkspace,
effort,
out this.tileDataLength);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1IntraTileWriter"/> class for high-bit-depth samples.
/// </summary>
/// <param name="writer">The operation-owned symbol encoder that retains the tile output memory.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1IntraTileWriter(
Av1SymbolEncoder writer,
Av1EncoderFrame<ushort> source,
Av1EncoderFrame<ushort> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
writer,
source,
reconstruction,
picture,
coefficientBuffer,
superblockWorkspace,
blockWorkspace,
effort,
out this.tileDataLength);
}
/// <inheritdoc/>
public ReadOnlySpan<byte> GetTileData(int tileNum) => this.tileData.Span[..this.tileDataLength];
private static ReadOnlyMemory<byte> Encode<TSample, TOperator>(
Av1SymbolEncoder writer,
Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort,
out int tileDataLength)
where TSample : unmanaged
where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample>
{
ObuFrameHeader frameHeader = picture.Parent.FrameHeader;
ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader;
const ushort TileIndex = 0;
Av1TileInfo tile = new(0, 0, frameHeader);
Av1Superblock superblock = new()
{
Workspace = superblockWorkspace,
TileInfo = tile
};
Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart);
Av1TileWriter.Av1EntropyCodingContext entropyContext = new()
{
MacroBlock = new Av1MacroBlockD { Tile = tile },
MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition)
};
int superblockModeInfoSize = sequenceHeader.SuperblockModeInfoSize;
int superblockShift = sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2;
if (frameHeader.AllowIntraBlockCopy)
{
// Hash the visible source once before reconstruction begins so candidate discovery never depends
// on coding order and the workspace can be reused as compact bucket links afterward.
picture.IntraBlockCopySearch.Initialize<TSample, TOperator>(
source.View.GetPlane(Av1Plane.Y));
}
for (int modeInfoRow = tile.ModeInfoRowStart;
modeInfoRow < tile.ModeInfoRowEnd;
modeInfoRow += superblockModeInfoSize)
{
for (int modeInfoColumn = tile.ModeInfoColumnStart;
modeInfoColumn < tile.ModeInfoColumnEnd;
modeInfoColumn += superblockModeInfoSize)
{
int superblockRow = modeInfoRow >> superblockShift;
int superblockColumn = modeInfoColumn >> superblockShift;
superblock.Index = (superblockRow * coefficientBuffer.SuperblockColumnCount) + superblockColumn;
entropyContext.SuperblockOrigin = new Point(
modeInfoColumn << Av1Constants.ModeInfoSizeLog2,
modeInfoRow << Av1Constants.ModeInfoSizeLog2);
Av1IntraSuperblockEncoder.Prepare(
picture,
superblock,
entropyContext.SuperblockOrigin);
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator> blockEncoder = new(
source,
reconstruction,
picture,
superblock,
coefficientBuffer,
blockWorkspace,
effort);
Av1TileWriter.WriteSuperblock(
picture,
entropyContext,
writer,
superblock,
coefficientBuffer,
TileIndex,
ref blockEncoder);
}
}
return writer.Exit(out tileDataLength);
}
}

158
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.Operator.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics; using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
@ -51,6 +52,57 @@ internal static partial class Av1ResidualBuilder
/// <param name="prediction">The prediction sample.</param> /// <param name="prediction">The prediction sample.</param>
/// <returns>The signed residual.</returns> /// <returns>The signed residual.</returns>
public static abstract short Subtract(TSample source, TSample prediction); public static abstract short Subtract(TSample source, TSample prediction);
/// <summary>
/// Measures one scalar absolute sample difference.
/// </summary>
/// <param name="source">The source sample.</param>
/// <param name="prediction">The prediction sample.</param>
/// <returns>The sum of absolute differences.</returns>
public static abstract int SumAbsoluteDifferences(TSample source, TSample prediction);
/// <summary>
/// Measures eight absolute sample differences; byte inputs occupy only the lower eight lanes.
/// </summary>
/// <param name="source">The eight source samples.</param>
/// <param name="prediction">The eight prediction samples.</param>
/// <returns>The sum of absolute differences.</returns>
public static abstract int SumAbsoluteDifferences(Vector128<TSample> source, Vector128<TSample> prediction);
/// <summary>
/// Measures four eight-sample predictions, returning their costs in candidate order.
/// Byte inputs occupy only the lower eight lanes.
/// </summary>
/// <param name="source">The eight source samples.</param>
/// <param name="prediction0">The eight samples for candidate 0.</param>
/// <param name="prediction1">The eight samples for candidate 1.</param>
/// <param name="prediction2">The eight samples for candidate 2.</param>
/// <param name="prediction3">The eight samples for candidate 3.</param>
/// <returns>Four absolute-difference sums in increasing candidate order.</returns>
public static abstract Vector128<int> SumFourAbsoluteDifferences(
Vector128<TSample> source,
Vector128<TSample> prediction0,
Vector128<TSample> prediction1,
Vector128<TSample> prediction2,
Vector128<TSample> prediction3);
/// <summary>
/// Calculates one squared difference and returns its signed difference for the first moment.
/// </summary>
/// <param name="source">The source sample.</param>
/// <param name="prediction">The prediction sample.</param>
/// <param name="sum">The signed sum of source-minus-prediction differences.</param>
/// <returns>The sum of squared differences.</returns>
public static abstract int SumSquaredDifferences(TSample source, TSample prediction, out int sum);
/// <summary>
/// Calculates eight squared differences and their signed sum; byte inputs occupy only the lower eight lanes.
/// </summary>
/// <param name="source">The eight source samples.</param>
/// <param name="prediction">The eight prediction samples.</param>
/// <param name="sum">The signed sum of source-minus-prediction differences.</param>
/// <returns>The sum of squared differences.</returns>
public static abstract int SumSquaredDifferences(Vector128<TSample> source, Vector128<TSample> prediction, out int sum);
} }
/// <summary> /// <summary>
@ -58,6 +110,59 @@ internal static partial class Av1ResidualBuilder
/// </summary> /// </summary>
internal readonly struct ByteOperator : IResidualOperator<byte> internal readonly struct ByteOperator : IResidualOperator<byte>
{ {
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumAbsoluteDifferences(byte source, byte prediction) => Math.Abs(Subtract(source, prediction));
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumAbsoluteDifferences(Vector128<byte> source, Vector128<byte> prediction)
{
// Widen byte samples before subtraction; twelve-bit word samples already fit signed-short lanes.
// Eight absolute residuals sum to at most 32760, so the signed-short horizontal sum remains exact.
Vector128<short> difference = Vector128.WidenLower(source).AsInt16() - Vector128.WidenLower(prediction).AsInt16();
return Vector128.Sum(Vector128.Abs(difference));
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<int> SumFourAbsoluteDifferences(
Vector128<byte> source,
Vector128<byte> prediction0,
Vector128<byte> prediction1,
Vector128<byte> prediction2,
Vector128<byte> prediction3)
{
// Reuse the source conversion across all four candidates. Each reduction contributes one independent
// 32-bit lane, allowing the traversal to accumulate all eight rows without extracting candidate costs.
Vector128<short> sourceSamples = Vector128.WidenLower(source).AsInt16();
return Vector128.Create(
(int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction0).AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction1).AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction2).AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - Vector128.WidenLower(prediction3).AsInt16())));
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumSquaredDifferences(byte source, byte prediction, out int sum)
{
sum = Subtract(source, prediction);
return sum * sum;
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumSquaredDifferences(Vector128<byte> source, Vector128<byte> prediction, out int sum)
{
Vector128<short> difference = Vector128.WidenLower(source).AsInt16() - Vector128.WidenLower(prediction).AsInt16();
sum = Vector128.Sum(difference);
// The shared square reduction widens to int before multiplying. All eight twelve-bit squares
// fit in the returned int; no short multiplication or premature bit-depth rounding is permitted.
return (int)SumSquares(difference);
}
/// <inheritdoc/> /// <inheritdoc/>
public static Vector128<short> Subtract(Vector128<byte> source, Vector128<byte> prediction, out Vector128<short> upper) public static Vector128<short> Subtract(Vector128<byte> source, Vector128<byte> prediction, out Vector128<short> upper)
{ {
@ -91,6 +196,59 @@ internal static partial class Av1ResidualBuilder
/// </summary> /// </summary>
internal readonly struct UInt16Operator : IResidualOperator<ushort> internal readonly struct UInt16Operator : IResidualOperator<ushort>
{ {
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumAbsoluteDifferences(ushort source, ushort prediction) => Math.Abs(Subtract(source, prediction));
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumAbsoluteDifferences(Vector128<ushort> source, Vector128<ushort> prediction)
{
// Widen byte samples before subtraction; twelve-bit word samples already fit signed-short lanes.
// Eight absolute residuals sum to at most 32760, so the signed-short horizontal sum remains exact.
Vector128<short> difference = source.AsInt16() - prediction.AsInt16();
return Vector128.Sum(Vector128.Abs(difference));
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static Vector128<int> SumFourAbsoluteDifferences(
Vector128<ushort> source,
Vector128<ushort> prediction0,
Vector128<ushort> prediction1,
Vector128<ushort> prediction2,
Vector128<ushort> prediction3)
{
// Reuse the source conversion across all four candidates. Each reduction contributes one independent
// 32-bit lane, allowing the traversal to accumulate all eight rows without extracting candidate costs.
Vector128<short> sourceSamples = source.AsInt16();
return Vector128.Create(
(int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction0.AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction1.AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction2.AsInt16())),
(int)Vector128.Sum(Vector128.Abs(sourceSamples - prediction3.AsInt16())));
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumSquaredDifferences(ushort source, ushort prediction, out int sum)
{
sum = Subtract(source, prediction);
return sum * sum;
}
/// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static int SumSquaredDifferences(Vector128<ushort> source, Vector128<ushort> prediction, out int sum)
{
Vector128<short> difference = source.AsInt16() - prediction.AsInt16();
sum = Vector128.Sum(difference);
// The shared square reduction widens to int before multiplying. All eight twelve-bit squares
// fit in the returned int; no short multiplication or premature bit-depth rounding is permitted.
return (int)SumSquares(difference);
}
/// <inheritdoc/> /// <inheritdoc/>
public static Vector128<short> Subtract(Vector128<ushort> source, Vector128<ushort> prediction, out Vector128<short> upper) public static Vector128<short> Subtract(Vector128<ushort> source, Vector128<ushort> prediction, out Vector128<short> upper)
{ {

220
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ResidualBuilder.cs

@ -12,6 +12,226 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// </summary> /// </summary>
internal static partial class Av1ResidualBuilder internal static partial class Av1ResidualBuilder
{ {
/// <summary>
/// The width and height of the encoder's fixed motion-search block, in samples.
/// </summary>
private const int SearchBlockDimension = 8;
/// <summary>
/// Calculates the sum of absolute differences for an 8x8 block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The prediction samples starting at the block origin.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <returns>The sum of absolute sample differences.</returns>
public static int SumAbsoluteDifferences8x8(ReadOnlySpan<byte> source, int sourceStride, ReadOnlySpan<byte> prediction, int predictionStride)
=> SumAbsoluteDifferences8x8<byte, ByteOperator>(source, sourceStride, prediction, predictionStride);
/// <summary>
/// Measures four horizontally adjacent 8x8 predictions against one source block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The first prediction, with three additional samples available at the right of each row.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <param name="sums">The four results in increasing horizontal-offset order.</param>
public static void SumFourAbsoluteDifferences8x8(
ReadOnlySpan<byte> source, int sourceStride, ReadOnlySpan<byte> prediction, int predictionStride, Span<int> sums)
=> SumFourAbsoluteDifferences8x8<byte, ByteOperator>(source, sourceStride, prediction, predictionStride, sums);
/// <summary>
/// Calculates the signed sum and squared sum of differences for an 8x8 block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The prediction samples starting at the block origin.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <param name="sum">The signed sum of sample differences.</param>
/// <param name="sumOfSquares">The sum of squared sample differences.</param>
public static void GetMoments8x8(
ReadOnlySpan<byte> source, int sourceStride, ReadOnlySpan<byte> prediction, int predictionStride, out int sum, out int sumOfSquares)
=> GetMoments8x8<byte, ByteOperator>(source, sourceStride, prediction, predictionStride, out sum, out sumOfSquares);
/// <summary>
/// Calculates the sum of absolute differences for an 8x8 block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The prediction samples starting at the block origin.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <returns>The sum of absolute sample differences.</returns>
public static int SumAbsoluteDifferences8x8(ReadOnlySpan<ushort> source, int sourceStride, ReadOnlySpan<ushort> prediction, int predictionStride)
=> SumAbsoluteDifferences8x8<ushort, UInt16Operator>(source, sourceStride, prediction, predictionStride);
/// <summary>
/// Measures four horizontally adjacent 8x8 predictions against one source block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The first prediction, with three additional samples available at the right of each row.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <param name="sums">The four results in increasing horizontal-offset order.</param>
public static void SumFourAbsoluteDifferences8x8(
ReadOnlySpan<ushort> source, int sourceStride, ReadOnlySpan<ushort> prediction, int predictionStride, Span<int> sums)
=> SumFourAbsoluteDifferences8x8<ushort, UInt16Operator>(source, sourceStride, prediction, predictionStride, sums);
/// <summary>
/// Calculates the signed sum and squared sum of differences for an 8x8 block.
/// </summary>
/// <param name="source">The source samples starting at the block origin.</param>
/// <param name="sourceStride">The source row stride, in samples.</param>
/// <param name="prediction">The prediction samples starting at the block origin.</param>
/// <param name="predictionStride">The prediction row stride, in samples.</param>
/// <param name="sum">The signed sum of sample differences.</param>
/// <param name="sumOfSquares">The sum of squared sample differences.</param>
public static void GetMoments8x8(
ReadOnlySpan<ushort> source, int sourceStride, ReadOnlySpan<ushort> prediction, int predictionStride, out int sum, out int sumOfSquares)
=> GetMoments8x8<ushort, UInt16Operator>(source, sourceStride, prediction, predictionStride, out sum, out sumOfSquares);
/// <summary>
/// Traverses an 8x8 block while its closed operator calculates scalar or eight-sample row costs.
/// </summary>
private static int SumAbsoluteDifferences8x8<TSample, TOperator>(
ReadOnlySpan<TSample> source, int sourceStride, ReadOnlySpan<TSample> prediction, int predictionStride)
where TSample : unmanaged
where TOperator : struct, IResidualOperator<TSample>
{
int sum = 0;
if (Vector128.IsHardwareAccelerated)
{
// Eight widened AV1 samples exactly fill 128 bits. Wider loads would cross the row boundary;
// byte storage is loaded as eight bytes and ushort storage as eight native-order words.
for (int row = 0; row < SearchBlockDimension; row++)
{
Vector128<TSample> sourceRow = LoadSearchRow(source[(row * sourceStride)..]);
Vector128<TSample> predictionRow = LoadSearchRow(prediction[(row * predictionStride)..]);
sum += TOperator.SumAbsoluteDifferences(sourceRow, predictionRow);
}
}
else
{
for (int row = 0; row < SearchBlockDimension; row++)
{
ReadOnlySpan<TSample> sourceRow = source.Slice(row * sourceStride, SearchBlockDimension);
ReadOnlySpan<TSample> predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension);
for (int column = 0; column < SearchBlockDimension; column++)
{
sum += TOperator.SumAbsoluteDifferences(sourceRow[column], predictionRow[column]);
}
}
}
return sum;
}
/// <summary>
/// Traverses four adjacent candidates together, retaining one source load per row or scalar sample.
/// </summary>
private static void SumFourAbsoluteDifferences8x8<TSample, TOperator>(
ReadOnlySpan<TSample> source, int sourceStride, ReadOnlySpan<TSample> prediction, int predictionStride, Span<int> sums)
where TSample : unmanaged
where TOperator : struct, IResidualOperator<TSample>
{
if (Vector128.IsHardwareAccelerated)
{
Vector128<int> totals = Vector128<int>.Zero;
for (int row = 0; row < SearchBlockDimension; row++)
{
ReadOnlySpan<TSample> predictionRow = prediction[(row * predictionStride)..];
Vector128<TSample> sourceRow = LoadSearchRow(source[(row * sourceStride)..]);
// The four prediction windows overlap, but each candidate owns one result lane. The operator
// widens the source only once and reuses it for all four independent absolute-difference sums.
totals += TOperator.SumFourAbsoluteDifferences(
sourceRow,
LoadSearchRow(predictionRow),
LoadSearchRow(predictionRow[1..]),
LoadSearchRow(predictionRow[2..]),
LoadSearchRow(predictionRow[3..]));
}
totals.CopyTo(sums);
}
else
{
int sum0 = 0;
int sum1 = 0;
int sum2 = 0;
int sum3 = 0;
for (int row = 0; row < SearchBlockDimension; row++)
{
ReadOnlySpan<TSample> sourceRow = source.Slice(row * sourceStride, SearchBlockDimension);
ReadOnlySpan<TSample> predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension + 3);
for (int column = 0; column < SearchBlockDimension; column++)
{
TSample sample = sourceRow[column];
sum0 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column]);
sum1 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 1]);
sum2 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 2]);
sum3 += TOperator.SumAbsoluteDifferences(sample, predictionRow[column + 3]);
}
}
sums[0] = sum0;
sums[1] = sum1;
sums[2] = sum2;
sums[3] = sum3;
}
}
/// <summary>
/// Accumulates both residual moments in one traversal without materializing a residual buffer.
/// </summary>
private static void GetMoments8x8<TSample, TOperator>(
ReadOnlySpan<TSample> source, int sourceStride, ReadOnlySpan<TSample> prediction, int predictionStride, out int sum, out int sumOfSquares)
where TSample : unmanaged
where TOperator : struct, IResidualOperator<TSample>
{
sum = 0;
sumOfSquares = 0;
// Even 64 maximum twelve-bit residual squares fit in a signed int. Preserve the unnormalized
// moments here; the caller applies the frame's precision-dependent rounding before deriving variance.
if (Vector128.IsHardwareAccelerated)
{
for (int row = 0; row < SearchBlockDimension; row++)
{
Vector128<TSample> sourceRow = LoadSearchRow(source[(row * sourceStride)..]);
Vector128<TSample> predictionRow = LoadSearchRow(prediction[(row * predictionStride)..]);
sumOfSquares += TOperator.SumSquaredDifferences(sourceRow, predictionRow, out int rowSum);
sum += rowSum;
}
}
else
{
for (int row = 0; row < SearchBlockDimension; row++)
{
ReadOnlySpan<TSample> sourceRow = source.Slice(row * sourceStride, SearchBlockDimension);
ReadOnlySpan<TSample> predictionRow = prediction.Slice(row * predictionStride, SearchBlockDimension);
for (int column = 0; column < SearchBlockDimension; column++)
{
sumOfSquares += TOperator.SumSquaredDifferences(sourceRow[column], predictionRow[column], out int difference);
sum += difference;
}
}
}
}
/// <summary>
/// Loads exactly eight native-order samples; byte rows occupy the lower half of the returned vector.
/// </summary>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<TSample> LoadSearchRow<TSample>(ReadOnlySpan<TSample> source)
where TSample : unmanaged
{
// The closed byte/ushort instantiation removes this storage-width choice. The eight-byte load
// never consumes padding or a following row; the operator widens only its eight populated lanes.
return Vector128<TSample>.Count == SearchBlockDimension
? Vector128.Create(source)
: Vector128.Create(Vector64.Create(source), Vector64<TSample>.Zero);
}
/// <summary> /// <summary>
/// Subtracts an 8-bit prediction plane from its source plane. /// Subtracts an 8-bit prediction plane from its source plane.
/// </summary> /// </summary>

383
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs

@ -0,0 +1,383 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Encodes one range-coded AV1 tile payload.
/// </summary>
internal readonly struct Av1TileEncoder : IAv1TileWriter
{
private readonly ReadOnlyMemory<byte> tileData;
private readonly Av1PictureControlSet picture;
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for eight-bit samples.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<byte> source,
Av1EncoderFrame<byte> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
writer,
source,
reconstruction,
reconstruction,
picture,
coefficientBuffer,
new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace),
blockWorkspace,
effort);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for an eight-bit inter frame.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reference">The reconstructed reference frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<byte> source,
Av1EncoderFrame<byte> reference,
Av1EncoderFrame<byte> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
writer,
source,
reference,
reconstruction,
picture,
coefficientBuffer,
new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace),
blockWorkspace,
effort);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for an eight-bit inter frame.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reference">The reconstructed reference frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="tileWorkspace">The retained tile, superblock, and entropy cursor graph.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<byte> source,
Av1EncoderFrame<byte> reference,
Av1EncoderFrame<byte> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderTileWorkspace tileWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
writer,
source,
reference,
reconstruction,
picture,
coefficientBuffer,
tileWorkspace,
blockWorkspace,
effort);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for high-bit-depth samples.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<ushort> source,
Av1EncoderFrame<ushort> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
writer,
source,
reconstruction,
reconstruction,
picture,
coefficientBuffer,
new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace),
blockWorkspace,
effort);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for a high-bit-depth inter frame.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reference">The reconstructed reference frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<ushort> source,
Av1EncoderFrame<ushort> reference,
Av1EncoderFrame<ushort> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
writer,
source,
reference,
reconstruction,
picture,
coefficientBuffer,
new Av1EncoderTileWorkspace(picture.Parent.FrameHeader, superblockWorkspace),
blockWorkspace,
effort);
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1TileEncoder"/> struct for a high-bit-depth inter frame.
/// </summary>
/// <param name="writer">The symbol encoder that retains tile output through the enclosing frame write.</param>
/// <param name="source">The coded source frame.</param>
/// <param name="reference">The reconstructed reference frame.</param>
/// <param name="reconstruction">The reconstructed frame updated during encoding.</param>
/// <param name="picture">The frame coding and mode-information state.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="tileWorkspace">The retained tile, superblock, and entropy cursor graph.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1TileEncoder(
Av1SymbolEncoder writer,
Av1EncoderFrame<ushort> source,
Av1EncoderFrame<ushort> reference,
Av1EncoderFrame<ushort> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderTileWorkspace tileWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
{
this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
writer,
source,
reference,
reconstruction,
picture,
coefficientBuffer,
tileWorkspace,
blockWorkspace,
effort);
}
/// <inheritdoc/>
public ReadOnlySpan<byte> GetTileData(int tileNum)
{
int offset = this.picture.TileDataOffsets.Span[tileNum];
int length = this.picture.TileDataLengths.Span[tileNum];
return this.tileData.Span.Slice(offset, length);
}
private static ReadOnlyMemory<byte> Encode<TSample, TOperator>(
Av1SymbolEncoder writer,
Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reference,
Av1EncoderFrame<TSample> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderTileWorkspace tileWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
where TSample : unmanaged
where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample>
{
ObuFrameHeader frameHeader = picture.Parent.FrameHeader;
ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader;
Av1TileInfo tile = tileWorkspace.Tile;
Av1Superblock superblock = tileWorkspace.Superblock;
Av1TileWriter.Av1EntropyCodingContext entropyContext = tileWorkspace.EntropyContext;
int superblockModeInfoSize = sequenceHeader.SuperblockModeInfoSize;
int superblockShift = sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2;
ObuTileGroupHeader tileLayout = frameHeader.TilesInfo;
Span<int> tileDataOffsets = picture.TileDataOffsets.Span;
Span<int> tileDataLengths = picture.TileDataLengths.Span;
if (frameHeader.AllowIntraBlockCopy)
{
// Hash the visible source once before reconstruction begins so candidate discovery never depends
// on coding order and the workspace can be reused as compact bucket links afterward.
picture.IntraBlockCopySearch.Initialize<TSample, TOperator>(
source.View.GetPlane(Av1Plane.Y));
}
int tileIndex = 0;
int tileDataEnd = 0;
for (int tileRow = 0; tileRow < tileLayout.TileRowCount; tileRow++)
{
tile.SetTileRow(tileLayout, frameHeader.ModeInfoRowCount, tileRow);
for (int tileColumn = 0; tileColumn < tileLayout.TileColumnCount; tileColumn++)
{
tile.SetTileColumn(tileLayout, frameHeader.ModeInfoColumnCount, tileColumn);
if (tileIndex > 0)
{
// Every tile begins from the same frame probabilities, while its bytes follow the preceding
// tile in the retained output allocation.
writer.Reset(tileDataEnd);
}
Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart);
entropyContext.MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition);
for (int modeInfoRow = tile.ModeInfoRowStart;
modeInfoRow < tile.ModeInfoRowEnd;
modeInfoRow += superblockModeInfoSize)
{
for (int modeInfoColumn = tile.ModeInfoColumnStart;
modeInfoColumn < tile.ModeInfoColumnEnd;
modeInfoColumn += superblockModeInfoSize)
{
int superblockRow = modeInfoRow >> superblockShift;
int superblockColumn = modeInfoColumn >> superblockShift;
superblock.Index = (superblockRow * coefficientBuffer.SuperblockColumnCount) + superblockColumn;
entropyContext.SuperblockOrigin = new Point(
modeInfoColumn << Av1Constants.ModeInfoSizeLog2,
modeInfoRow << Av1Constants.ModeInfoSizeLog2);
Av1IntraSuperblockEncoder.Prepare(
picture,
superblock,
entropyContext.SuperblockOrigin);
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator> blockEncoder = new(
source,
reference,
reconstruction,
picture,
superblock,
coefficientBuffer,
blockWorkspace,
effort);
Av1TileWriter.WriteSuperblock(
picture,
entropyContext,
writer,
superblock,
coefficientBuffer,
(ushort)tileIndex,
ref blockEncoder);
}
}
_ = writer.Exit(out int tileDataLength);
tileDataOffsets[tileIndex] = tileDataEnd;
tileDataLengths[tileIndex] = tileDataLength;
tileDataEnd += tileDataLength;
tileIndex++;
}
}
return writer.GetOutput(tileDataEnd);
}
}
/// <summary>
/// Retains the mutable tile, superblock, and entropy cursor graph reused by serial frame encoding.
/// </summary>
internal readonly struct Av1EncoderTileWorkspace
{
/// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderTileWorkspace"/> struct.
/// </summary>
/// <param name="frameHeader">The fixed-geometry frame header defining tile boundaries.</param>
/// <param name="superblockWorkspace">The retained superblock decision storage.</param>
public Av1EncoderTileWorkspace(
ObuFrameHeader frameHeader,
Av1EncoderSuperblockWorkspace superblockWorkspace)
{
this.Tile = new Av1TileInfo(0, 0, frameHeader);
this.Superblock = new Av1Superblock
{
Workspace = superblockWorkspace,
TileInfo = this.Tile
};
this.EntropyContext = new Av1TileWriter.Av1EntropyCodingContext
{
MacroBlock = new Av1MacroBlockD { Tile = this.Tile },
MacroBlockModeInfo = default
};
}
/// <summary>
/// Gets the mutable tile boundaries selected during raster traversal.
/// </summary>
public Av1TileInfo Tile { get; }
/// <summary>
/// Gets the mutable superblock cursor connected to the retained decision workspace.
/// </summary>
public Av1Superblock Superblock { get; }
/// <summary>
/// Gets the mutable entropy cursor shared by successive superblocks.
/// </summary>
public Av1TileWriter.Av1EntropyCodingContext EntropyContext { get; }
}

2
src/ImageSharp/Formats/Heif/Av1/Prediction/Inter/Av1TranslationalInterPredictor.cs

@ -35,7 +35,7 @@ internal static partial class Av1TranslationalInterPredictor
/// <summary> /// <summary>
/// The maximum number of source rows added by an eight-tap vertical filter. /// The maximum number of source rows added by an eight-tap vertical filter.
/// </summary> /// </summary>
private const int MaximumExtraRows = FilterCoefficientCount - 1; internal const int MaximumExtraRows = FilterCoefficientCount - 1;
/// <summary> /// <summary>
/// The minimum scratch stride that lets a 128-bit byte kernel handle four- and eight-sample blocks. /// The minimum scratch stride that lets a 128-bit byte kernel handle four- and eight-sample blocks.

58
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockModeInfo.cs

@ -2,6 +2,7 @@
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -11,16 +12,36 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// </summary> /// </summary>
internal struct Av1EncoderBlockModeInfo internal struct Av1EncoderBlockModeInfo
{ {
/// <summary>The residual-skip flag in the packed prediction state.</summary>
private const byte SkipMask = 1 << 0; private const byte SkipMask = 1 << 0;
/// <summary>The compound-skip flag in the packed prediction state.</summary>
private const byte SkipModeMask = 1 << 1; private const byte SkipModeMask = 1 << 1;
/// <summary>The intra-block-copy flag in the packed prediction state.</summary>
private const byte IntraBlockCopyMask = 1 << 2; private const byte IntraBlockCopyMask = 1 << 2;
// Every stored syntax value has an AV1-defined range below 256. Byte fields and one shared flag byte /// <summary>The three low bits store the segment identifier, followed by the primary reference.</summary>
// keep the frame-wide mode allocation compact without losing any representable encoder state. private const int ReferenceFrameShift = 3;
/// <summary>Both segment identifiers and primary references have eight possible values.</summary>
private const int SegmentAndReferenceMask = 7;
/// <summary>The vertical filter follows the three prediction flags.</summary>
private const int VerticalFilterShift = 3;
/// <summary>The horizontal filter follows the two-bit vertical filter.</summary>
private const int HorizontalFilterShift = 5;
/// <summary>Two bits represent each concrete interpolation filter, excluding the frame-level switchable sentinel.</summary>
private const int InterpolationFilterMask = 3;
// Primary references never use the absent-secondary sentinel. Pack their three bits beside the segment,
// and the concrete filters beside the flags, so adding inter syntax does not enlarge the frame-wide grid.
private byte blockSize; private byte blockSize;
private byte partitionType; private byte partitionType;
private byte flags; private byte flags;
private byte segmentId; private byte segmentAndReference;
private byte transformSize; private byte transformSize;
private byte mode; private byte mode;
private byte uvMode; private byte uvMode;
@ -75,8 +96,8 @@ internal struct Av1EncoderBlockModeInfo
/// </summary> /// </summary>
public int SegmentId public int SegmentId
{ {
readonly get => this.segmentId; readonly get => this.segmentAndReference & SegmentAndReferenceMask;
set => this.segmentId = (byte)value; set => this.segmentAndReference = (byte)((this.segmentAndReference & ~SegmentAndReferenceMask) | value);
} }
/// <summary> /// <summary>
@ -105,4 +126,31 @@ internal struct Av1EncoderBlockModeInfo
readonly get => (Av1ChromaPredictionMode)this.uvMode; readonly get => (Av1ChromaPredictionMode)this.uvMode;
set => this.uvMode = (byte)value; set => this.uvMode = (byte)value;
} }
/// <summary>
/// Gets or sets the primary prediction reference selected for the block.
/// </summary>
public Av1ReferenceFrameType ReferenceFrame
{
readonly get => (Av1ReferenceFrameType)(this.segmentAndReference >> ReferenceFrameShift);
set => this.segmentAndReference = (byte)((this.segmentAndReference & SegmentAndReferenceMask) | ((int)value << ReferenceFrameShift));
}
/// <summary>
/// Gets or sets the concrete vertical interpolation filter selected for the block.
/// </summary>
public Av1InterpolationFilter VerticalInterpolationFilter
{
readonly get => (Av1InterpolationFilter)((this.flags >> VerticalFilterShift) & InterpolationFilterMask);
set => this.flags = (byte)((this.flags & ~(InterpolationFilterMask << VerticalFilterShift)) | ((int)value << VerticalFilterShift));
}
/// <summary>
/// Gets or sets the concrete horizontal interpolation filter selected for the block.
/// </summary>
public Av1InterpolationFilter HorizontalInterpolationFilter
{
readonly get => (Av1InterpolationFilter)((this.flags >> HorizontalFilterShift) & InterpolationFilterMask);
set => this.flags = (byte)((this.flags & ~(InterpolationFilterMask << HorizontalFilterShift)) | ((int)value << HorizontalFilterShift));
}
} }

12
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderBlockStruct.cs

@ -65,6 +65,18 @@ internal struct Av1EncoderBlockStruct
set => this.filterIntraMode = (byte)value; set => this.filterIntraMode = (byte)value;
} }
/// <summary>
/// Gets or sets the dynamic-reference-list index selected for an inter block.
/// </summary>
/// <remarks>
/// Filter-intra and inter prediction are mutually exclusive, so both syntax branches share one packed byte.
/// </remarks>
public int ReferenceMotionVectorIndex
{
readonly get => this.filterIntraMode;
set => this.filterIntraMode = (byte)value;
}
/// <summary> /// <summary>
/// Gets the encoder prediction-unit state for the block. /// Gets the encoder prediction-unit state for the block.
/// </summary> /// </summary>

18
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPaletteMapBuffer.cs

@ -1,13 +1,12 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Buffers;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary> /// <summary>
/// Owns the reusable luma and chroma palette color-index maps for encoder block decisions. /// Exposes reusable luma and chroma palette color-index maps over superblock-workspace storage.
/// </summary> /// </summary>
internal sealed class Av1EncoderPaletteMapBuffer : IDisposable internal sealed class Av1EncoderPaletteMapBuffer : IDisposable
{ {
@ -21,21 +20,19 @@ internal sealed class Av1EncoderPaletteMapBuffer : IDisposable
/// </summary> /// </summary>
public const int StorageLength = 2 * MapLength * MapLength; public const int StorageLength = 2 * MapLength * MapLength;
private readonly IMemoryOwner<byte> owner;
private readonly Buffer2D<byte> luma; private readonly Buffer2D<byte> luma;
private readonly Buffer2D<byte> chroma; private readonly Buffer2D<byte> chroma;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderPaletteMapBuffer"/> class. /// Initializes a new instance of the <see cref="Av1EncoderPaletteMapBuffer"/> class.
/// </summary> /// </summary>
/// <param name="configuration">The configuration providing the encoder allocator.</param> /// <param name="storage">The superblock-owned backing storage.</param>
public Av1EncoderPaletteMapBuffer(Configuration configuration) public Av1EncoderPaletteMapBuffer(Memory<byte> storage)
{ {
this.owner = configuration.MemoryAllocator.Allocate<byte>(StorageLength);
int mapArea = MapLength * MapLength; int mapArea = MapLength * MapLength;
Memory<byte> storage = this.owner.Memory[..StorageLength]; Memory<byte> mapStorage = storage[..StorageLength];
this.luma = Buffer2D<byte>.WrapMemory(storage[..mapArea], MapLength, MapLength); this.luma = Buffer2D<byte>.WrapMemory(mapStorage[..mapArea], MapLength, MapLength);
this.chroma = Buffer2D<byte>.WrapMemory(storage[mapArea..], MapLength, MapLength); this.chroma = Buffer2D<byte>.WrapMemory(mapStorage[mapArea..], MapLength, MapLength);
} }
/// <summary> /// <summary>
@ -51,12 +48,11 @@ internal sealed class Av1EncoderPaletteMapBuffer : IDisposable
new Rectangle(0, 0, width, height)); new Rectangle(0, 0, width, height));
/// <summary> /// <summary>
/// Returns the shared palette-map owner to the configured allocator. /// Releases the non-owning two-dimensional wrappers.
/// </summary> /// </summary>
public void Dispose() public void Dispose()
{ {
this.luma.Dispose(); this.luma.Dispose();
this.chroma.Dispose(); this.chroma.Dispose();
this.owner.Dispose();
} }
} }

126
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs

@ -10,12 +10,18 @@ using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
/// <summary> /// <summary>
/// Owns mode information, segmentation data, and tile-neighbor contexts for one encoded AV1 picture. /// Owns reusable mode information, segmentation data, and tile-neighbor contexts for fixed-geometry AV1 pictures.
/// </summary> /// </summary>
internal sealed class Av1EncoderPictureBuffer : IDisposable internal sealed class Av1EncoderPictureBuffer : IDisposable
{ {
private readonly Av1EncoderModeInfoBuffer modeInfo; private readonly Av1EncoderModeInfoBuffer modeInfo;
private readonly IMemoryOwner<byte> stateStorage; private readonly IMemoryOwner<byte> stateStorage;
/// <summary>
/// The exact packed state region cleared between frames without touching excess pool capacity.
/// </summary>
private readonly Memory<byte> stateMemory;
private readonly ByteMemoryManager<Av1PartitionContext> partitionContextMemory; private readonly ByteMemoryManager<Av1PartitionContext> partitionContextMemory;
private readonly Av1NeighborArrayUnit<Av1PartitionContext>[] partitionContexts; private readonly Av1NeighborArrayUnit<Av1PartitionContext>[] partitionContexts;
private readonly Av1NeighborArrayUnit<byte>[] lumaCoefficientContexts; private readonly Av1NeighborArrayUnit<byte>[] lumaCoefficientContexts;
@ -40,6 +46,42 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
int width, int width,
int height, int height,
bool disallow4x4AllFrames) bool disallow4x4AllFrames)
: this(
configuration,
sequenceHeader,
frameHeader,
width,
height,
disallow4x4AllFrames,
frameHeader.AllowScreenContentTools,
frameHeader.AllowIntraBlockCopy || !frameHeader.IsIntra,
frameHeader.AllowIntraBlockCopy)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderPictureBuffer"/> class with the maximum state
/// required by a fixed-geometry sequence.
/// </summary>
/// <param name="configuration">The configuration providing picture-lifetime memory.</param>
/// <param name="sequenceHeader">The sequence header defining superblock and chroma geometry.</param>
/// <param name="frameHeader">The initial frame header defining dimensions and tiles.</param>
/// <param name="width">The visible luma width.</param>
/// <param name="height">The visible luma height.</param>
/// <param name="disallow4x4AllFrames">Whether each allocated mode-information value represents an 8x8 region.</param>
/// <param name="allocateScreenContentState">Whether palette neighbor state can be required by any frame.</param>
/// <param name="allocateMotionVectorState">Whether inter or intra-block-copy vectors can be required by any frame.</param>
/// <param name="allocateIntraBlockCopySearch">Whether intra-block-copy search state can be required by any frame.</param>
public Av1EncoderPictureBuffer(
Configuration configuration,
ObuSequenceHeader sequenceHeader,
ObuFrameHeader frameHeader,
int width,
int height,
bool disallow4x4AllFrames,
bool allocateScreenContentState,
bool allocateMotionVectorState,
bool allocateIntraBlockCopySearch)
{ {
const int ContextAlignmentLog2 = Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2; const int ContextAlignmentLog2 = Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2;
this.modeInfo = new Av1EncoderModeInfoBuffer( this.modeInfo = new Av1EncoderModeInfoBuffer(
@ -79,17 +121,17 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
int paletteLeftLength = alignedModeInfoRowCount; int paletteLeftLength = alignedModeInfoRowCount;
int paletteTopLength = this.modeInfo.ModeInfoStride; int paletteTopLength = this.modeInfo.ModeInfoStride;
int paletteContextLength = checked(paletteLeftLength + paletteTopLength); int paletteContextLength = checked(paletteLeftLength + paletteTopLength);
int paletteStorageOffset = frameHeader.AllowScreenContentTools int paletteStorageOffset = allocateScreenContentState
? Av1Math.AlignPowerOf2(byteContextStorageEnd, 1) ? Av1Math.AlignPowerOf2(byteContextStorageEnd, 1)
: byteContextStorageEnd; : byteContextStorageEnd;
int paletteStorageLength = frameHeader.AllowScreenContentTools int paletteStorageLength = allocateScreenContentState
? checked(tileCount * paletteContextLength * Unsafe.SizeOf<Av1EncoderPaletteInfo>()) ? checked(tileCount * paletteContextLength * Unsafe.SizeOf<Av1EncoderPaletteInfo>())
: 0; : 0;
int paletteStorageEnd = checked(paletteStorageOffset + paletteStorageLength); int paletteStorageEnd = checked(paletteStorageOffset + paletteStorageLength);
int displacementVectorLength = frameHeader.AllowIntraBlockCopy ? this.modeInfo.Allocation.Length : 0; int displacementVectorLength = allocateMotionVectorState ? this.modeInfo.Allocation.Length : 0;
int displacementVectorStorageOffset = frameHeader.AllowIntraBlockCopy int displacementVectorStorageOffset = allocateMotionVectorState
? Av1Math.AlignPowerOf2(paletteStorageEnd, 1) ? Av1Math.AlignPowerOf2(paletteStorageEnd, 1)
: paletteStorageEnd; : paletteStorageEnd;
@ -97,23 +139,32 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
displacementVectorLength * Unsafe.SizeOf<Av1EncoderDisplacementVector>()); displacementVectorLength * Unsafe.SizeOf<Av1EncoderDisplacementVector>());
int displacementVectorStorageEnd = checked(displacementVectorStorageOffset + displacementVectorStorageLength); int displacementVectorStorageEnd = checked(displacementVectorStorageOffset + displacementVectorStorageLength);
int intraBlockCopySearchStorageOffset = frameHeader.AllowIntraBlockCopy int intraBlockCopySearchStorageOffset = allocateIntraBlockCopySearch
? Av1Math.AlignPowerOf2(displacementVectorStorageEnd, 2) ? Av1Math.AlignPowerOf2(displacementVectorStorageEnd, 2)
: displacementVectorStorageEnd; : displacementVectorStorageEnd;
int intraBlockCopySearchStorageLength = frameHeader.AllowIntraBlockCopy int intraBlockCopySearchStorageLength = allocateIntraBlockCopySearch
? Av1IntraBlockCopySearchIndex.GetStorageLength(width, height) ? Av1IntraBlockCopySearchIndex.GetStorageLength(width, height)
: 0; : 0;
int stateStorageLength = checked(intraBlockCopySearchStorageOffset + intraBlockCopySearchStorageLength); int intraBlockCopySearchStorageEnd = checked(
intraBlockCopySearchStorageOffset + intraBlockCopySearchStorageLength);
int tileStateStorageOffset = Av1Math.AlignPowerOf2(intraBlockCopySearchStorageEnd, 2);
int cdefPresetLength = tileCount * Av1Constants.CdefUnitsPerSuperblock;
int tileStateLength = cdefPresetLength + (3 * tileCount);
int tileStateStorageLength = tileStateLength * sizeof(int);
int stateStorageLength = checked(tileStateStorageOffset + tileStateStorageLength);
// Segmentation and every tile edge share one clean picture lifetime. The partition region begins at its // Segmentation and every tile edge share one clean picture lifetime. The partition region begins at its
// native alignment, while typed views keep the entropy writer independent from the packed byte owner. // native alignment. CDEF, quantizer, and encoded-tile bounds occupy one aligned trailing integer region
// instead of allocating separate managed arrays for every picture.
this.stateStorage = configuration.MemoryAllocator.Allocate<byte>( this.stateStorage = configuration.MemoryAllocator.Allocate<byte>(
stateStorageLength, stateStorageLength,
AllocationOptions.Clean); AllocationOptions.Clean);
Memory<byte> stateStorage = this.stateStorage.Memory[..stateStorageLength]; this.stateMemory = this.stateStorage.Memory[..stateStorageLength];
Memory<byte> stateStorage = this.stateMemory;
this.partitionContextMemory = new ByteMemoryManager<Av1PartitionContext>( this.partitionContextMemory = new ByteMemoryManager<Av1PartitionContext>(
stateStorage.Slice(partitionStorageOffset, partitionStorageLength)); stateStorage.Slice(partitionStorageOffset, partitionStorageLength));
@ -125,7 +176,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
this.redCoefficientContexts = new Av1NeighborArrayUnit<byte>[tileCount]; this.redCoefficientContexts = new Av1NeighborArrayUnit<byte>[tileCount];
this.transformContexts = new Av1NeighborArrayUnit<byte>[tileCount]; this.transformContexts = new Av1NeighborArrayUnit<byte>[tileCount];
Memory<Av1EncoderPaletteInfo> paletteStorage = Memory<Av1EncoderPaletteInfo>.Empty; Memory<Av1EncoderPaletteInfo> paletteStorage = Memory<Av1EncoderPaletteInfo>.Empty;
if (frameHeader.AllowScreenContentTools) if (allocateScreenContentState)
{ {
// Palette entries contain 16-bit colors, so their packed typed region begins at an even byte offset. // Palette entries contain 16-bit colors, so their packed typed region begins at an even byte offset.
ByteMemoryManager<Av1EncoderPaletteInfo> paletteMemory = new( ByteMemoryManager<Av1EncoderPaletteInfo> paletteMemory = new(
@ -140,10 +191,10 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
} }
Memory<Av1EncoderDisplacementVector> displacementVectors = Memory<Av1EncoderDisplacementVector>.Empty; Memory<Av1EncoderDisplacementVector> displacementVectors = Memory<Av1EncoderDisplacementVector>.Empty;
if (frameHeader.AllowIntraBlockCopy) if (allocateMotionVectorState)
{ {
// Each component lies strictly inside plus or minus 16384, so two signed 16-bit fields preserve the // Each component lies strictly inside plus or minus 16384. Two signed 16-bit fields preserve both
// complete syntax domain without expanding every frame's compact mode-information allocation. // inter and intra-block-copy vectors without expanding every compact mode-information entry.
ByteMemoryManager<Av1EncoderDisplacementVector> displacementVectorMemory = new( ByteMemoryManager<Av1EncoderDisplacementVector> displacementVectorMemory = new(
stateStorage.Slice(displacementVectorStorageOffset, displacementVectorStorageLength)); stateStorage.Slice(displacementVectorStorageOffset, displacementVectorStorageLength));
@ -151,7 +202,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
} }
Av1IntraBlockCopySearchIndex intraBlockCopySearch = default; Av1IntraBlockCopySearchIndex intraBlockCopySearch = default;
if (frameHeader.AllowIntraBlockCopy) if (allocateIntraBlockCopySearch)
{ {
// The search index casts its packed workspace to 32-bit links, so its non-owning region begins at // The search index casts its packed workspace to 32-bit links, so its non-owning region begins at
// a four-byte boundary inside the existing picture-state rent. // a four-byte boundary inside the existing picture-state rent.
@ -161,8 +212,15 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
height); height);
} }
int[][] cdefPreset = new int[tileCount][]; ByteMemoryManager<int> tileStateMemory = new(
int[] previousQIndex = new int[tileCount]; stateStorage.Slice(tileStateStorageOffset, tileStateStorageLength));
Memory<int> tileState = tileStateMemory.Memory;
Memory<int> cdefPreset = tileState[..cdefPresetLength];
Memory<int> previousQIndex = tileState.Slice(cdefPresetLength, tileCount);
Memory<int> tileDataOffsets = tileState.Slice(cdefPresetLength + tileCount, tileCount);
Memory<int> tileDataLengths = tileState.Slice(cdefPresetLength + (2 * tileCount), tileCount);
cdefPreset.Span.Fill(-1);
for (int tileIndex = 0; tileIndex < tileCount; tileIndex++) for (int tileIndex = 0; tileIndex < tileCount; tileIndex++)
{ {
this.partitionContexts[tileIndex] = new Av1NeighborArrayUnit<Av1PartitionContext>( this.partitionContexts[tileIndex] = new Av1NeighborArrayUnit<Av1PartitionContext>(
@ -214,7 +272,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
this.transformContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize); this.transformContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize);
this.transformContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize); this.transformContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize);
if (frameHeader.AllowScreenContentTools) if (allocateScreenContentState)
{ {
this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>( this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>(
paletteStorage.Slice(tileIndex * paletteContextLength, paletteContextLength), paletteStorage.Slice(tileIndex * paletteContextLength, paletteContextLength),
@ -225,8 +283,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
}; };
} }
cdefPreset[tileIndex] = [-1, -1, -1, -1]; previousQIndex.Span[tileIndex] = frameHeader.QuantizationParameters.BaseQIndex;
previousQIndex[tileIndex] = frameHeader.QuantizationParameters.BaseQIndex;
} }
this.Picture = new Av1PictureControlSet this.Picture = new Av1PictureControlSet
@ -258,7 +315,9 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
IntraBlockCopySearch = intraBlockCopySearch, IntraBlockCopySearch = intraBlockCopySearch,
ModeInfoStride = this.modeInfo.ModeInfoStride, ModeInfoStride = this.modeInfo.ModeInfoStride,
Disallow4x4AllFrames = this.modeInfo.Disallow4x4AllFrames, Disallow4x4AllFrames = this.modeInfo.Disallow4x4AllFrames,
CdefPreset = cdefPreset CdefPreset = cdefPreset,
TileDataOffsets = tileDataOffsets,
TileDataLengths = tileDataLengths
}; };
} }
@ -267,6 +326,31 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
/// </summary> /// </summary>
public Av1PictureControlSet Picture { get; } public Av1PictureControlSet Picture { get; }
/// <summary>
/// Restores clean per-frame state while retaining every fixed-geometry allocation.
/// </summary>
/// <param name="frameHeader">The frame header consumed by the next encoding pass.</param>
public void Reset(ObuFrameHeader frameHeader)
{
this.modeInfo.Grid.Span.Clear();
this.modeInfo.Allocation.Span.Clear();
this.stateMemory.Span.Clear();
// Transform contexts begin at the largest transform size until an encoded neighbor publishes its
// selected size. This sentinel must be restored after the packed state owner is cleared.
foreach (Av1NeighborArrayUnit<byte> context in this.transformContexts)
{
context.Left.Fill((byte)Av1Constants.MaxTransformSize);
context.Top.Fill((byte)Av1Constants.MaxTransformSize);
}
this.Picture.CdefPreset.Span.Fill(-1);
this.Picture.Parent.PreviousQIndex.Span.Fill(frameHeader.QuantizationParameters.BaseQIndex);
this.Picture.Parent.FrameHeader = frameHeader;
this.Picture.Parent.Common.FrameSize = frameHeader.FrameSize;
this.Picture.Parent.Common.TilesInfo = frameHeader.TilesInfo;
}
/// <summary> /// <summary>
/// Returns every picture-lifetime allocation to the configured allocator. /// Returns every picture-lifetime allocation to the configured allocator.
/// </summary> /// </summary>

62
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderSuperblockWorkspace.cs

@ -3,6 +3,7 @@
using System.Buffers; using System.Buffers;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -22,14 +23,26 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable
public const int MaximumPartitionCount = 1 + 4 + 16 + 64 + 256; public const int MaximumPartitionCount = 1 + 4 + 16 + 64 + 256;
/// <summary> /// <summary>
/// The complete workspace length in packed final-block storage elements. /// The decision-region length in packed final-block storage elements.
/// </summary> /// </summary>
public const int StorageLength = MaximumFinalBlockCount + ((MaximumPartitionCount + Av1EncoderBlockStruct.StorageSize - 1) / Av1EncoderBlockStruct.StorageSize); public const int DecisionStorageLength = MaximumFinalBlockCount + ((MaximumPartitionCount + Av1EncoderBlockStruct.StorageSize - 1) / Av1EncoderBlockStruct.StorageSize);
private readonly Configuration configuration; /// <summary>
private readonly IMemoryOwner<Av1EncoderBlockStruct> owner; /// The byte length of the aligned final-block and partition decision region.
private Av1EncoderPaletteMapBuffer? paletteMaps; /// </summary>
public const int DecisionStorageByteLength = DecisionStorageLength * Av1EncoderBlockStruct.StorageSize;
/// <summary>
/// The complete byte length of the decision and palette-map regions.
/// </summary>
public const int StorageByteLength = DecisionStorageByteLength + Av1EncoderPaletteMapBuffer.StorageLength;
private const int PartitionStorageOffset = MaximumFinalBlockCount * Av1EncoderBlockStruct.StorageSize;
private readonly IMemoryOwner<byte> owner;
private readonly Av1EncoderPaletteMapBuffer paletteMaps;
private Av1EncoderPaletteInfo paletteInfo; private Av1EncoderPaletteInfo paletteInfo;
private Av1ReferenceMotionVectors referenceMotionVectors;
/// <summary> /// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderSuperblockWorkspace"/> class. /// Initializes a new instance of the <see cref="Av1EncoderSuperblockWorkspace"/> class.
@ -37,21 +50,27 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable
/// <param name="configuration">The configuration providing the encoder allocator.</param> /// <param name="configuration">The configuration providing the encoder allocator.</param>
public Av1EncoderSuperblockWorkspace(Configuration configuration) public Av1EncoderSuperblockWorkspace(Configuration configuration)
{ {
this.configuration = configuration; this.owner = configuration.MemoryAllocator.Allocate<byte>(StorageByteLength);
this.owner = configuration.MemoryAllocator.Allocate<Av1EncoderBlockStruct>(StorageLength); Memory<byte> storage = this.owner.Memory[..StorageByteLength];
// Decisions and palette maps have the same serial superblock lifetime. Keeping both regions in one
// owner preserves their distinct layouts while removing a separate palette allocation and cleanup path.
this.paletteMaps = new Av1EncoderPaletteMapBuffer(
storage.Slice(DecisionStorageByteLength, Av1EncoderPaletteMapBuffer.StorageLength));
this.Reset(); this.Reset();
} }
/// <summary> /// <summary>
/// Gets the maximum-size final-block decision span in partition traversal order. /// Gets the maximum-size final-block decision span in partition traversal order.
/// </summary> /// </summary>
public Span<Av1EncoderBlockStruct> FinalBlocks => this.owner.Memory.Span[..MaximumFinalBlockCount]; public Span<Av1EncoderBlockStruct> FinalBlocks
=> MemoryMarshal.Cast<byte, Av1EncoderBlockStruct>(this.owner.Memory.Span[..DecisionStorageByteLength])[..MaximumFinalBlockCount];
/// <summary> /// <summary>
/// Gets the maximum-size partition-type span in partition-tree preorder. /// Gets the maximum-size partition-type span in partition-tree preorder.
/// </summary> /// </summary>
public Span<byte> PartitionTypes public Span<byte> PartitionTypes => this.owner.Memory.Span.Slice(PartitionStorageOffset, MaximumPartitionCount);
=> MemoryMarshal.AsBytes(this.owner.Memory.Span[MaximumFinalBlockCount..])[..MaximumPartitionCount];
/// <summary> /// <summary>
/// Gets the palette sizes and colors selected for the block currently being written. /// Gets the palette sizes and colors selected for the block currently being written.
@ -59,20 +78,15 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable
public ref Av1EncoderPaletteInfo PaletteInfo => ref this.paletteInfo; public ref Av1EncoderPaletteInfo PaletteInfo => ref this.paletteInfo;
/// <summary> /// <summary>
/// Gets the reusable palette maps, allocating their shared owner only after a block enters palette search. /// Gets the reusable reference-vector stack used while writing inter syntax.
/// </summary> /// </summary>
/// <returns>The reusable luma and chroma palette maps.</returns> public ref Av1ReferenceMotionVectors ReferenceMotionVectors => ref this.referenceMotionVectors;
public Av1EncoderPaletteMapBuffer GetPaletteMaps()
{
Av1EncoderPaletteMapBuffer? maps = this.paletteMaps;
if (maps is null)
{
maps = new Av1EncoderPaletteMapBuffer(this.configuration);
this.paletteMaps = maps;
}
return maps; /// <summary>
} /// Gets the reusable palette maps within the superblock-workspace owner.
/// </summary>
/// <returns>The reusable luma and chroma palette maps.</returns>
public Av1EncoderPaletteMapBuffer GetPaletteMaps() => this.paletteMaps;
/// <summary> /// <summary>
/// Clears all decisions before the workspace is reused for another superblock. /// Clears all decisions before the workspace is reused for another superblock.
@ -86,7 +100,7 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable
}; };
this.FinalBlocks.Fill(initialBlock); this.FinalBlocks.Fill(initialBlock);
MemoryMarshal.AsBytes(this.owner.Memory.Span[MaximumFinalBlockCount..]).Clear(); this.PartitionTypes.Clear();
this.paletteInfo = default; this.paletteInfo = default;
} }
@ -95,7 +109,7 @@ internal sealed class Av1EncoderSuperblockWorkspace : IDisposable
/// </summary> /// </summary>
public void Dispose() public void Dispose()
{ {
this.paletteMaps?.Dispose(); this.paletteMaps.Dispose();
this.owner.Dispose(); this.owner.Dispose();
} }
} }

15
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs

@ -86,9 +86,20 @@ internal class Av1PictureControlSet
public bool Disallow4x4AllFrames { get; set; } public bool Disallow4x4AllFrames { get; set; }
/// <summary> /// <summary>
/// Gets or sets the constrained directional enhancement filter presets for each filter block. /// Gets or sets the constrained directional enhancement filter presets for each tile.
/// Each tile occupies <see cref="Av1Constants.CdefUnitsPerSuperblock"/> consecutive entries.
/// </summary> /// </summary>
public required int[][] CdefPreset { get; set; } public required Memory<int> CdefPreset { get; set; }
/// <summary>
/// Gets or sets the starting byte of each tile in the shared encoded output buffer.
/// </summary>
public required Memory<int> TileDataOffsets { get; set; }
/// <summary>
/// Gets or sets the encoded byte length of each tile.
/// </summary>
public required Memory<int> TileDataLengths { get; set; }
/// <summary> /// <summary>
/// Gets the mode-information entry mapped to a frame position. /// Gets the mode-information entry mapped to a frame position.

2
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs

@ -23,7 +23,7 @@ internal class Av1PictureParentControlSet
/// <summary> /// <summary>
/// Gets or sets the preceding quantizer index for each tile context. /// Gets or sets the preceding quantizer index for each tile context.
/// </summary> /// </summary>
public required int[] PreviousQIndex { get; set; } public required Memory<int> PreviousQIndex { get; set; }
/// <summary> /// <summary>
/// Gets or sets the encoder palette-search level. /// Gets or sets the encoder palette-search level.

9
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs

@ -134,7 +134,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable
/// <summary> /// <summary>
/// Reusable fixed-capacity storage for one block's weighted reference-motion-vector candidates. /// Reusable fixed-capacity storage for one block's weighted reference-motion-vector candidates.
/// </summary> /// </summary>
private readonly Av1ReferenceMotionVectors referenceMotionVectors = new(); private Av1ReferenceMotionVectors referenceMotionVectors;
/// <summary> /// <summary>
/// Reusable fixed-capacity state for motion-mode eligibility and local warped-motion projection. /// Reusable fixed-capacity state for motion-mode eligibility and local warped-motion projection.
@ -250,6 +250,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable
Av1ReferenceFrameStore? referenceFrames, Av1ReferenceFrameStore? referenceFrames,
PaletteColorIndexMaps? sharedPaletteColorIndexMaps) PaletteColorIndexMaps? sharedPaletteColorIndexMaps)
{ {
this.referenceMotionVectors = default;
this.FrameHeader = frameHeader; this.FrameHeader = frameHeader;
this.configuration = configuration; this.configuration = configuration;
this.SequenceHeader = sequenceHeader; this.SequenceHeader = sequenceHeader;
@ -2044,7 +2045,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable
Av1ReferenceFrameType secondaryReferenceFrame = modeInfo.ReferenceFrames[1]; Av1ReferenceFrameType secondaryReferenceFrame = modeInfo.ReferenceFrames[1];
bool isCompound = secondaryReferenceFrame > Av1ReferenceFrameType.Intra; bool isCompound = secondaryReferenceFrame > Av1ReferenceFrameType.Intra;
Av1ReferenceMotionVectors referenceMotionVectors = this.referenceMotionVectors; ref Av1ReferenceMotionVectors referenceMotionVectors = ref this.referenceMotionVectors;
referenceMotionVectors.Build( referenceMotionVectors.Build(
ref partitionInfo, ref partitionInfo,
tileInfo, tileInfo,
@ -2110,9 +2111,7 @@ internal sealed class Av1TileReader : IAv1TileReader, IDisposable
} }
} }
Av1MotionVectorPrecision precision = this.FrameHeader.ForceIntegerMotionVector Av1MotionVectorPrecision precision = this.FrameHeader.MotionVectorPrecision;
? Av1MotionVectorPrecision.Integer
: this.FrameHeader.AllowHighPrecisionMotionVector ? Av1MotionVectorPrecision.EighthSample : Av1MotionVectorPrecision.QuarterSample;
Span<Av1MotionVector> motionVectors = modeInfo.MotionVectors; Span<Av1MotionVector> motionVectors = modeInfo.MotionVectors;
if (!isCompound) if (!isCompound)

337
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs

@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
@ -832,18 +833,35 @@ internal partial class Av1TileWriter
bool skipWritingCoefficients = macroBlockModeInfo.Block.Skip; bool skipWritingCoefficients = macroBlockModeInfo.Block.Skip;
// This encoder path currently writes intra frames only, so every block follows the key-frame mode syntax. // Segmentation, skip, filter, and quantizer syntax precede the prediction-domain branch in both
// intra and inter frames. Keeping this prefix shared preserves the decoder's symbol order.
{ {
if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip)
{ {
WriteSegmentId(pcs, writer, blockSize, blockOrigin, macroBlock, ref blk_ptr, skipWritingCoefficients); WriteSegmentId(
pcs,
writer,
blockSize,
blockOrigin,
macroBlock,
ref blk_ptr,
skipWritingCoefficients,
beforeSkip: true);
} }
EncodeSkipCoefficients(writer, macroBlock, skipWritingCoefficients); EncodeSkipCoefficients(writer, macroBlock, skipWritingCoefficients);
if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && !pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip) if (pcs.Parent.FrameHeader.SegmentationParameters.Enabled && !pcs.Parent.FrameHeader.SegmentationParameters.SegmentIdPrecedesSkip)
{ {
WriteSegmentId(pcs, writer, blockSize, blockOrigin, macroBlock, ref blk_ptr, skipWritingCoefficients); WriteSegmentId(
pcs,
writer,
blockSize,
blockOrigin,
macroBlock,
ref blk_ptr,
skipWritingCoefficients,
beforeSkip: false);
} }
WriteCdef( WriteCdef(
@ -862,17 +880,122 @@ internal partial class Av1TileWriter
if ((blockSize != scs.SequenceHeader.SuperblockSize || !skipWritingCoefficients) && super_block_upper_left) if ((blockSize != scs.SequenceHeader.SuperblockSize || !skipWritingCoefficients) && super_block_upper_left)
{ {
Guard.MustBeGreaterThan(current_q_index, 0, nameof(current_q_index)); Guard.MustBeGreaterThan(current_q_index, 0, nameof(current_q_index));
int reduced_delta_qindex = (current_q_index - pcs.Parent.PreviousQIndex[tile_idx]) / int reduced_delta_qindex = (current_q_index - pcs.Parent.PreviousQIndex.Span[tile_idx]) /
frm_hdr.DeltaQParameters.Resolution; frm_hdr.DeltaQParameters.Resolution;
writer.WriteDeltaQuantizerIndex(reduced_delta_qindex); writer.WriteDeltaQuantizerIndex(reduced_delta_qindex);
pcs.Parent.PreviousQIndex[tile_idx] = current_q_index; pcs.Parent.PreviousQIndex.Span[tile_idx] = current_q_index;
} }
} }
Av1PredictionMode intra_luma_mode = macroBlockModeInfo.Block.Mode; bool isInterBlock = macroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra;
bool isGlobalMotionForced = false;
bool isReferenceForced = false;
if (!frm_hdr.IsIntra)
{
ObuSegmentationParameters segmentation = frm_hdr.SegmentationParameters;
int segmentId = macroBlockModeInfo.Block.SegmentId;
isReferenceForced = segmentation.IsFeatureActive(
segmentId,
ObuSegmentationLevelFeature.ReferenceFrame);
isGlobalMotionForced = segmentation.IsFeatureActive(
segmentId,
ObuSegmentationLevelFeature.GlobalMotionVector);
if (!isReferenceForced && !isGlobalMotionForced)
{
int intraInterContext = GetIntraInterContext(macroBlock);
writer.WriteIsInter(isInterBlock, intraInterContext);
}
}
Av1PredictionMode lumaMode = macroBlockModeInfo.Block.Mode;
Av1ChromaPredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode; Av1ChromaPredictionMode intra_chroma_mode = macroBlockModeInfo.Block.UvMode;
if (IsIntraBlockCopyAllowed(pcs.Parent.FrameHeader/*, pcs.Parent.SliceType*/)) if (isInterBlock)
{
if (!isReferenceForced && !isGlobalMotionForced)
{
Span<byte> referenceCounts = stackalloc byte[Av1Constants.ReferenceFrameCount];
CollectNeighborReferenceCounts(macroBlock, referenceCounts);
writer.WriteSingleReference(
macroBlockModeInfo.Block.ReferenceFrame,
referenceCounts);
}
if (!isGlobalMotionForced)
{
ref Av1ReferenceMotionVectors referenceMotionVectors = ref tb_ptr.Workspace.ReferenceMotionVectors;
referenceMotionVectors.Build(
pcs,
macroBlock,
modeInfoPosition,
blockSize,
macroBlockModeInfo.Block.PartitionType,
scs.SequenceHeader,
frm_hdr,
macroBlockModeInfo.Block.ReferenceFrame);
writer.WriteInterMode(lumaMode, referenceMotionVectors.ModeContext);
int referenceMotionVectorIndex = blk_ptr.ReferenceMotionVectorIndex;
if (lumaMode == Av1PredictionMode.NearMotionVector)
{
// NEARMV reserves stack entry zero for NEARESTMV, so its DRL decisions advance from
// near entry zero to one and then from one to two.
for (int index = 1; index < 3 && referenceMotionVectors.Count > index + 1; index++)
{
bool advance = referenceMotionVectorIndex >= index;
int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index);
writer.WriteDynamicReferenceList(advance, context);
if (!advance)
{
break;
}
}
}
else if (lumaMode == Av1PredictionMode.NewMotionVector)
{
// NEWMV begins at stack entry zero and can advance through entries one and two.
for (int index = 0; index < 2 && referenceMotionVectors.Count > index + 1; index++)
{
bool advance = referenceMotionVectorIndex > index;
int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index);
writer.WriteDynamicReferenceList(advance, context);
if (!advance)
{
break;
}
}
Av1MotionVector vector = pcs.GetDisplacementVector(modeInfoPosition);
writer.WriteMotionVector(
vector,
referenceMotionVectors.GetNewReference(referenceMotionVectorIndex),
frm_hdr.MotionVectorPrecision);
}
}
if (UsesSwitchableInterpolation(frm_hdr, macroBlockModeInfo.Block))
{
// The vertical symbol is first and supplies both axes unless the sequence enables dual filters.
int verticalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext(
macroBlockModeInfo.Block,
macroBlock,
direction: 0);
writer.WriteSwitchableInterpolationFilter(macroBlockModeInfo.Block.VerticalInterpolationFilter, verticalContext);
if (scs.SequenceHeader.EnableDualFilter)
{
int horizontalContext = Av1SymbolContextHelper.GetSwitchableInterpolationContext(
macroBlockModeInfo.Block,
macroBlock,
direction: 1);
writer.WriteSwitchableInterpolationFilter(macroBlockModeInfo.Block.HorizontalInterpolationFilter, horizontalContext);
}
}
}
else if (IsIntraBlockCopyAllowed(pcs.Parent.FrameHeader/*, pcs.Parent.SliceType*/))
{ {
WriteIntraBlockCopyInfo( WriteIntraBlockCopyInfo(
pcs, pcs,
@ -882,12 +1005,19 @@ internal partial class Av1TileWriter
macroBlockModeInfo); macroBlockModeInfo);
} }
if (!macroBlockModeInfo.Block.UseIntraBlockCopy) if (!isInterBlock && !macroBlockModeInfo.Block.UseIntraBlockCopy)
{ {
EncodeIntraLumaMode(writer, macroBlockModeInfo, macroBlock, ref blk_ptr, blockSize, intra_luma_mode); EncodeIntraLumaMode(
writer,
frm_hdr,
macroBlockModeInfo,
macroBlock,
ref blk_ptr,
blockSize,
lumaMode);
} }
if (!macroBlockModeInfo.Block.UseIntraBlockCopy) if (!isInterBlock && !macroBlockModeInfo.Block.UseIntraBlockCopy)
{ {
if (blk_ptr.HasChroma) if (blk_ptr.HasChroma)
{ {
@ -898,12 +1028,13 @@ internal partial class Av1TileWriter
macroBlockModeInfo, macroBlockModeInfo,
ref blk_ptr, ref blk_ptr,
blockSize, blockSize,
intra_luma_mode, lumaMode,
intra_chroma_mode); intra_chroma_mode);
} }
} }
bool paletteAllowed = !macroBlockModeInfo.Block.UseIntraBlockCopy && bool paletteAllowed = !isInterBlock &&
!macroBlockModeInfo.Block.UseIntraBlockCopy &&
IsPaletteAllowed(frm_hdr.AllowScreenContentTools, blockSize); IsPaletteAllowed(frm_hdr.AllowScreenContentTools, blockSize);
if (paletteAllowed) if (paletteAllowed)
@ -921,12 +1052,13 @@ internal partial class Av1TileWriter
blk_ptr.HasChroma); blk_ptr.HasChroma);
} }
if (!macroBlockModeInfo.Block.UseIntraBlockCopy && if (!isInterBlock &&
!macroBlockModeInfo.Block.UseIntraBlockCopy &&
IsFilterIntraAllowed( IsFilterIntraAllowed(
scs.SequenceHeader.EnableFilterIntra, scs.SequenceHeader.EnableFilterIntra,
blockSize, blockSize,
paletteInfo.PaletteSizes[0], paletteInfo.PaletteSizes[0],
intra_luma_mode)) lumaMode))
{ {
writer.WriteFilterIntraMode(blk_ptr.FilterIntraMode, blockSize); writer.WriteFilterIntraMode(blk_ptr.FilterIntraMode, blockSize);
} }
@ -983,7 +1115,7 @@ internal partial class Av1TileWriter
writer, writer,
ref blk_ptr, ref blk_ptr,
blockOrigin, blockOrigin,
intra_luma_mode, lumaMode,
blockSize, blockSize,
coefficientBuffer, coefficientBuffer,
tb_ptr.Index, tb_ptr.Index,
@ -1035,7 +1167,8 @@ internal partial class Av1TileWriter
ref Av1MacroBlockModeInfo aboveModeInfo = ref Av1MacroBlockModeInfo aboveModeInfo =
ref macroBlock.GetRelativeModeInfo(-macroBlock.ModeInfoStride); ref macroBlock.GetRelativeModeInfo(-macroBlock.ModeInfoStride);
if (aboveModeInfo.Block.UseIntraBlockCopy) if (aboveModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra ||
aboveModeInfo.Block.UseIntraBlockCopy)
{ {
above = aboveModeInfo.Block.BlockSize.GetWidth() >= maximumTransformSize.GetWidth() ? 1 : 0; above = aboveModeInfo.Block.BlockSize.GetWidth() >= maximumTransformSize.GetWidth() ? 1 : 0;
} }
@ -1044,7 +1177,8 @@ internal partial class Av1TileWriter
if (macroBlock.IsLeftAvailable) if (macroBlock.IsLeftAvailable)
{ {
ref Av1MacroBlockModeInfo leftModeInfo = ref macroBlock.GetRelativeModeInfo(-1); ref Av1MacroBlockModeInfo leftModeInfo = ref macroBlock.GetRelativeModeInfo(-1);
if (leftModeInfo.Block.UseIntraBlockCopy) if (leftModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra ||
leftModeInfo.Block.UseIntraBlockCopy)
{ {
left = leftModeInfo.Block.BlockSize.GetHeight() >= maximumTransformSize.GetHeight() ? 1 : 0; left = leftModeInfo.Block.BlockSize.GetHeight() >= maximumTransformSize.GetHeight() ? 1 : 0;
} }
@ -1076,7 +1210,8 @@ internal partial class Av1TileWriter
{ {
ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; ObuFrameHeader frameHeader = pcs.Parent.FrameHeader;
bool isLossless = frameHeader.LosslessArray[macroBlockModeInfo.Block.SegmentId]; bool isLossless = frameHeader.LosslessArray[macroBlockModeInfo.Block.SegmentId];
bool isInter = macroBlockModeInfo.Block.UseIntraBlockCopy; bool isInter = macroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra ||
macroBlockModeInfo.Block.UseIntraBlockCopy;
bool writesUniformTransformSize = !isLossless && bool writesUniformTransformSize = !isLossless &&
frameHeader.TransformMode == Av1TransformMode.Select && frameHeader.TransformMode == Av1TransformMode.Select &&
!isInter && !isInter &&
@ -1110,7 +1245,7 @@ internal partial class Av1TileWriter
blockSize, blockSize,
blockSize.GetMaximumTransformSize()); blockSize.GetMaximumTransformSize());
// Intra-block copy currently retains the maximum transform, so its variable-transform tree has one unsplit root. // Current inter decisions retain the maximum transform, so their variable-transform tree has one unsplit root.
writer.WriteTransformPartition(false, context); writer.WriteTransformPartition(false, context);
} }
@ -1247,23 +1382,34 @@ internal partial class Av1TileWriter
} }
/// <summary> /// <summary>
/// Gets the key-frame luma mode rate against the current neighboring modes and tile probabilities. /// Gets the luma mode rate from the frame-appropriate distribution.
/// </summary> /// </summary>
/// <param name="writer">The live tile symbol encoder.</param> /// <param name="writer">The live tile symbol encoder.</param>
/// <param name="macroBlock">The current block's mapped neighbor state.</param> /// <param name="macroBlock">The current block's mapped neighbor state.</param>
/// <param name="blockSize">The selected block size.</param> /// <param name="blockSize">The selected block size.</param>
/// <param name="mode">The candidate luma mode.</param> /// <param name="mode">The candidate luma mode.</param>
/// <param name="angleDelta">The signed directional-angle adjustment.</param> /// <param name="angleDelta">The signed directional-angle adjustment.</param>
/// <param name="isIntraFrame">Whether the frame uses key-frame neighbor-conditioned mode syntax.</param>
/// <returns>The luma mode and directional-angle rate in 1/512-bit units.</returns> /// <returns>The luma mode and directional-angle rate in 1/512-bit units.</returns>
public static int GetLumaModeCost( public static int GetLumaModeCost(
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock, Av1MacroBlockD macroBlock,
Av1BlockSize blockSize, Av1BlockSize blockSize,
Av1PredictionMode mode, Av1PredictionMode mode,
int angleDelta) int angleDelta,
bool isIntraFrame)
{ {
GetYModeContext(macroBlock, out byte topContext, out byte leftContext); int cost;
int cost = writer.GetLumaModeCost(mode, topContext, leftContext); if (isIntraFrame)
{
GetYModeContext(macroBlock, out byte topContext, out byte leftContext);
cost = writer.GetLumaModeCost(mode, topContext, leftContext);
}
else
{
cost = writer.GetInterFrameLumaModeCost(mode, blockSize);
}
if (blockSize >= Av1BlockSize.Block8x8 && mode.IsDirectional()) if (blockSize >= Av1BlockSize.Block8x8 && mode.IsDirectional())
{ {
cost += writer.GetAngleDeltaCost(angleDelta + Av1Constants.MaxAngleDelta, mode); cost += writer.GetAngleDeltaCost(angleDelta + Av1Constants.MaxAngleDelta, mode);
@ -1273,9 +1419,10 @@ internal partial class Av1TileWriter
} }
/// <summary> /// <summary>
/// Writes the key-frame luma prediction mode and any directional angle adjustment. /// Writes the frame-appropriate luma prediction mode and any directional angle adjustment.
/// </summary> /// </summary>
/// <param name="writer">The tile symbol encoder.</param> /// <param name="writer">The tile symbol encoder.</param>
/// <param name="frameHeader">The frame header that selects the luma-mode probability model.</param>
/// <param name="macroBlockModeInfo">The selected block modes.</param> /// <param name="macroBlockModeInfo">The selected block modes.</param>
/// <param name="macroBlock">The reusable macroblock edge and neighbor state.</param> /// <param name="macroBlock">The reusable macroblock edge and neighbor state.</param>
/// <param name="blk_ptr">The encoder prediction-unit state.</param> /// <param name="blk_ptr">The encoder prediction-unit state.</param>
@ -1283,14 +1430,22 @@ internal partial class Av1TileWriter
/// <param name="lumaMode">The selected luma prediction mode.</param> /// <param name="lumaMode">The selected luma prediction mode.</param>
private static void EncodeIntraLumaMode( private static void EncodeIntraLumaMode(
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
ObuFrameHeader frameHeader,
Av1MacroBlockModeInfo macroBlockModeInfo, Av1MacroBlockModeInfo macroBlockModeInfo,
Av1MacroBlockD macroBlock, Av1MacroBlockD macroBlock,
ref Av1EncoderBlockStruct blk_ptr, ref Av1EncoderBlockStruct blk_ptr,
Av1BlockSize blockSize, Av1BlockSize blockSize,
Av1PredictionMode lumaMode) Av1PredictionMode lumaMode)
{ {
GetYModeContext(macroBlock, out byte topContext, out byte leftContext); if (frameHeader.IsIntra)
writer.WriteLumaMode(lumaMode, topContext, leftContext); {
GetYModeContext(macroBlock, out byte topContext, out byte leftContext);
writer.WriteLumaMode(lumaMode, topContext, leftContext);
}
else
{
writer.WriteInterFrameLumaMode(lumaMode, blockSize);
}
if (blockSize >= Av1BlockSize.Block8x8 && macroBlockModeInfo.Block.Mode.IsDirectional()) if (blockSize >= Av1BlockSize.Block8x8 && macroBlockModeInfo.Block.Mode.IsDirectional())
{ {
@ -1298,6 +1453,91 @@ internal partial class Av1TileWriter
} }
} }
/// <summary>
/// Gets the prediction-domain context from the immediately above and left encoder blocks.
/// </summary>
/// <param name="macroBlock">The current block's mapped neighbor state.</param>
/// <returns>The context in the inclusive range zero through three.</returns>
public static int GetIntraInterContext(Av1MacroBlockD macroBlock)
{
bool hasAbove = macroBlock.IsUpAvailable;
bool hasLeft = macroBlock.IsLeftAvailable;
if (hasAbove && hasLeft)
{
bool aboveIsIntra = macroBlock
.GetRelativeModeInfo(-macroBlock.ModeInfoStride)
.Block.ReferenceFrame <= Av1ReferenceFrameType.Intra;
bool leftIsIntra = macroBlock
.GetRelativeModeInfo(-1)
.Block.ReferenceFrame <= Av1ReferenceFrameType.Intra;
if (aboveIsIntra && leftIsIntra)
{
return 3;
}
return aboveIsIntra || leftIsIntra ? 1 : 0;
}
if (hasAbove)
{
return macroBlock
.GetRelativeModeInfo(-macroBlock.ModeInfoStride)
.Block.ReferenceFrame <= Av1ReferenceFrameType.Intra
? 2
: 0;
}
if (hasLeft)
{
return macroBlock
.GetRelativeModeInfo(-1)
.Block.ReferenceFrame <= Av1ReferenceFrameType.Intra
? 2
: 0;
}
return 0;
}
/// <summary>
/// Counts the single-reference labels used by the immediately above and left encoded blocks.
/// </summary>
/// <param name="macroBlock">The current block's mapped neighbor state.</param>
/// <param name="referenceCounts">The eight-entry destination indexed by reference-frame label.</param>
public static void CollectNeighborReferenceCounts(
Av1MacroBlockD macroBlock,
Span<byte> referenceCounts)
{
// The caller supplies short-lived fixed storage for one block. Clearing it here keeps unavailable
// neighbors from retaining votes collected for a preceding block.
referenceCounts.Clear();
if (macroBlock.IsUpAvailable)
{
Av1ReferenceFrameType referenceFrame = macroBlock
.GetRelativeModeInfo(-macroBlock.ModeInfoStride)
.Block.ReferenceFrame;
if (referenceFrame > Av1ReferenceFrameType.Intra)
{
referenceCounts[(int)referenceFrame]++;
}
}
if (macroBlock.IsLeftAvailable)
{
Av1ReferenceFrameType referenceFrame = macroBlock
.GetRelativeModeInfo(-1)
.Block.ReferenceFrame;
if (referenceFrame > Av1ReferenceFrameType.Intra)
{
referenceCounts[(int)referenceFrame]++;
}
}
}
/// <summary> /// <summary>
/// Writes luma and chroma palette-mode syntax for a block. /// Writes luma and chroma palette-mode syntax for a block.
/// </summary> /// </summary>
@ -1498,6 +1738,26 @@ internal partial class Av1TileWriter
} }
} }
/// <summary>
/// Determines whether a single-reference encoder block carries switchable interpolation symbols.
/// </summary>
/// <param name="frameHeader">The current frame header.</param>
/// <param name="modeInfo">The selected block syntax.</param>
/// <returns>Whether the block writes a vertical filter and, when enabled, a horizontal filter.</returns>
public static bool UsesSwitchableInterpolation(ObuFrameHeader frameHeader, Av1EncoderBlockModeInfo modeInfo)
{
if (frameHeader.InterpolationFilter != Av1InterpolationFilter.Switchable || modeInfo.SkipMode)
{
return false;
}
// Global identity and affine models infer the regular filter on blocks at least 8x8. Translation still
// carries filter symbols, including integer translations. Residual skip does not suppress these symbols.
return modeInfo.Mode != Av1PredictionMode.GlobalMotionVector ||
Math.Min(modeInfo.BlockSize.GetWidth(), modeInfo.BlockSize.GetHeight()) < Av1BlockSize.Block8x8.GetWidth() ||
frameHeader.GetGlobalMotionParameters()[(int)modeInfo.ReferenceFrame - 1].Type == Av1GlobalMotionType.Translation;
}
/// <summary> /// <summary>
/// Determines whether the current frame permits intra block copy. /// Determines whether the current frame permits intra block copy.
/// </summary> /// </summary>
@ -1603,14 +1863,15 @@ internal partial class Av1TileWriter
return; return;
} }
Span<int> cdefPreset = pcs.CdefPreset.Span.Slice(
tileIndex * Av1Constants.CdefUnitsPerSuperblock,
Av1Constants.CdefUnitsPerSuperblock);
// Each superblock begins with all contained 64x64 filter units unassigned. // Each superblock begins with all contained 64x64 filter units unassigned.
if ((modeInfoPosition.Y & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0 && if ((modeInfoPosition.Y & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0 &&
(modeInfoPosition.X & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0) (modeInfoPosition.X & (scs.SequenceHeader.SuperblockModeInfoSize - 1)) == 0)
{ {
pcs.CdefPreset[tileIndex][0] = -1; cdefPreset.Fill(-1);
pcs.CdefPreset[tileIndex][1] = -1;
pcs.CdefPreset[tileIndex][2] = -1;
pcs.CdefPreset[tileIndex][3] = -1;
} }
// The strength is coded once, at the first non-skipped block in each 64x64 CDEF filter unit. // The strength is coded once, at the first non-skipped block in each 64x64 CDEF filter unit.
@ -1619,7 +1880,7 @@ internal partial class Av1TileWriter
int unitRow = (modeInfoPosition.Y & cdefSize) != 0 ? 1 : 0; int unitRow = (modeInfoPosition.Y & cdefSize) != 0 ? 1 : 0;
int index = scs.SequenceHeader.Use128x128Superblock ? unitColumn + (2 * unitRow) : 0; int index = scs.SequenceHeader.Use128x128Superblock ? unitColumn + (2 * unitRow) : 0;
if (pcs.CdefPreset[tileIndex][index] == -1 && !skip) if (cdefPreset[index] == -1 && !skip)
{ {
int firstBlockMask = ~(cdefSize - 1); int firstBlockMask = ~(cdefSize - 1);
Point firstBlockPosition = new( Point firstBlockPosition = new(
@ -1630,7 +1891,7 @@ internal partial class Av1TileWriter
// CDEF strength belongs to the first mode-info block in the 64x64 filter unit even when skipped // CDEF strength belongs to the first mode-info block in the 64x64 filter unit even when skipped
// blocks delay transmission until a later coding block. // blocks delay transmission until a later coding block.
writer.WriteCdefStrength(firstBlock.CdefStrength, frameHeader.CdefParameters.BitCount); writer.WriteCdefStrength(firstBlock.CdefStrength, frameHeader.CdefParameters.BitCount);
pcs.CdefPreset[tileIndex][index] = firstBlock.CdefStrength; cdefPreset[index] = firstBlock.CdefStrength;
} }
} }
@ -2036,7 +2297,9 @@ internal partial class Av1TileWriter
int transformBlockHeight = transformSize.Get4x4HighCount(); int transformBlockHeight = transformSize.Get4x4HighCount();
int transformWidth = transformSize.GetWidth(); int transformWidth = transformSize.GetWidth();
int transformHeight = transformSize.GetHeight(); int transformHeight = transformSize.GetHeight();
bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy; bool usesInterTransformSet =
entropyCodingContext.MacroBlockModeInfo.Block.ReferenceFrame > Av1ReferenceFrameType.Intra ||
entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy;
Av1ComponentType componentType = isLuma Av1ComponentType componentType = isLuma
? Av1ComponentType.Luminance ? Av1ComponentType.Luminance
: Av1ComponentType.Chroma; : Av1ComponentType.Chroma;
@ -2240,6 +2503,7 @@ internal partial class Av1TileWriter
/// <param name="macroBlock">The reusable macroblock edge and neighbor state.</param> /// <param name="macroBlock">The reusable macroblock edge and neighbor state.</param>
/// <param name="block">The encoder block state.</param> /// <param name="block">The encoder block state.</param>
/// <param name="skip">A value indicating whether residual coefficients are omitted.</param> /// <param name="skip">A value indicating whether residual coefficients are omitted.</param>
/// <param name="beforeSkip">Whether the segment identifier is written before the skip flag.</param>
private static void WriteSegmentId( private static void WriteSegmentId(
Av1PictureControlSet pcs, Av1PictureControlSet pcs,
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
@ -2247,7 +2511,8 @@ internal partial class Av1TileWriter
Point blockOrigin, Point blockOrigin,
Av1MacroBlockD macroBlock, Av1MacroBlockD macroBlock,
ref Av1EncoderBlockStruct block, ref Av1EncoderBlockStruct block,
bool skip) bool skip,
bool beforeSkip)
{ {
ObuSegmentationParameters segmentation_params = pcs.Parent.FrameHeader.SegmentationParameters; ObuSegmentationParameters segmentation_params = pcs.Parent.FrameHeader.SegmentationParameters;
if (!segmentation_params.Enabled) if (!segmentation_params.Enabled)
@ -2256,9 +2521,9 @@ internal partial class Av1TileWriter
} }
int spatial_pred = GetSpatialSegmentationPrediction(pcs, macroBlock, blockOrigin, out int cdf_num); int spatial_pred = GetSpatialSegmentationPrediction(pcs, macroBlock, blockOrigin, out int cdf_num);
if (skip) if (!beforeSkip && skip)
{ {
// With segment-id-before-skip syntax, a skipped block inherits the spatial predictor without coding a residual ID. // Post-skip segment syntax can infer the spatial predictor once the decoder already knows the block is skipped.
pcs.UpdateSegmentation(blockSize, blockOrigin, spatial_pred); pcs.UpdateSegmentation(blockSize, blockOrigin, spatial_pred);
block.SegmentId = spatial_pred; block.SegmentId = spatial_pred;
return; return;

102
src/ImageSharp/Formats/Heif/Components/Alpha/HeifPlanarAlphaEncoder.cs

@ -15,6 +15,28 @@ namespace SixLabors.ImageSharp.Formats.Heif.Components.Alpha;
/// </summary> /// </summary>
internal static class HeifPlanarAlphaEncoder internal static class HeifPlanarAlphaEncoder
{ {
/// <summary>
/// The number of float elements sharing storage with one packed <see cref="Rgba64"/> value.
/// </summary>
private const int Rgba64FloatElementCount = 2;
/// <summary>
/// The additional float element receiving one extracted alpha value.
/// </summary>
private const int AlphaFloatElementCount = 1;
/// <summary>
/// The complete reusable row-storage length per source pixel.
/// </summary>
private const int RowFloatElementCount = Rgba64FloatElementCount + AlphaFloatElementCount;
/// <summary>
/// Gets the reusable row-storage length required for the specified source width.
/// </summary>
/// <param name="width">The source-row width.</param>
/// <returns>The required number of float elements.</returns>
public static int GetRowStorageLength(int width) => width * RowFloatElementCount;
/// <summary> /// <summary>
/// Converts one packed image frame into a full-range native alpha plane. /// Converts one packed image frame into a full-range native alpha plane.
/// </summary> /// </summary>
@ -34,19 +56,85 @@ internal static class HeifPlanarAlphaEncoder
where TSample : unmanaged where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample> where TStorer : struct, IHeifSampleConverter<TSample>
{ {
int width = image.Width; Rectangle sourceRectangle = new(0, 0, image.Width, image.Height);
Convert<TPixel, TBuffer, TSample, TStorer>(configuration, image, sourceRectangle, buffer);
}
/// <summary>
/// Converts one packed image region into a full-range native alpha plane.
/// </summary>
/// <typeparam name="TPixel">The packed source pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the destination plane.</typeparam>
/// <typeparam name="TSample">The native unsigned sample storage type.</typeparam>
/// <typeparam name="TStorer">The SIMD narrowing and storage operations for the sample type.</typeparam>
/// <param name="configuration">The configuration used for row allocation and pixel conversion.</param>
/// <param name="image">The packed source image frame.</param>
/// <param name="sourceRectangle">The source region mapped to the complete destination plane.</param>
/// <param name="buffer">The monochrome destination buffer.</param>
public static void Convert<TPixel, TBuffer, TSample, TStorer>(
Configuration configuration,
ImageFrame<TPixel> image,
Rectangle sourceRectangle,
TBuffer buffer)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample>
{
int width = sourceRectangle.Width;
// Rgba64 preserves the source pixel's normalized alpha precision before quantization to the requested AV1 // Rgba64 preserves the source pixel's normalized alpha precision before quantization to the requested AV1
// depth. Both row views share one owner because their lifetimes never escape this conversion operation. // depth. Both row views share one owner because their lifetimes never escape this conversion operation.
using IMemoryOwner<float> rowOwner = configuration.MemoryAllocator.Allocate<float>(width * 3); using IMemoryOwner<float> rowOwner = configuration.MemoryAllocator.Allocate<float>(
Span<float> rowStorage = rowOwner.GetSpan(); GetRowStorageLength(width));
Span<Rgba64> packed = MemoryMarshal.Cast<float, Rgba64>(rowStorage[..(width * 2)]);
Span<float> alpha = rowStorage.Slice(width * 2, width); Convert<TPixel, TBuffer, TSample, TStorer>(
configuration,
image,
sourceRectangle,
buffer,
rowOwner.GetSpan());
}
/// <summary>
/// Converts one packed image region using caller-owned reusable row storage.
/// </summary>
/// <typeparam name="TPixel">The packed source pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the destination plane.</typeparam>
/// <typeparam name="TSample">The native unsigned sample storage type.</typeparam>
/// <typeparam name="TStorer">The SIMD narrowing and storage operations for the sample type.</typeparam>
/// <param name="configuration">The configuration used for pixel conversion.</param>
/// <param name="image">The packed source image frame.</param>
/// <param name="sourceRectangle">The source region mapped to the complete destination plane.</param>
/// <param name="buffer">The monochrome destination buffer.</param>
/// <param name="rowStorage">Storage for one packed high-precision row and its extracted alpha values.</param>
public static void Convert<TPixel, TBuffer, TSample, TStorer>(
Configuration configuration,
ImageFrame<TPixel> image,
Rectangle sourceRectangle,
TBuffer buffer,
Span<float> rowStorage)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample>
{
int width = sourceRectangle.Width;
Span<Rgba64> packed = MemoryMarshal.Cast<float, Rgba64>(
rowStorage[..(width * Rgba64FloatElementCount)]);
Span<float> alpha = rowStorage.Slice(
width * Rgba64FloatElementCount,
width * AlphaFloatElementCount);
float maximum = (1 << buffer.LumaBitDepth) - 1; float maximum = (1 << buffer.LumaBitDepth) - 1;
float scale = maximum / ushort.MaxValue; float scale = maximum / ushort.MaxValue;
for (int y = 0; y < image.Height; y++) for (int y = 0; y < sourceRectangle.Height; y++)
{ {
ReadOnlySpan<TPixel> source = image.PixelBuffer.DangerousGetRowSpan(y); ReadOnlySpan<TPixel> source = image.PixelBuffer
.DangerousGetRowSpan(sourceRectangle.Y + y)
.Slice(sourceRectangle.X, sourceRectangle.Width);
PixelOperations<TPixel>.Instance.ToRgba64(configuration, source, packed); PixelOperations<TPixel>.Instance.ToRgba64(configuration, source, packed);
ExtractAlpha(packed, alpha); ExtractAlpha(packed, alpha);
HeifSampleConversion.WriteSamples<TSample, TStorer>( HeifSampleConversion.WriteSamples<TSample, TStorer>(

166
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifPlanarColorConverter.cs

@ -3,8 +3,6 @@
using System.Buffers; using System.Buffers;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Advanced;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.PixelFormats;
@ -15,6 +13,16 @@ namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// </summary> /// </summary>
internal static class HeifPlanarColorConverter internal static class HeifPlanarColorConverter
{ {
/// <summary>
/// The number of planar color components retained for each source row.
/// </summary>
private const int ColorComponentCount = 3;
/// <summary>
/// The number of source rows consumed together by vertically subsampled chroma.
/// </summary>
private const int VerticallySubsampledRowCount = 2;
/// <summary> /// <summary>
/// The largest value represented by an eight-bit packed RGB component. /// The largest value represented by an eight-bit packed RGB component.
/// </summary> /// </summary>
@ -26,7 +34,20 @@ internal static class HeifPlanarColorConverter
private const float UShortMaximum = ushort.MaxValue; private const float UShortMaximum = ushort.MaxValue;
/// <summary> /// <summary>
/// Converts native unsigned 16-bit component storage to packed pixels and selects eligible exact integer kernels. /// Converts complete native unsigned 16-bit component planes to packed pixels.
/// </summary>
public static void ConvertToRgb<TPixel, TBuffer>(
Configuration configuration,
TBuffer buffer,
ImageFrame<TPixel> image,
in HeifColorConversionParameters parameters,
HeifColorConversionMode mode)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<ushort>
=> ConvertToRgb(configuration, buffer, image, in parameters, mode, 0, 0);
/// <summary>
/// Converts a region of native unsigned 16-bit component storage to packed pixels and selects eligible exact integer kernels.
/// </summary> /// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam> /// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam> /// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam>
@ -43,8 +64,8 @@ internal static class HeifPlanarColorConverter
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
in HeifColorConversionParameters parameters, in HeifColorConversionParameters parameters,
HeifColorConversionMode mode, HeifColorConversionMode mode,
int sourceX = 0, int sourceX,
int sourceY = 0) int sourceY)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<ushort> where TBuffer : struct, IHeifPlanarSampleBuffer<ushort>
{ {
@ -74,7 +95,22 @@ internal static class HeifPlanarColorConverter
} }
/// <summary> /// <summary>
/// Converts native component planes to packed pixels. /// Converts complete native component planes to packed pixels.
/// </summary>
public static void ConvertToRgb<TPixel, TBuffer, TSample, TLoader>(
Configuration configuration,
TBuffer buffer,
ImageFrame<TPixel> image,
in HeifColorConversionParameters parameters,
HeifColorConversionMode mode)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged
where TLoader : struct, IHeifSampleConverter<TSample>
=> ConvertToRgb<TPixel, TBuffer, TSample, TLoader>(configuration, buffer, image, in parameters, mode, 0, 0);
/// <summary>
/// Converts a region of native component planes to packed pixels.
/// </summary> /// </summary>
/// <typeparam name="TPixel">The destination pixel type.</typeparam> /// <typeparam name="TPixel">The destination pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam> /// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam>
@ -93,8 +129,8 @@ internal static class HeifPlanarColorConverter
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
in HeifColorConversionParameters parameters, in HeifColorConversionParameters parameters,
HeifColorConversionMode mode, HeifColorConversionMode mode,
int sourceX = 0, int sourceX,
int sourceY = 0) int sourceY)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample> where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged where TSample : unmanaged
@ -139,12 +175,48 @@ internal static class HeifPlanarColorConverter
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample> where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample> where TStorer : struct, IHeifSampleConverter<TSample>
{
Rectangle sourceRectangle = new(0, 0, image.Width, image.Height);
ConvertFromRgb<TPixel, TBuffer, TSample, TStorer>(
configuration,
image,
sourceRectangle,
buffer,
in parameters,
mode);
}
/// <summary>
/// Converts a rectangular packed-pixel region to native component planes.
/// </summary>
/// <typeparam name="TPixel">The source pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam>
/// <typeparam name="TSample">The native unsigned sample storage type.</typeparam>
/// <typeparam name="TStorer">The SIMD narrowing and storage operations for the sample type.</typeparam>
/// <param name="configuration">The configuration used for allocation and pixel conversion.</param>
/// <param name="image">The source image frame.</param>
/// <param name="sourceRectangle">The source region mapped to the complete destination buffer.</param>
/// <param name="buffer">The destination component-plane buffer.</param>
/// <param name="parameters">The resolved H.273 conversion parameters.</param>
/// <param name="mode">The resolved H.273 conversion mode.</param>
public static void ConvertFromRgb<TPixel, TBuffer, TSample, TStorer>(
Configuration configuration,
ImageFrame<TPixel> image,
Rectangle sourceRectangle,
TBuffer buffer,
in HeifColorConversionParameters parameters,
HeifColorConversionMode mode)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample>
{ {
HeifColorConverterBase colorConverter = HeifColorConverterBase.Create(mode, in parameters, buffer.IsMonochrome); HeifColorConverterBase colorConverter = HeifColorConverterBase.Create(mode, in parameters, buffer.IsMonochrome);
RgbToYuvRowConverter<TPixel, TBuffer, TSample, TStorer> converter = new( RgbToYuvRowConverter<TPixel, TBuffer, TSample, TStorer> converter = new(
configuration, configuration,
buffer, buffer,
image, image,
sourceRectangle,
colorConverter, colorConverter,
in parameters); in parameters);
@ -156,10 +228,60 @@ internal static class HeifPlanarColorConverter
return; return;
} }
using IMemoryOwner<Rgb48> packedOwner = configuration.MemoryAllocator.Allocate<Rgb48>(image.Width); using IMemoryOwner<Rgb48> packedOwner = configuration.MemoryAllocator.Allocate<Rgb48>(sourceRectangle.Width);
converter.Convert(packedOwner.GetSpan()[..image.Width], components); converter.Convert(packedOwner.GetSpan()[..sourceRectangle.Width], components);
} }
/// <summary>
/// Converts a rectangular packed-pixel region using a retained color converter and caller-owned row storage.
/// </summary>
/// <typeparam name="TPixel">The source pixel type.</typeparam>
/// <typeparam name="TBuffer">The codec adapter exposing the native component planes.</typeparam>
/// <typeparam name="TSample">The native unsigned sample storage type.</typeparam>
/// <typeparam name="TStorer">The SIMD narrowing and storage operations for the sample type.</typeparam>
/// <param name="configuration">The configuration used for pixel conversion.</param>
/// <param name="image">The source image frame.</param>
/// <param name="sourceRectangle">The source region mapped to the complete destination buffer.</param>
/// <param name="buffer">The destination component-plane buffer.</param>
/// <param name="parameters">The resolved H.273 conversion parameters.</param>
/// <param name="colorConverter">The retained converter matching <paramref name="parameters"/>.</param>
/// <param name="packed">The reusable high-bit-depth packed RGB row, or an empty span for eight-bit input.</param>
/// <param name="components">The reusable planar component rows.</param>
public static void ConvertFromRgb<TPixel, TBuffer, TSample, TStorer>(
Configuration configuration,
ImageFrame<TPixel> image,
Rectangle sourceRectangle,
TBuffer buffer,
in HeifColorConversionParameters parameters,
HeifColorConverterBase colorConverter,
Span<Rgb48> packed,
Span<float> components)
where TPixel : unmanaged, IPixel<TPixel>
where TBuffer : struct, IHeifPlanarSampleBuffer<TSample>
where TSample : unmanaged
where TStorer : struct, IHeifSampleConverter<TSample>
{
RgbToYuvRowConverter<TPixel, TBuffer, TSample, TStorer> converter = new(
configuration,
buffer,
image,
sourceRectangle,
colorConverter,
in parameters);
converter.Convert(packed, components);
}
/// <summary>
/// Gets the planar float storage required to convert one row, or one vertically subsampled row pair.
/// </summary>
/// <param name="width">The source-row width.</param>
/// <param name="isMonochrome">Whether only luma is written.</param>
/// <param name="subsamplingY">The vertical chroma-subsampling shift.</param>
/// <returns>The required number of float elements.</returns>
public static int GetRgbToYuvComponentBufferLength(int width, bool isMonochrome, int subsamplingY)
=> width * ColorComponentCount * (subsamplingY == 0 || isMonochrome ? 1 : VerticallySubsampledRowCount);
/// <summary> /// <summary>
/// Resolves the two chroma rows and quarter-sample weight surrounding a luma row. /// Resolves the two chroma rows and quarter-sample weight surrounding a luma row.
/// </summary> /// </summary>
@ -484,6 +606,11 @@ internal static class HeifPlanarColorConverter
/// </summary> /// </summary>
private readonly ImageFrame<TPixel> image; private readonly ImageFrame<TPixel> image;
/// <summary>
/// The source region mapped to the complete destination planes.
/// </summary>
private readonly Rectangle sourceRectangle;
/// <summary> /// <summary>
/// The selected H.273 color converter. /// The selected H.273 color converter.
/// </summary> /// </summary>
@ -535,18 +662,21 @@ internal static class HeifPlanarColorConverter
/// <param name="configuration">The configuration used for pixel conversion.</param> /// <param name="configuration">The configuration used for pixel conversion.</param>
/// <param name="buffer">The codec adapter exposing the destination component planes.</param> /// <param name="buffer">The codec adapter exposing the destination component planes.</param>
/// <param name="image">The source image frame.</param> /// <param name="image">The source image frame.</param>
/// <param name="sourceRectangle">The source region mapped to the complete destination planes.</param>
/// <param name="colorConverter">The selected H.273 color converter.</param> /// <param name="colorConverter">The selected H.273 color converter.</param>
/// <param name="parameters">The resolved H.273 component ranges.</param> /// <param name="parameters">The resolved H.273 component ranges.</param>
public RgbToYuvRowConverter( public RgbToYuvRowConverter(
Configuration configuration, Configuration configuration,
TBuffer buffer, TBuffer buffer,
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
Rectangle sourceRectangle,
HeifColorConverterBase colorConverter, HeifColorConverterBase colorConverter,
in HeifColorConversionParameters parameters) in HeifColorConversionParameters parameters)
{ {
this.configuration = configuration; this.configuration = configuration;
this.buffer = buffer; this.buffer = buffer;
this.image = image; this.image = image;
this.sourceRectangle = sourceRectangle;
this.colorConverter = colorConverter; this.colorConverter = colorConverter;
this.lumaMaximum = parameters.LumaSampleMaximum; this.lumaMaximum = parameters.LumaSampleMaximum;
this.chromaMaximum = parameters.ChromaSampleMaximum; this.chromaMaximum = parameters.ChromaSampleMaximum;
@ -566,7 +696,8 @@ internal static class HeifPlanarColorConverter
/// <summary> /// <summary>
/// Gets the number of float elements required by the reusable component buffer. /// Gets the number of float elements required by the reusable component buffer.
/// </summary> /// </summary>
public readonly int ComponentBufferLength => this.image.Width * (this.subsamplingY == 0 || this.isMonochrome ? 3 : 6); public readonly int ComponentBufferLength
=> GetRgbToYuvComponentBufferLength(this.sourceRectangle.Width, this.isMonochrome, this.subsamplingY);
/// <summary> /// <summary>
/// Converts every packed source row to the destination component planes. /// Converts every packed source row to the destination component planes.
@ -575,13 +706,13 @@ internal static class HeifPlanarColorConverter
/// <param name="components">The reusable planar component buffer.</param> /// <param name="components">The reusable planar component buffer.</param>
public void Convert(Span<Rgb48> packed, Span<float> components) public void Convert(Span<Rgb48> packed, Span<float> components)
{ {
int width = this.image.Width; int width = this.sourceRectangle.Width;
Span<float> luma0 = components[..width]; Span<float> luma0 = components[..width];
Span<float> blue0 = components.Slice(width, width); Span<float> blue0 = components.Slice(width, width);
Span<float> red0 = components.Slice(width * 2, width); Span<float> red0 = components.Slice(width * 2, width);
if (this.subsamplingY == 0) if (this.subsamplingY == 0)
{ {
for (int y = 0; y < this.image.Height; y++) for (int y = 0; y < this.sourceRectangle.Height; y++)
{ {
this.ConvertSourceRow(y, packed, luma0, blue0, red0); this.ConvertSourceRow(y, packed, luma0, blue0, red0);
HeifSampleConversion.WriteSamples<TSample, TStorer>( HeifSampleConversion.WriteSamples<TSample, TStorer>(
@ -603,7 +734,7 @@ internal static class HeifPlanarColorConverter
Span<float> luma1 = this.isMonochrome ? luma0 : components.Slice(width * 3, width); Span<float> luma1 = this.isMonochrome ? luma0 : components.Slice(width * 3, width);
Span<float> blue1 = this.isMonochrome ? blue0 : components.Slice(width * 4, width); Span<float> blue1 = this.isMonochrome ? blue0 : components.Slice(width * 4, width);
Span<float> red1 = this.isMonochrome ? red0 : components.Slice(width * 5, width); Span<float> red1 = this.isMonochrome ? red0 : components.Slice(width * 5, width);
int chromaHeight = (this.image.Height + 1) >> 1; int chromaHeight = (this.sourceRectangle.Height + 1) >> 1;
for (int destinationY = 0; destinationY < chromaHeight; destinationY++) for (int destinationY = 0; destinationY < chromaHeight; destinationY++)
{ {
// A vertically subsampled chroma row is owned by one two-row luma cell. Processing that cell as a // A vertically subsampled chroma row is owned by one two-row luma cell. Processing that cell as a
@ -617,7 +748,7 @@ internal static class HeifPlanarColorConverter
this.colorConverter.LumaBias, this.colorConverter.LumaBias,
this.lumaMaximum); this.lumaMaximum);
bool hasSecondRow = sourceY + 1 < this.image.Height; bool hasSecondRow = sourceY + 1 < this.sourceRectangle.Height;
if (hasSecondRow) if (hasSecondRow)
{ {
this.ConvertSourceRow(sourceY + 1, packed, luma1, blue1, red1); this.ConvertSourceRow(sourceY + 1, packed, luma1, blue1, red1);
@ -659,7 +790,10 @@ internal static class HeifPlanarColorConverter
/// <param name="chromaRed">The destination red-difference or third component values.</param> /// <param name="chromaRed">The destination red-difference or third component values.</param>
private void ConvertSourceRow(int y, Span<Rgb48> packed, Span<float> luma, Span<float> chromaBlue, Span<float> chromaRed) private void ConvertSourceRow(int y, Span<Rgb48> packed, Span<float> luma, Span<float> chromaBlue, Span<float> chromaRed)
{ {
ReadOnlySpan<TPixel> source = this.image.PixelBuffer.DangerousGetRowSpan(y); ReadOnlySpan<TPixel> source = this.image.PixelBuffer
.DangerousGetRowSpan(this.sourceRectangle.Y + y)
.Slice(this.sourceRectangle.X, this.sourceRectangle.Width);
if (this.UsesByteInput) if (this.UsesByteInput)
{ {
// JPEG's planar unpack contract reaches the existing pixel-specific SIMD implementation before // JPEG's planar unpack contract reaches the existing pixel-specific SIMD implementation before

212
src/ImageSharp/Formats/Heif/GridHeifItemDecoder.cs

@ -3,7 +3,6 @@
using System.Buffers; using System.Buffers;
using System.Buffers.Binary; using System.Buffers.Binary;
using SixLabors.ImageSharp.Common.Helpers;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Components.Alpha; using SixLabors.ImageSharp.Formats.Heif.Components.Alpha;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
@ -19,6 +18,31 @@ namespace SixLabors.ImageSharp.Formats.Heif;
internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IHeifAlphaItemDecoder<TPixel> internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IHeifAlphaItemDecoder<TPixel>
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
/// <summary>
/// The image-grid descriptor version defined by HEIF.
/// </summary>
private const byte GridDescriptorVersion = 0;
/// <summary>
/// The descriptor flag that selects 32-bit output dimensions instead of 16-bit dimensions.
/// </summary>
private const byte LargeDimensionsFlag = 1;
/// <summary>
/// The descriptor length when output dimensions use 16-bit fields.
/// </summary>
private const int ShortGridDescriptorLength = 8;
/// <summary>
/// The descriptor length when output dimensions use 32-bit fields.
/// </summary>
private const int LongGridDescriptorLength = 12;
/// <summary>
/// The minimum width and height of the first cell in a MIAF image grid.
/// </summary>
private const int MinimumGridCellDimension = 64;
/// <summary> /// <summary>
/// The item definitions available to the grid, indexed by item identifier. /// The item definitions available to the grid, indexed by item identifier.
/// </summary> /// </summary>
@ -122,9 +146,16 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
ref av1GridConfiguration, ref av1GridConfiguration,
cancellationToken); cancellationToken);
if (tile.Width != tileWidth || tile.Height != tileHeight) Size copySize = GetGridTileCopySize(
descriptor,
tileWidth,
tileHeight,
tileIndex);
if (!IsGridTileExtentValid(tile.Size, copySize, tileWidth, tileHeight))
{ {
throw new InvalidImageContentException("The HEIF image grid contains tiles with mismatched dimensions."); throw new InvalidImageContentException(
$"HEIF image grid tile {item.Id} has dimensions {tile.Size}, which cannot cover its {copySize} grid region.");
} }
CopyGridTile(tile, result, descriptor, tileIndex, tileWidth, tileHeight); CopyGridTile(tile, result, descriptor, tileIndex, tileWidth, tileHeight);
@ -165,16 +196,13 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
tileWidth = tile.Width; tileWidth = tile.Width;
tileHeight = tile.Height; tileHeight = tile.Height;
if (((long)tileWidth * descriptor.Columns) < descriptor.OutputSize.Width || ValidateGridCoverage(descriptor, tileWidth, tileHeight);
((long)tileHeight * descriptor.Rows) < descriptor.OutputSize.Height) ValidateGridDimensions(descriptor, tile.Size, av1GridConfiguration);
{ Size copySize = GetGridTileCopySize(descriptor, tileWidth, tileHeight, 0);
throw new InvalidImageContentException("The HEIF image grid tiles do not cover the output canvas."); if (!IsGridTileExtentValid(tile.Size, copySize, tileWidth, tileHeight))
}
if (((long)tileWidth * (descriptor.Columns - 1)) >= descriptor.OutputSize.Width ||
((long)tileHeight * (descriptor.Rows - 1)) >= descriptor.OutputSize.Height)
{ {
throw new InvalidImageContentException("The HEIF image grid edge tiles do not overlap the output canvas."); throw new InvalidImageContentException(
$"HEIF image grid tile {item.Id} has dimensions {tile.Size}, which cannot cover its {copySize} grid region.");
} }
Image<TPixel> result = new( Image<TPixel> result = new(
@ -279,12 +307,20 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
IReadOnlyList<uint> linked = this.GetLinkedTileIds(gridItem, descriptor); IReadOnlyList<uint> linked = this.GetLinkedTileIds(gridItem, descriptor);
Heif4CharCode tileType = default; Heif4CharCode tileType = default;
Av1CodecConfiguration? av1GridConfiguration = null; Av1CodecConfiguration? av1GridConfiguration = null;
Size tileSize = default; HeifItem firstItem = this.items[linked[0]];
if (firstItem.Extent == default)
{
throw new InvalidImageContentException($"HEIF alpha grid tile {firstItem.Id} has no spatial extent.");
}
Size tileSize = firstItem.Extent;
ValidateGridCoverage(descriptor, tileSize.Width, tileSize.Height);
// Validate the complete grid before mutating the color frame. IgnoreImageData can then omit a failed alpha // Validate the complete grid before mutating the color frame. IgnoreImageData can then omit a failed alpha
// grid without leaving a partially composed prefix in the returned image. // grid without leaving a partially composed prefix in the returned image.
foreach (uint id in linked) for (int tileIndex = 0; tileIndex < linked.Count; tileIndex++)
{ {
uint id = linked[tileIndex];
HeifItem item = this.items[id]; HeifItem item = this.items[id];
ValidateTileConfiguration(item, ref tileType, ref av1GridConfiguration); ValidateTileConfiguration(item, ref tileType, ref av1GridConfiguration);
if (HeifCompressionFactory.GetDecoder<TPixel>(item.Type) is not IHeifAlphaItemDecoder<TPixel>) if (HeifCompressionFactory.GetDecoder<TPixel>(item.Type) is not IHeifAlphaItemDecoder<TPixel>)
@ -297,29 +333,23 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
throw new InvalidImageContentException($"HEIF alpha grid tile {item.Id} has no spatial extent."); throw new InvalidImageContentException($"HEIF alpha grid tile {item.Id} has no spatial extent.");
} }
if (tileSize == default) Size copySize = GetGridTileCopySize(
{ descriptor,
tileSize = item.Extent; tileSize.Width,
} tileSize.Height,
else if (item.Extent != tileSize) tileIndex);
if (!IsGridTileExtentValid(item.Extent, copySize, tileSize.Width, tileSize.Height))
{ {
throw new InvalidImageContentException("The HEIF alpha grid contains tiles with mismatched dimensions."); throw new InvalidImageContentException(
$"HEIF alpha grid tile {item.Id} has dimensions {item.Extent}, which cannot cover its {copySize} grid region.");
} }
} }
ValidateGridDimensions(descriptor, tileSize, av1GridConfiguration);
int gridWidth = descriptor.OutputSize.Width; int gridWidth = descriptor.OutputSize.Width;
int gridHeight = descriptor.OutputSize.Height; int gridHeight = descriptor.OutputSize.Height;
if (((long)tileSize.Width * descriptor.Columns) < gridWidth || ((long)tileSize.Height * descriptor.Rows) < gridHeight)
{
throw new InvalidImageContentException("The HEIF alpha grid tiles do not cover the output canvas.");
}
if (((long)tileSize.Width * (descriptor.Columns - 1)) >= gridWidth ||
((long)tileSize.Height * (descriptor.Rows - 1)) >= gridHeight)
{
throw new InvalidImageContentException("The HEIF alpha grid edge tiles do not overlap the output canvas.");
}
if (descriptor.OutputSize != outputSize || destinationRectangle.Size != outputSize) if (descriptor.OutputSize != outputSize || destinationRectangle.Size != outputSize)
{ {
throw new InvalidImageContentException("The HEIF alpha grid dimensions do not match the color grid dimensions."); throw new InvalidImageContentException("The HEIF alpha grid dimensions do not match the color grid dimensions.");
@ -340,22 +370,109 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
int row = tileIndex / descriptor.Columns; int row = tileIndex / descriptor.Columns;
int destinationX = destinationRectangle.X + (column * tileSize.Width); int destinationX = destinationRectangle.X + (column * tileSize.Width);
int destinationY = destinationRectangle.Y + (row * tileSize.Height); int destinationY = destinationRectangle.Y + (row * tileSize.Height);
int copyWidth = Math.Min(tileSize.Width, destinationRectangle.Right - destinationX); Size copySize = GetGridTileCopySize(
int copyHeight = Math.Min(tileSize.Height, destinationRectangle.Bottom - destinationY); descriptor,
Rectangle tileDestination = new(destinationX, destinationY, copyWidth, copyHeight); tileSize.Width,
tileSize.Height,
tileIndex);
Rectangle tileDestination = new(destinationX, destinationY, copySize.Width, copySize.Height);
decoder.DecodeAlphaItemData( decoder.DecodeAlphaItemData(
options, options,
item, item,
itemMemory.GetSpan(), itemMemory.GetSpan(),
destination, destination,
tileSize, item.Extent,
tileDestination, tileDestination,
premultiplied, premultiplied,
cancellationToken); cancellationToken);
} }
} }
/// <summary>
/// Validates that the first cell dimensions cover the grid while leaving a nonempty final row and column.
/// </summary>
private static void ValidateGridCoverage(in GridDescriptor descriptor, int tileWidth, int tileHeight)
{
if (((long)tileWidth * descriptor.Columns) < descriptor.OutputSize.Width ||
((long)tileHeight * descriptor.Rows) < descriptor.OutputSize.Height)
{
throw new InvalidImageContentException("The HEIF image grid tiles do not cover the output canvas.");
}
if (((long)tileWidth * (descriptor.Columns - 1)) >= descriptor.OutputSize.Width ||
((long)tileHeight * (descriptor.Rows - 1)) >= descriptor.OutputSize.Height)
{
throw new InvalidImageContentException("The HEIF image grid edge tiles do not overlap the output canvas.");
}
}
/// <summary>
/// Gets the portion of one cell that overlaps the output canvas.
/// </summary>
private static Size GetGridTileCopySize(
in GridDescriptor descriptor,
int tileWidth,
int tileHeight,
int tileIndex)
{
int column = tileIndex % descriptor.Columns;
int row = tileIndex / descriptor.Columns;
int copyWidth = column == descriptor.Columns - 1
? descriptor.OutputSize.Width - (tileWidth * (descriptor.Columns - 1))
: tileWidth;
int copyHeight = row == descriptor.Rows - 1
? descriptor.OutputSize.Height - (tileHeight * (descriptor.Rows - 1))
: tileHeight;
return new Size(copyWidth, copyHeight);
}
/// <summary>
/// Determines whether a cell can cover its output region without exceeding the first cell's dimensions.
/// </summary>
private static bool IsGridTileExtentValid(Size extent, Size copySize, int tileWidth, int tileHeight)
=> extent.Width >= copySize.Width
&& extent.Width <= tileWidth
&& extent.Height >= copySize.Height
&& extent.Height <= tileHeight;
/// <summary>
/// Validates the MIAF cell-size and chroma-alignment rules established by the first grid cell.
/// </summary>
private static void ValidateGridDimensions(
in GridDescriptor descriptor,
Size tileSize,
Av1CodecConfiguration? av1GridConfiguration)
{
if (tileSize.Width < MinimumGridCellDimension || tileSize.Height < MinimumGridCellDimension)
{
throw new InvalidImageContentException(
$"HEIF image grid cells must be at least {MinimumGridCellDimension} samples wide and high.");
}
if (av1GridConfiguration is null || av1GridConfiguration.IsMonochrome)
{
return;
}
if (av1GridConfiguration.ChromaSubsamplingX &&
(((descriptor.OutputSize.Width & 1) != 0) || ((tileSize.Width & 1) != 0)))
{
throw new InvalidImageContentException(
"HEIF image grid widths must be even when AV1 chroma is horizontally subsampled.");
}
if (av1GridConfiguration.ChromaSubsamplingY &&
(((descriptor.OutputSize.Height & 1) != 0) || ((tileSize.Height & 1) != 0)))
{
throw new InvalidImageContentException(
"HEIF image grid heights must be even when AV1 chroma is vertically subsampled.");
}
}
/// <summary> /// <summary>
/// Parses and validates the fixed HEIF image-grid descriptor fields used by both color and alpha composition. /// Parses and validates the fixed HEIF image-grid descriptor fields used by both color and alpha composition.
/// </summary> /// </summary>
@ -363,38 +480,43 @@ internal sealed class GridHeifItemDecoder<TPixel> : IHeifItemDecoder<TPixel>, IH
/// <returns>The validated row, column, and output dimensions.</returns> /// <returns>The validated row, column, and output dimensions.</returns>
private static GridDescriptor ParseGridDescriptor(ReadOnlySpan<byte> data) private static GridDescriptor ParseGridDescriptor(ReadOnlySpan<byte> data)
{ {
if (data.Length < 8) if (data.Length < ShortGridDescriptorLength)
{ {
throw new InvalidImageContentException("The HEIF image grid descriptor is truncated."); throw new InvalidImageContentException("The HEIF image grid descriptor is truncated.");
} }
byte version = data[0]; int offset = 0;
if (version != 0) byte version = data[offset++];
if (version != GridDescriptorVersion)
{ {
throw new InvalidImageContentException($"The HEIF image grid descriptor has unsupported version {version}."); throw new InvalidImageContentException($"The HEIF image grid descriptor has unsupported version {version}.");
} }
bool usesLargeDimensions = (data[1] & 1) != 0; byte flags = data[offset++];
int descriptorLength = usesLargeDimensions ? 12 : 8; bool usesLargeDimensions = (flags & LargeDimensionsFlag) != 0;
int descriptorLength = usesLargeDimensions ? LongGridDescriptorLength : ShortGridDescriptorLength;
if (data.Length != descriptorLength) if (data.Length != descriptorLength)
{ {
throw new InvalidImageContentException("The HEIF image grid descriptor has an invalid length."); throw new InvalidImageContentException("The HEIF image grid descriptor has an invalid length.");
} }
int rows = data[offset++] + 1;
int columns = data[offset++] + 1;
uint outputWidth = usesLargeDimensions uint outputWidth = usesLargeDimensions
? BinaryPrimitives.ReadUInt32BigEndian(data[4..]) ? BinaryPrimitives.ReadUInt32BigEndian(data[offset..])
: BinaryPrimitives.ReadUInt16BigEndian(data[4..]); : BinaryPrimitives.ReadUInt16BigEndian(data[offset..]);
offset += usesLargeDimensions ? sizeof(uint) : sizeof(ushort);
uint outputHeight = usesLargeDimensions uint outputHeight = usesLargeDimensions
? BinaryPrimitives.ReadUInt32BigEndian(data[8..]) ? BinaryPrimitives.ReadUInt32BigEndian(data[offset..])
: BinaryPrimitives.ReadUInt16BigEndian(data[6..]); : BinaryPrimitives.ReadUInt16BigEndian(data[offset..]);
if (outputWidth is 0 or > int.MaxValue || outputHeight is 0 or > int.MaxValue) if (outputWidth is 0 or > int.MaxValue || outputHeight is 0 or > int.MaxValue)
{ {
throw new InvalidImageContentException("The HEIF image grid descriptor has invalid output dimensions."); throw new InvalidImageContentException("The HEIF image grid descriptor has invalid output dimensions.");
} }
return new GridDescriptor(data[2] + 1, data[3] + 1, new Size((int)outputWidth, (int)outputHeight)); return new GridDescriptor(rows, columns, new Size((int)outputWidth, (int)outputHeight));
} }
/// <summary> /// <summary>

454
src/ImageSharp/Formats/Heif/HeifDecoderCore.cs

@ -16,6 +16,7 @@ using SixLabors.ImageSharp.Metadata.Profiles.Icc;
using SixLabors.ImageSharp.Metadata.Profiles.Xmp; using SixLabors.ImageSharp.Metadata.Profiles.Xmp;
using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing; using SixLabors.ImageSharp.Processing;
using SixLabors.ImageSharp.Processing.Processors.Transforms;
namespace SixLabors.ImageSharp.Formats.Heif; namespace SixLabors.ImageSharp.Formats.Heif;
@ -266,23 +267,32 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
/// <returns>The bounded selected image-sequence model.</returns> /// <returns>The bounded selected image-sequence model.</returns>
private HeifSequence ParseImageSequence(BufferedReadStream stream) private HeifSequence ParseImageSequence(BufferedReadStream stream)
{ {
this.items.Clear();
this.itemLinks.Clear();
this.itemDataOffset = -1;
this.itemDataLength = 0;
HeifSequence? sequence = null; HeifSequence? sequence = null;
while (stream.Position < stream.Length) while (stream.Position < stream.Length)
{ {
long boxLength = HeifBoxReader.ReadHeader(stream, stream.Length, this.boxHeaderScratch, out Heif4CharCode boxType, true); long boxLength = HeifBoxReader.ReadHeader(stream, stream.Length, this.boxHeaderScratch, out Heif4CharCode boxType, true);
if (boxType == Heif4CharCode.Moov) switch (boxType)
{ {
if (sequence is not null) case Heif4CharCode.Meta:
{ this.ParseMetadata(stream, boxLength);
throw new InvalidImageContentException("The HEIF image sequence contains more than one movie box."); break;
} case Heif4CharCode.Moov:
if (sequence is not null)
{
throw new InvalidImageContentException("The HEIF image sequence contains more than one movie box.");
}
sequence = this.sequenceParser.Parse(stream, boxLength, this.fileStartOffset); sequence = this.sequenceParser.Parse(stream, boxLength, this.fileStartOffset);
} break;
else default:
{ // Sequence samples and image items use file-relative offsets, so payload boxes never need buffering.
// Sequence samples use absolute file offsets, so unrelated top-level payloads never need buffering. HeifBoxReader.Skip(stream, boxLength);
HeifBoxReader.Skip(stream, boxLength); break;
} }
} }
@ -297,9 +307,18 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
private ImageInfo IdentifyImageSequence(HeifSequence sequence) private ImageInfo IdentifyImageSequence(HeifSequence sequence)
{ {
HeifSequenceTrack colorTrack = sequence.ColorTrack; HeifSequenceTrack colorTrack = sequence.ColorTrack;
this.UpdateSequenceMetadata(this.metadata, sequence); HeifItem primaryItem = this.FindSequencePrimaryItem();
ImageFrameMetadata[] frameMetadata = CreateSequenceFrameMetadata(colorTrack); bool animateRootFrame = IsPrimaryItemFirstSequenceSample(primaryItem, colorTrack);
this.Dimensions = GetSequencePresentationExtent(colorTrack); Size sequenceExtent = GetSequencePresentationExtent(colorTrack);
this.Dimensions = animateRootFrame ? sequenceExtent : GetPresentationExtent(primaryItem);
if (!animateRootFrame && this.Dimensions != sequenceExtent)
{
throw new InvalidImageContentException("The primary image and image sequence have different presentation dimensions.");
}
this.UpdateMetadata(this.metadata, primaryItem);
this.UpdateSequenceMetadata(this.metadata, sequence, animateRootFrame);
ImageFrameMetadata[] frameMetadata = CreateSequenceFrameMetadata(colorTrack, animateRootFrame);
return new ImageInfo(this.Dimensions, this.metadata, frameMetadata); return new ImageInfo(this.Dimensions, this.metadata, frameMetadata);
} }
@ -318,170 +337,241 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
HeifSequenceTrack colorTrack = sequence.ColorTrack; HeifSequenceTrack colorTrack = sequence.ColorTrack;
this.UpdateSequenceMetadata(this.metadata, sequence); HeifItem primaryItem = this.FindSequencePrimaryItem();
ImageFrame<TPixel>[] colorFrames = this.DecodeVisibleSequenceFrames<TPixel>( bool animateRootFrame = IsPrimaryItemFirstSequenceSample(primaryItem, colorTrack);
stream, this.UpdateSequenceMetadata(this.metadata, sequence, animateRootFrame);
colorTrack, Size codedSize = new(colorTrack.CodedWidth, colorTrack.CodedHeight);
cancellationToken, Rectangle sourceRectangle = colorTrack.CleanAperture is not null
out int[] sampleIndices); ? colorTrack.CleanAperture.Value.ToRectangle(codedSize)
: new Rectangle(Point.Empty, codedSize);
// HEIF stores counter-clockwise quarter turns; ImageSharp's exact modes are clockwise.
RotateMode rotation = colorTrack.RotationAngle switch
{
1 => RotateMode.Rotate270,
2 => RotateMode.Rotate180,
3 => RotateMode.Rotate90,
_ => RotateMode.None
};
Size presentationSize = rotation is RotateMode.Rotate90 or RotateMode.Rotate270
? new Size(sourceRectangle.Height, sourceRectangle.Width)
: sourceRectangle.Size;
// The returned image owns every presented frame from the outset. A separate primary becomes its root;
// otherwise the first successfully decoded timed sample fills the root allocated here.
Image<TPixel> image = animateRootFrame
? new Image<TPixel>(
this.configuration,
presentationSize.Width,
presentationSize.Height,
this.metadata)
: this.DecodePrimaryItem<TPixel>(stream, cancellationToken);
Image<TPixel>? image = null;
try try
{ {
// Each codec frame owns its pixel buffer. The multi-frame image adopts those buffers directly instead if (image.Size != presentationSize)
// of cloning a complete decoded frame on every append. {
image = new Image<TPixel>(this.configuration, this.metadata, colorFrames); throw new InvalidImageContentException(
"The primary image and image sequence have different presentation dimensions.");
}
using Av1Decoder colorDecoder = new(this.configuration);
HeifSequenceTrack? alphaTrack = sequence.AlphaTrack; HeifSequenceTrack? alphaTrack = sequence.AlphaTrack;
if (alphaTrack is not null) (HeifSequenceTrack Track, Av1Decoder Decoder)? alphaState = alphaTrack is null
? null
: (alphaTrack, new Av1Decoder(this.configuration));
using Av1Decoder? alphaDecoder = alphaState?.Decoder;
// Quarter turns need source and destination frames with opposite dimensions. Reuse one source frame
// across the sequence, then rotate each completed color-and-alpha sample into its final owned frame.
using ImageFrame<TPixel>? rotationSource = rotation == RotateMode.None
? null
: new ImageFrame<TPixel>(
this.configuration,
sourceRectangle.Width,
sourceRectangle.Height);
int decodedFrameCount = 0;
for (int sampleIndex = 0; sampleIndex < colorTrack.Samples.Length; sampleIndex++)
{ {
int frameIndex = 0; cancellationToken.ThrowIfCancellationRequested();
using Av1Decoder alphaDecoder = new(this.configuration); HeifSequenceSample colorSample = colorTrack.Samples[sampleIndex];
for (int sampleIndex = 0; sampleIndex < alphaTrack.Samples.Length; sampleIndex++) if (colorSample.IsHidden)
{ {
cancellationToken.ThrowIfCancellationRequested(); this.ExecuteImageDataSegmentAction(
HeifSequenceSample alphaSample = alphaTrack.Samples[sampleIndex]; () => this.DecodeSequenceReference(stream, colorTrack, colorSample, colorDecoder));
if (alphaSample.IsHidden ||
frameIndex >= colorFrames.Length || if (alphaState is not null)
sampleIndices[frameIndex] != sampleIndex)
{ {
(HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value;
HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex];
this.ExecuteImageDataSegmentAction( this.ExecuteImageDataSegmentAction(
() => this.DecodeSequenceReference(stream, alphaTrack, alphaSample, alphaDecoder)); () => this.DecodeSequenceReference(
stream,
continue; currentAlphaState.Track,
alphaSample,
currentAlphaState.Decoder));
} }
this.ExecuteImageDataSegmentAction( continue;
() => this.DecodeSequenceAlphaFrame( }
stream,
alphaTrack,
alphaSample,
alphaDecoder,
colorFrames[frameIndex],
colorTrack.IsPremultiplied));
frameIndex++; bool appendedDestination = false;
ImageFrame<TPixel> decodedFrame;
if (rotationSource is not null)
{
decodedFrame = rotationSource;
}
else if (animateRootFrame && decodedFrameCount == 0)
{
decodedFrame = image.Frames.RootFrame;
}
else
{
decodedFrame = image.Frames.CreateFrame();
appendedDestination = true;
} }
}
ApplyPresentationTransforms( bool colorDecoded = false;
image, this.ExecuteImageDataSegmentAction(
colorTrack.CleanAperture, () =>
colorTrack.RotationAngle, {
colorTrack.MirrorAxis); this.DecodeSequenceFrame(
stream,
colorTrack,
colorSample,
colorDecoder,
sourceRectangle,
decodedFrame);
if (!this.Options.SkipMetadata) colorDecoded = true;
{ });
image.Metadata.CicpProfile ??= image.Frames.RootFrame.Metadata.CicpProfile?.DeepClone();
_ = this.TryConvertIccProfile(image); if (!colorDecoded)
}
else
{
foreach (ImageFrame<TPixel> frame in image.Frames)
{ {
frame.Metadata.CicpProfile = null; if (alphaState is not null)
{
(HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value;
HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex];
this.ExecuteImageDataSegmentAction(
() => this.DecodeSequenceReference(
stream,
currentAlphaState.Track,
alphaSample,
currentAlphaState.Decoder));
}
if (appendedDestination)
{
image.Frames.RemoveFrame(image.Frames.Count - 1);
}
continue;
} }
}
this.Dimensions = image.Size; bool alphaDecoded = true;
return image; if (alphaState is not null)
}
catch
{
if (image is not null)
{
image.Dispose();
}
else
{
// Ownership transfers to Image only after its constructor validates every decoded frame.
foreach (ImageFrame<TPixel> frame in colorFrames)
{ {
frame.Dispose(); alphaDecoded = false;
(HeifSequenceTrack Track, Av1Decoder Decoder) currentAlphaState = alphaState.Value;
HeifSequenceSample alphaSample = currentAlphaState.Track.Samples[sampleIndex];
this.ExecuteImageDataSegmentAction(
() =>
{
this.DecodeSequenceAlphaFrame(
stream,
currentAlphaState.Track,
alphaSample,
currentAlphaState.Decoder,
sourceRectangle,
decodedFrame,
colorTrack.IsPremultiplied);
alphaDecoded = true;
});
} }
}
throw; if (!alphaDecoded)
} {
} if (appendedDestination)
{
image.Frames.RemoveFrame(image.Frames.Count - 1);
}
/// <summary> continue;
/// Decodes visible samples while preserving their source indices for frame-aligned alpha lookup. }
/// </summary>
/// <typeparam name="TPixel">The destination pixel format.</typeparam>
/// <param name="stream">The complete seekable HEIF stream.</param>
/// <param name="track">The selected coded-image track.</param>
/// <param name="cancellationToken">The token used to cancel work between coded samples.</param>
/// <param name="sampleIndices">Receives the decode-order sample index for each returned visible frame.</param>
/// <returns>The exact array of successfully decoded visible frames.</returns>
private ImageFrame<TPixel>[] DecodeVisibleSequenceFrames<TPixel>(
BufferedReadStream stream,
HeifSequenceTrack track,
CancellationToken cancellationToken,
out int[] sampleIndices)
where TPixel : unmanaged, IPixel<TPixel>
{
int visibleFrameCount = 0;
foreach (HeifSequenceSample sample in track.Samples)
{
visibleFrameCount += sample.IsHidden ? 0 : 1;
}
ImageFrame<TPixel>[] frames = new ImageFrame<TPixel>[visibleFrameCount]; ImageFrame<TPixel> presentedFrame = decodedFrame;
sampleIndices = new int[visibleFrameCount]; if (rotationSource is not null)
int decodedFrameCount = 0;
using Av1Decoder decoder = new(this.configuration);
try
{
for (int sampleIndex = 0; sampleIndex < track.Samples.Length; sampleIndex++)
{
HeifSequenceSample sample = track.Samples[sampleIndex];
if (sample.IsHidden)
{ {
this.ExecuteImageDataSegmentAction( presentedFrame = animateRootFrame && decodedFrameCount == 0
() => this.DecodeSequenceReference(stream, track, sample, decoder)); ? image.Frames.RootFrame
: image.Frames.CreateFrame();
continue; RotateProcessor<TPixel>.ApplyQuarterTurn(
rotation,
rotationSource,
presentedFrame,
this.configuration);
presentedFrame.Metadata.CicpProfile = rotationSource.Metadata.CicpProfile;
} }
cancellationToken.ThrowIfCancellationRequested(); if (colorTrack.MirrorAxis is not null)
ImageFrame<TPixel>? frame = null; {
this.ExecuteImageDataSegmentAction( // Axis zero reflects top-to-bottom around the horizontal axis; axis one reflects left-to-right.
() => frame = this.DecodeSequenceFrame<TPixel>(stream, track, sample, decoder)); FlipMode flip = colorTrack.MirrorAxis.Value == 0 ? FlipMode.Vertical : FlipMode.Horizontal;
FlipProcessor<TPixel>.Apply(flip, presentedFrame, this.configuration);
}
presentedFrame.Metadata.GetHeifMetadata().FrameDelay = new Rational(
colorSample.Duration,
colorTrack.MediaTimescale);
if (frame is null) if (!animateRootFrame && !this.Options.SkipMetadata)
{ {
continue; presentedFrame.Metadata.IccProfile = colorTrack.IccProfile;
_ = this.TryConvertIccProfile(presentedFrame);
} }
frame.Metadata.GetHeifMetadata().FrameDelay = new Rational(sample.Duration, track.MediaTimescale);
frames[decodedFrameCount] = frame;
sampleIndices[decodedFrameCount] = sampleIndex;
decodedFrameCount++; decodedFrameCount++;
} }
if (decodedFrameCount == 0) if (decodedFrameCount == 0)
{ {
throw new InvalidImageContentException("The HEIF image sequence contains no decodable visible samples."); throw new InvalidImageContentException(
"The HEIF image sequence contains no decodable visible samples.");
} }
if (decodedFrameCount != frames.Length) if (!this.Options.SkipMetadata)
{
if (animateRootFrame)
{
image.Metadata.CicpProfile ??= image.Frames.RootFrame.Metadata.CicpProfile?.DeepClone();
_ = this.TryConvertIccProfile(image);
}
}
else
{ {
// Compaction occurs only in IgnoreImageData mode after a recoverable coded-sample failure. foreach (ImageFrame<TPixel> frame in image.Frames)
Array.Resize(ref frames, decodedFrameCount); {
Array.Resize(ref sampleIndices, decodedFrameCount); frame.Metadata.CicpProfile = null;
}
} }
return frames; HeifMetadata resultMetadata = image.Metadata.GetHeifMetadata();
resultMetadata.RepeatCount = colorTrack.RepeatCount;
resultMetadata.AnimateRootFrame = animateRootFrame;
resultMetadata.HasAlpha |= alphaState is not null;
this.Dimensions = image.Size;
return image;
} }
catch catch
{ {
// Frames are independently allocated before the final Image adopts them. Retain ownership until this image.Dispose();
// method returns so a later sample failure cannot leak the successfully decoded prefix.
for (int frameIndex = 0; frameIndex < decodedFrameCount; frameIndex++)
{
frames[frameIndex].Dispose();
}
throw; throw;
} }
} }
@ -494,12 +584,15 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
/// <param name="track">The track supplying the codec configuration and color description.</param> /// <param name="track">The track supplying the codec configuration and color description.</param>
/// <param name="sample">The validated sample range.</param> /// <param name="sample">The validated sample range.</param>
/// <param name="decoder">The decoder retaining earlier sequence references.</param> /// <param name="decoder">The decoder retaining earlier sequence references.</param>
/// <returns>The independently owned decoded frame.</returns> /// <param name="sourceRectangle">The clean-aperture region mapped to the destination frame.</param>
private ImageFrame<TPixel> DecodeSequenceFrame<TPixel>( /// <param name="destination">The caller-owned frame receiving the presented sample.</param>
private void DecodeSequenceFrame<TPixel>(
BufferedReadStream stream, BufferedReadStream stream,
HeifSequenceTrack track, HeifSequenceTrack track,
HeifSequenceSample sample, HeifSequenceSample sample,
Av1Decoder decoder) Av1Decoder decoder,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
if (track.CodecType != Heif4CharCode.Av01) if (track.CodecType != Heif4CharCode.Av01)
@ -513,18 +606,13 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
using IMemoryOwner<byte> sampleOwner = this.ReadSequenceSample(stream, track, sample); using IMemoryOwner<byte> sampleOwner = this.ReadSequenceSample(stream, track, sample);
Span<byte> sampleData = sampleOwner.GetSpan()[..sample.Length]; Span<byte> sampleData = sampleOwner.GetSpan()[..sample.Length];
ImageFrame<TPixel> frame = decoder.DecodeSequenceFrame<TPixel>( decoder.DecodeSequenceFrame(
sampleData, sampleData,
track.CicpProfile, track.CicpProfile,
codecConfiguration); codecConfiguration,
new Size(track.CodedWidth, track.CodedHeight),
if (frame.Width != track.CodedWidth || frame.Height != track.CodedHeight) sourceRectangle,
{ destination);
frame.Dispose();
throw new InvalidImageContentException("The decoded image-sequence sample dimensions do not match its visual sample entry.");
}
return frame;
} }
/// <summary> /// <summary>
@ -535,6 +623,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
/// <param name="track">The alpha track supplying the codec configuration and color description.</param> /// <param name="track">The alpha track supplying the codec configuration and color description.</param>
/// <param name="sample">The validated alpha sample range.</param> /// <param name="sample">The validated alpha sample range.</param>
/// <param name="decoder">The decoder retaining earlier alpha-sequence references.</param> /// <param name="decoder">The decoder retaining earlier alpha-sequence references.</param>
/// <param name="sourceRectangle">The clean-aperture region mapped to the destination frame.</param>
/// <param name="destination">The decoded color frame receiving alpha values.</param> /// <param name="destination">The decoded color frame receiving alpha values.</param>
/// <param name="premultiplied">Whether stored color samples must be converted to unassociated alpha.</param> /// <param name="premultiplied">Whether stored color samples must be converted to unassociated alpha.</param>
private void DecodeSequenceAlphaFrame<TPixel>( private void DecodeSequenceAlphaFrame<TPixel>(
@ -542,6 +631,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
HeifSequenceTrack track, HeifSequenceTrack track,
HeifSequenceSample sample, HeifSequenceSample sample,
Av1Decoder decoder, Av1Decoder decoder,
Rectangle sourceRectangle,
ImageFrame<TPixel> destination, ImageFrame<TPixel> destination,
bool premultiplied) bool premultiplied)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
@ -566,6 +656,7 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
track.CicpProfile, track.CicpProfile,
codecConfiguration, codecConfiguration,
new Size(track.CodedWidth, track.CodedHeight), new Size(track.CodedWidth, track.CodedHeight),
sourceRectangle,
destination, destination,
destination.Size, destination.Size,
destination.Bounds, destination.Bounds,
@ -642,28 +733,33 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
/// </summary> /// </summary>
/// <param name="metadata">The image metadata receiving the sequence description.</param> /// <param name="metadata">The image metadata receiving the sequence description.</param>
/// <param name="sequence">The parsed selected image sequence.</param> /// <param name="sequence">The parsed selected image sequence.</param>
private void UpdateSequenceMetadata(ImageMetadata metadata, HeifSequence sequence) /// <param name="animateRootFrame">Whether the primary image is also the first timed sample.</param>
private void UpdateSequenceMetadata(ImageMetadata metadata, HeifSequence sequence, bool animateRootFrame)
{ {
HeifSequenceTrack colorTrack = sequence.ColorTrack; HeifSequenceTrack colorTrack = sequence.ColorTrack;
HeifMetadata heifMetadata = metadata.GetHeifMetadata(); HeifMetadata heifMetadata = metadata.GetHeifMetadata();
heifMetadata.RepeatCount = colorTrack.RepeatCount; heifMetadata.RepeatCount = colorTrack.RepeatCount;
heifMetadata.AnimateRootFrame = true; heifMetadata.AnimateRootFrame = animateRootFrame;
heifMetadata.HasAlpha = sequence.AlphaTrack is not null; heifMetadata.HasAlpha |= sequence.AlphaTrack is not null;
switch (colorTrack.CodecType) switch (colorTrack.CodecType)
{ {
case Heif4CharCode.Av01: case Heif4CharCode.Av01:
Av1CodecConfiguration av1Configuration = colorTrack.Av1CodecConfiguration Av1CodecConfiguration av1Configuration = colorTrack.Av1CodecConfiguration
?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration."); ?? throw new InvalidImageContentException("The AV1 image-sequence track has no codec configuration.");
heifMetadata.CompressionMethod = HeifCompressionMethod.Av1; if (animateRootFrame)
heifMetadata.BitDepth = av1Configuration.BitDepth; {
heifMetadata.IsMonochrome = av1Configuration.IsMonochrome; heifMetadata.CompressionMethod = HeifCompressionMethod.Av1;
heifMetadata.BitDepth = av1Configuration.BitDepth;
heifMetadata.IsMonochrome = av1Configuration.IsMonochrome;
}
break; break;
default: default:
throw new InvalidImageContentException($"The image-sequence sample entry '{colorTrack.CodecType}' is not supported."); throw new InvalidImageContentException($"The image-sequence sample entry '{colorTrack.CodecType}' is not supported.");
} }
if (this.Options.SkipMetadata) if (this.Options.SkipMetadata || !animateRootFrame)
{ {
return; return;
} }
@ -696,8 +792,11 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
/// Creates one HEIF frame-metadata entry for each visible retained sequence sample. /// Creates one HEIF frame-metadata entry for each visible retained sequence sample.
/// </summary> /// </summary>
/// <param name="track">The selected color track supplying sample durations.</param> /// <param name="track">The selected color track supplying sample durations.</param>
/// <param name="animateRootFrame">Whether the first sequence sample occupies the root-frame slot.</param>
/// <returns>The exact visible-frame metadata array in presentation order.</returns> /// <returns>The exact visible-frame metadata array in presentation order.</returns>
private static ImageFrameMetadata[] CreateSequenceFrameMetadata(HeifSequenceTrack track) private static ImageFrameMetadata[] CreateSequenceFrameMetadata(
HeifSequenceTrack track,
bool animateRootFrame)
{ {
int visibleFrameCount = 0; int visibleFrameCount = 0;
foreach (HeifSequenceSample sample in track.Samples) foreach (HeifSequenceSample sample in track.Samples)
@ -705,8 +804,14 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
visibleFrameCount += sample.IsHidden ? 0 : 1; visibleFrameCount += sample.IsHidden ? 0 : 1;
} }
ImageFrameMetadata[] result = new ImageFrameMetadata[visibleFrameCount]; int firstSequenceFrameIndex = animateRootFrame ? 0 : 1;
int frameIndex = 0; ImageFrameMetadata[] result = new ImageFrameMetadata[visibleFrameCount + firstSequenceFrameIndex];
if (!animateRootFrame)
{
result[0] = new ImageFrameMetadata();
}
int frameIndex = firstSequenceFrameIndex;
foreach (HeifSequenceSample sample in track.Samples) foreach (HeifSequenceSample sample in track.Samples)
{ {
if (sample.IsHidden) if (sample.IsHidden)
@ -722,6 +827,39 @@ internal sealed class HeifDecoderCore : ImageDecoderCore
return result; return result;
} }
/// <summary>
/// Gets the primary image item required alongside an AVIF image sequence.
/// </summary>
private HeifItem FindSequencePrimaryItem()
=> this.FindItemById(this.primaryItem)
?? throw new InvalidImageContentException("The HEIF image sequence contains no primary image item.");
/// <summary>
/// Determines whether the primary image item reuses the first presented sequence sample.
/// </summary>
private static bool IsPrimaryItemFirstSequenceSample(HeifItem primaryItem, HeifSequenceTrack colorTrack)
{
if (primaryItem.DataLocations.Count != 1)
{
return false;
}
HeifLocation location = primaryItem.DataLocations[0];
foreach (HeifSequenceSample sample in colorTrack.Samples)
{
if (sample.IsHidden)
{
continue;
}
return location.Origin == HeifLocationOffsetOrigin.FileOffset
&& location.BaseOffset + location.Offset == sample.Offset
&& location.Length == sample.Length;
}
return false;
}
/// <summary> /// <summary>
/// Updates identification metadata from the primary item or its decodable thumbnail fallback. /// Updates identification metadata from the primary item or its decodable thumbnail fallback.
/// </summary> /// </summary>

8
src/ImageSharp/Formats/Heif/HeifEncoder.cs

@ -92,20 +92,22 @@ public sealed class HeifEncoder : AnimatedImageEncoder
/// <summary> /// <summary>
/// Gets a value indicating whether the primary and auxiliary alpha images are encoded without loss. When /// Gets a value indicating whether the primary and auxiliary alpha images are encoded without loss. When
/// <see langword="true"/>, <see cref="Quality"/> and <see cref="AlphaQuality"/> do not affect the encoded image. /// <see langword="true"/>, <see cref="Quality"/> and <see cref="AlphaQuality"/> do not affect the encoded image.
/// Legacy JPEG image items do not support lossless encoding. The default is <see langword="false"/>. /// This option has no effect on legacy JPEG image items. The default is <see langword="false"/>.
/// </summary> /// </summary>
public bool Lossless { get; init; } public bool Lossless { get; init; }
/// <summary> /// <summary>
/// Gets the encoded precision of each image component, or <see langword="null"/> to use the HEIF metadata bit /// Gets the encoded precision of each image component, or <see langword="null"/> to use the HEIF metadata bit
/// depth. Metadata that does not specify a bit depth defaults to <see cref="HeifBitDepth.Bit8"/>. Legacy JPEG /// depth. Metadata that does not specify a bit depth defaults to <see cref="HeifBitDepth.Bit8"/>. Legacy JPEG
/// image items support only <see cref="HeifBitDepth.Bit8"/>. /// image items are always encoded with <see cref="HeifBitDepth.Bit8"/>.
/// </summary> /// </summary>
public HeifBitDepth? BitDepth { get; init; } public HeifBitDepth? BitDepth { get; init; }
/// <summary> /// <summary>
/// Gets the encoded chroma sampling, or <see langword="null"/> to use <see cref="HeifChromaSubsampling.Yuv420"/> /// Gets the encoded chroma sampling, or <see langword="null"/> to use <see cref="HeifChromaSubsampling.Yuv420"/>
/// for lossy encoding and <see cref="HeifChromaSubsampling.Yuv444"/> for lossless encoding. /// for lossy encoding and <see cref="HeifChromaSubsampling.Yuv444"/> for lossless encoding. Oversized still
/// images use <see cref="HeifChromaSubsampling.Yuv444"/> when a subsampled AVIF grid cannot represent an odd
/// output dimension.
/// </summary> /// </summary>
public HeifChromaSubsampling? ChromaSubsampling { get; init; } public HeifChromaSubsampling? ChromaSubsampling { get; init; }

265
src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs

@ -16,16 +16,59 @@ namespace SixLabors.ImageSharp.Formats.Heif;
internal sealed partial class HeifEncoderCore internal sealed partial class HeifEncoderCore
{ {
/// <summary>
/// The millisecond media timescale used when every frame delay can be represented exactly.
/// </summary>
private const uint DefaultSequenceTimescale = 1000; private const uint DefaultSequenceTimescale = 1000;
/// <summary>
/// The microsecond fallback used when the exact common frame-delay timescale exceeds 32 bits.
/// </summary>
private const uint FallbackSequenceTimescale = 1000000; private const uint FallbackSequenceTimescale = 1000000;
/// <summary>
/// The identity value for signed 16.16 movie and track matrix entries.
/// </summary>
private const uint UnityFixed16Point16 = 1U << 16; private const uint UnityFixed16Point16 = 1U << 16;
/// <summary>
/// The identity value for the signed 2.30 homogeneous movie and track matrix entry.
/// </summary>
private const uint UnityFixed2Point30 = 1U << 30; private const uint UnityFixed2Point30 = 1U << 30;
/// <summary>
/// The identity value for unsigned 8.8 track volume.
/// </summary>
private const ushort UnityFixed8Point8 = 1 << 8; private const ushort UnityFixed8Point8 = 1 << 8;
/// <summary>
/// The packed ISO 639-2/T language code for undetermined content.
/// </summary>
private const ushort PackedUndeterminedLanguage = 0x55C4; private const ushort PackedUndeterminedLanguage = 0x55C4;
/// <summary>
/// The coding-constraints flag stating that every reference picture is intra.
/// </summary>
private const uint AllReferencePicturesIntraMask = 1U << 31; private const uint AllReferencePicturesIntraMask = 1U << 31;
/// <summary>
/// The coding-constraints flag stating that intra prediction is used.
/// </summary>
private const uint IntraPicturePredictionUsedMask = 1U << 30; private const uint IntraPicturePredictionUsedMask = 1U << 30;
/// <summary>
/// The conventional 72-dpi horizontal and vertical resolution stored as unsigned 16.16.
/// </summary>
private const uint DefaultVisualSampleResolution = 72U << 16; private const uint DefaultVisualSampleResolution = 72U << 16;
/// <summary>
/// The fixed visual-sample-entry compressor-name field length.
/// </summary>
private const int VisualSampleCompressorNameLength = 32; private const int VisualSampleCompressorNameLength = 32;
/// <summary>
/// The visual-sample-entry depth used for color pictures.
/// </summary>
private const ushort VisualSampleDepth = 24; private const ushort VisualSampleDepth = 24;
private Av1EncodingSettings ResolveAv1Encoding<TPixel>(Image<TPixel> image) private Av1EncodingSettings ResolveAv1Encoding<TPixel>(Image<TPixel> image)
@ -48,6 +91,18 @@ internal sealed partial class HeifEncoderCore
HeifChromaSubsampling chromaSubsampling = this.encoder.ChromaSubsampling ?? HeifChromaSubsampling chromaSubsampling = this.encoder.ChromaSubsampling ??
(metadata.IsMonochrome ? HeifChromaSubsampling.Monochrome : defaultChromaSubsampling); (metadata.IsMonochrome ? HeifChromaSubsampling.Monochrome : defaultChromaSubsampling);
if (this.encoder.ChromaSubsampling is null
&& image.Frames.Count == 1
&& (image.Width > Av1Constants.MaxFrameDimension || image.Height > Av1Constants.MaxFrameDimension)
&& ((chromaSubsampling == HeifChromaSubsampling.Yuv420
&& (((image.Width & 1) != 0) || ((image.Height & 1) != 0)))
|| (chromaSubsampling == HeifChromaSubsampling.Yuv422 && (image.Width & 1) != 0)))
{
// A derived grid requires even output dimensions on every subsampled axis. When sampling was not
// explicitly requested, retain the complete image dimensions by selecting full-resolution chroma.
chromaSubsampling = HeifChromaSubsampling.Yuv444;
}
(bool isMonochrome, bool subsamplingX, bool subsamplingY) = chromaSubsampling switch (bool isMonochrome, bool subsamplingX, bool subsamplingY) = chromaSubsampling switch
{ {
HeifChromaSubsampling.Monochrome => (true, true, true), HeifChromaSubsampling.Monochrome => (true, true, true),
@ -143,14 +198,10 @@ internal sealed partial class HeifEncoderCore
ChunkedMemoryStream stream, ChunkedMemoryStream stream,
Av1EncodingSettings settings, Av1EncodingSettings settings,
Memory<HeifSequenceSampleInfo> samples, Memory<HeifSequenceSampleInfo> samples,
int firstFrameIndex,
CancellationToken cancellationToken) CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
if (image.Width > ushort.MaxValue || image.Height > ushort.MaxValue)
{
throw new NotSupportedException("AV1 image-sequence dimensions cannot exceed 65535 pixels.");
}
byte[]? exifData = null; byte[]? exifData = null;
uint tiffHeaderOffset = 0; uint tiffHeaderOffset = 0;
byte[]? xmpData = null; byte[]? xmpData = null;
@ -164,47 +215,58 @@ internal sealed partial class HeifEncoderCore
} }
} }
int frameCount = image.Frames.Count; int frameCount = image.Frames.Count - firstFrameIndex;
uint timescale = GetSequenceTimescale(image); uint timescale = GetSequenceTimescale(image, firstFrameIndex);
// The container needs only offset, length, and duration after each frame is streamed. Color and alpha // The container needs only offset, length, and duration after each frame is streamed. Color and alpha
// share one allocator-owned table, with each track occupying one contiguous slice until moov is written. // share one allocator-owned table, with each track occupying one contiguous slice until moov is written.
Span<HeifSequenceSampleInfo> colorSamples = samples.Span[..frameCount]; Span<HeifSequenceSampleInfo> colorSamples = samples.Span[..frameCount];
ImageFrame<TPixel> rootFrame = image.Frames.RootFrame; ImageFrame<TPixel> firstFrame = image.Frames[firstFrameIndex];
uint duration = GetSequenceSampleDuration(rootFrame.Metadata.GetHeifMetadata().FrameDelay, timescale); ObuSequenceHeader colorHeader;
cancellationToken.ThrowIfCancellationRequested(); bool colorUsesInterPrediction = settings.ColorQIndex != 0;
long colorOffset = stream.Length; using (Av1FrameEncoder.SequenceEncoder colorEncoder = Av1FrameEncoder.CreateColorSequenceEncoder(
ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode(
this.configuration, this.configuration,
rootFrame, image.Width,
stream, image.Height,
settings.ColorConfig, settings.ColorConfig,
settings.ColorQIndex, settings.ColorQIndex,
this.encoder.Effort); this.encoder.Effort))
colorSamples[0] = new HeifSequenceSampleInfo(
colorOffset,
checked((int)(stream.Length - colorOffset)),
duration);
for (int frameIndex = 1; frameIndex < frameCount; frameIndex++)
{ {
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
ImageFrame<TPixel> frame = image.Frames[frameIndex]; long colorOffset = stream.Length;
duration = GetSequenceSampleDuration(frame.Metadata.GetHeifMetadata().FrameDelay, timescale); colorEncoder.EncodeKeyFrame(firstFrame, stream);
colorOffset = stream.Length; colorHeader = colorEncoder.SequenceHeader;
_ = Av1FrameEncoder.Encode(
this.configuration,
frame,
stream,
settings.ColorConfig,
settings.ColorQIndex,
this.encoder.Effort);
colorSamples[frameIndex] = new HeifSequenceSampleInfo( colorSamples[0] = new HeifSequenceSampleInfo(
colorOffset, colorOffset,
checked((int)(stream.Length - colorOffset)), checked((int)(stream.Length - colorOffset)),
duration); GetSequenceSampleDuration(firstFrame.Metadata.GetHeifMetadata().FrameDelay, timescale),
isSyncSample: true);
for (int sampleIndex = 1; sampleIndex < frameCount; sampleIndex++)
{
cancellationToken.ThrowIfCancellationRequested();
int frameIndex = firstFrameIndex + sampleIndex;
ImageFrame<TPixel> frame = image.Frames[frameIndex];
uint duration = GetSequenceSampleDuration(frame.Metadata.GetHeifMetadata().FrameDelay, timescale);
colorOffset = stream.Length;
if (colorUsesInterPrediction)
{
colorEncoder.EncodeInterFrame(frame, stream);
}
else
{
// Lossless AV1 requires 4x4 transforms. Until the inter path supports that reversible size,
// continuation samples remain independent key frames instead of weakening losslessness.
colorEncoder.EncodeKeyFrame(frame, stream);
}
colorSamples[sampleIndex] = new HeifSequenceSampleInfo(
colorOffset,
checked((int)(stream.Length - colorOffset)),
duration,
isSyncSample: !colorUsesInterPrediction);
}
} }
HeifSequenceTrackEncoding colorTrack = new( HeifSequenceTrackEncoding colorTrack = new(
@ -217,37 +279,47 @@ internal sealed partial class HeifEncoderCore
{ {
Memory<HeifSequenceSampleInfo> alphaSampleMemory = samples.Slice(frameCount, frameCount); Memory<HeifSequenceSampleInfo> alphaSampleMemory = samples.Slice(frameCount, frameCount);
Span<HeifSequenceSampleInfo> alphaSamples = alphaSampleMemory.Span; Span<HeifSequenceSampleInfo> alphaSamples = alphaSampleMemory.Span;
cancellationToken.ThrowIfCancellationRequested(); ObuSequenceHeader alphaHeader;
long alphaOffset = stream.Length; bool alphaUsesInterPrediction = settings.AlphaQIndex != 0;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha( using (Av1FrameEncoder.SequenceEncoder alphaEncoder = Av1FrameEncoder.CreateAlphaSequenceEncoder(
this.configuration, this.configuration,
rootFrame, image.Width,
stream, image.Height,
settings.AlphaConfig, settings.AlphaConfig,
settings.AlphaQIndex, settings.AlphaQIndex,
this.encoder.Effort); this.encoder.Effort))
alphaSamples[0] = new HeifSequenceSampleInfo(
alphaOffset,
checked((int)(stream.Length - alphaOffset)),
colorSamples[0].Duration);
for (int frameIndex = 1; frameIndex < frameCount; frameIndex++)
{ {
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
alphaOffset = stream.Length; long alphaOffset = stream.Length;
_ = Av1FrameEncoder.EncodeAlpha( alphaEncoder.EncodeKeyFrame(firstFrame, stream);
this.configuration, alphaHeader = alphaEncoder.SequenceHeader;
image.Frames[frameIndex],
stream, alphaSamples[0] = new HeifSequenceSampleInfo(
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
alphaSamples[frameIndex] = new HeifSequenceSampleInfo(
alphaOffset, alphaOffset,
checked((int)(stream.Length - alphaOffset)), checked((int)(stream.Length - alphaOffset)),
colorSamples[frameIndex].Duration); colorSamples[0].Duration,
isSyncSample: true);
for (int sampleIndex = 1; sampleIndex < frameCount; sampleIndex++)
{
cancellationToken.ThrowIfCancellationRequested();
int frameIndex = firstFrameIndex + sampleIndex;
alphaOffset = stream.Length;
if (alphaUsesInterPrediction)
{
alphaEncoder.EncodeInterFrame(image.Frames[frameIndex], stream);
}
else
{
alphaEncoder.EncodeKeyFrame(image.Frames[frameIndex], stream);
}
alphaSamples[sampleIndex] = new HeifSequenceSampleInfo(
alphaOffset,
checked((int)(stream.Length - alphaOffset)),
colorSamples[sampleIndex].Duration,
isSyncSample: !alphaUsesInterPrediction);
}
} }
alphaTrack = new HeifSequenceTrackEncoding( alphaTrack = new HeifSequenceTrackEncoding(
@ -266,7 +338,7 @@ internal sealed partial class HeifEncoderCore
return new HeifSequenceEncoding( return new HeifSequenceEncoding(
image.Width, image.Width,
image.Height, image.Height,
image.Metadata.GetHeifMetadata().RepeatCount, this.encoder.RepeatCount ?? image.Metadata.GetHeifMetadata().RepeatCount,
timescale, timescale,
colorTrack, colorTrack,
alphaTrack, alphaTrack,
@ -279,7 +351,7 @@ internal sealed partial class HeifEncoderCore
private int WriteSequenceFileTypeBox(Stream stream) private int WriteSequenceFileTypeBox(Stream stream)
{ {
Span<byte> buffer = stackalloc byte[32]; Span<byte> buffer = stackalloc byte[44];
int bytesWritten = WriteBoxHeader(buffer, Heif4CharCode.Ftyp); int bytesWritten = WriteBoxHeader(buffer, Heif4CharCode.Ftyp);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis);
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
@ -287,18 +359,24 @@ internal sealed partial class HeifEncoderCore
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avif); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avif);
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avio);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avis);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Msf1); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Msf1);
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Iso8); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Iso8);
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Avio); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Mif1);
bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)Heif4CharCode.Miaf);
bytesWritten += sizeof(uint); bytesWritten += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(buffer, (uint)bytesWritten); BinaryPrimitives.WriteUInt32BigEndian(buffer, (uint)bytesWritten);
stream.Write(buffer[..bytesWritten]); stream.Write(buffer[..bytesWritten]);
return bytesWritten; return bytesWritten;
} }
private void WriteSequenceMovieBox(HeifSequenceEncoding sequence, int fileTypeLength, Stream stream) private void WriteSequenceMovieBox(HeifSequenceEncoding sequence, int precedingBoxLength, Stream stream)
{ {
int movieLength = GetSequenceMovieBoxLength(sequence); int movieLength = GetSequenceMovieBoxLength(sequence);
using IMemoryOwner<byte> movieOwner = this.configuration.MemoryAllocator.Allocate<byte>(movieLength); using IMemoryOwner<byte> movieOwner = this.configuration.MemoryAllocator.Allocate<byte>(movieLength);
@ -343,7 +421,7 @@ internal sealed partial class HeifEncoderCore
} }
EndSequenceBox(memory, movieStart, offset); EndSequenceBox(memory, movieStart, offset);
ulong mediaDataOffset = checked((ulong)fileTypeLength + (uint)offset + 8U); ulong mediaDataOffset = checked((ulong)precedingBoxLength + (uint)offset + 8U);
BinaryPrimitives.WriteUInt64BigEndian( BinaryPrimitives.WriteUInt64BigEndian(
memory[colorChunkOffsetPosition..], memory[colorChunkOffsetPosition..],
checked(mediaDataOffset + (ulong)sequence.ColorTrack.Samples[0].Offset)); checked(mediaDataOffset + (ulong)sequence.ColorTrack.Samples[0].Offset));
@ -377,7 +455,8 @@ internal sealed partial class HeifEncoderCore
const int chunkOffsetBoxLength = 24; const int chunkOffsetBoxLength = 24;
const int syncSampleBoxFixedLength = 16; const int syncSampleBoxFixedLength = 16;
const int timingRunLength = 8; const int timingRunLength = 8;
const int sampleSizeAndSyncEntryLength = 8; const int sampleSizeEntryLength = sizeof(uint);
const int syncSampleEntryLength = sizeof(uint);
const int sampleTableFixedLength = const int sampleTableFixedLength =
sampleTableBoxHeaderLength sampleTableBoxHeaderLength
+ sampleDescriptionBoxLength + sampleDescriptionBoxLength
@ -398,10 +477,12 @@ internal sealed partial class HeifEncoderCore
int repeatBoxLength = sequence.RepeatCount == 1 ? 0 : editListBoxLength; int repeatBoxLength = sequence.RepeatCount == 1 ? 0 : editListBoxLength;
int colorRunCount = GetSequenceTimingRunCount(sequence.ColorTrack.Samples); int colorRunCount = GetSequenceTimingRunCount(sequence.ColorTrack.Samples);
int colorSyncSampleCount = GetSequenceSyncSampleCount(sequence.ColorTrack.Samples);
long colorSampleTableLength = long colorSampleTableLength =
(long)sampleTableFixedLength (long)sampleTableFixedLength
+ (colorRunCount * timingRunLength) + (colorRunCount * timingRunLength)
+ (sequence.ColorTrack.Samples.Length * sampleSizeAndSyncEntryLength) + (sequence.ColorTrack.Samples.Length * sampleSizeEntryLength)
+ (colorSyncSampleCount * syncSampleEntryLength)
+ colorInformationBoxLength; + colorInformationBoxLength;
if (!sequence.IccProfileData.IsEmpty) if (!sequence.IccProfileData.IsEmpty)
@ -436,6 +517,7 @@ internal sealed partial class HeifEncoderCore
{ {
HeifSequenceTrackEncoding alphaTrack = sequence.AlphaTrack.GetValueOrDefault(); HeifSequenceTrackEncoding alphaTrack = sequence.AlphaTrack.GetValueOrDefault();
int alphaRunCount = GetSequenceTimingRunCount(alphaTrack.Samples); int alphaRunCount = GetSequenceTimingRunCount(alphaTrack.Samples);
int alphaSyncSampleCount = GetSequenceSyncSampleCount(alphaTrack.Samples);
int auxiliaryTypeBoxLength = int auxiliaryTypeBoxLength =
FullBoxHeaderLength FullBoxHeaderLength
+ Encoding.UTF8.GetByteCount(HeifConstants.AlphaAuxiliaryType) + Encoding.UTF8.GetByteCount(HeifConstants.AlphaAuxiliaryType)
@ -444,7 +526,8 @@ internal sealed partial class HeifEncoderCore
long alphaSampleTableLength = long alphaSampleTableLength =
(long)sampleTableFixedLength (long)sampleTableFixedLength
+ (alphaRunCount * timingRunLength) + (alphaRunCount * timingRunLength)
+ (alphaTrack.Samples.Length * sampleSizeAndSyncEntryLength) + (alphaTrack.Samples.Length * sampleSizeEntryLength)
+ (alphaSyncSampleCount * syncSampleEntryLength)
+ auxiliaryTypeBoxLength; + auxiliaryTypeBoxLength;
alphaTrackLength = alphaTrackLength =
@ -774,13 +857,19 @@ internal sealed partial class HeifEncoderCore
WriteSequenceUInt64(memory, ref offset, 0); WriteSequenceUInt64(memory, ref offset, 0);
EndSequenceBox(memory, chunkOffsetsStart, offset); EndSequenceBox(memory, chunkOffsetsStart, offset);
// The current bounded sequence encoder emits independent all-intra pictures; every sample is seekable. int syncSampleCount = GetSequenceSyncSampleCount(track.Samples);
int syncSamplesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stss); int syncSamplesStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Stss);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0); WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
WriteSequenceUInt32(memory, ref offset, (uint)track.Samples.Length); WriteSequenceUInt32(memory, ref offset, (uint)syncSampleCount);
for (uint sampleIndex = 1; sampleIndex <= track.Samples.Length; sampleIndex++) uint sampleNumber = 1;
foreach (HeifSequenceSampleInfo sample in track.Samples)
{ {
WriteSequenceUInt32(memory, ref offset, sampleIndex); if (sample.IsSyncSample)
{
WriteSequenceUInt32(memory, ref offset, sampleNumber);
}
sampleNumber++;
} }
EndSequenceBox(memory, syncSamplesStart, offset); EndSequenceBox(memory, syncSamplesStart, offset);
@ -839,9 +928,15 @@ internal sealed partial class HeifEncoderCore
int codingConstraintsStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Ccst); int codingConstraintsStart = BeginSequenceBox(memory, ref offset, Heif4CharCode.Ccst);
WriteSequenceFullBoxHeader(memory, ref offset, 0, 0); WriteSequenceFullBoxHeader(memory, ref offset, 0, 0);
// Every emitted sequence sample is independently decodable, while intra prediction remains available inside uint codingConstraints = IntraPicturePredictionUsedMask;
// each picture. No inter-picture reference slot is therefore advertised. if (GetSequenceSyncSampleCount(track.Samples) == track.Samples.Length)
WriteSequenceUInt32(memory, ref offset, AllReferencePicturesIntraMask | IntraPicturePredictionUsedMask); {
codingConstraints |= AllReferencePicturesIntraMask;
}
// Sync samples are key frames in this encoder. The all-intra flag is therefore valid when every sample
// is independently decodable, including lossless sequences that deliberately avoid inter transforms.
WriteSequenceUInt32(memory, ref offset, codingConstraints);
EndSequenceBox(memory, codingConstraintsStart, offset); EndSequenceBox(memory, codingConstraintsStart, offset);
EndSequenceBox(memory, sampleEntryStart, offset); EndSequenceBox(memory, sampleEntryStart, offset);
EndSequenceBox(memory, descriptionStart, offset); EndSequenceBox(memory, descriptionStart, offset);
@ -892,6 +987,20 @@ internal sealed partial class HeifEncoderCore
return runCount; return runCount;
} }
private static int GetSequenceSyncSampleCount(ReadOnlySpan<HeifSequenceSampleInfo> samples)
{
int count = 0;
foreach (HeifSequenceSampleInfo sample in samples)
{
if (sample.IsSyncSample)
{
count++;
}
}
return count;
}
private static uint GetSequenceSampleDuration(Rational delay, uint timescale) private static uint GetSequenceSampleDuration(Rational delay, uint timescale)
{ {
// HEIF metadata uses either a zero numerator or a zero denominator for an unspecified duration. // HEIF metadata uses either a zero numerator or a zero denominator for an unspecified duration.
@ -905,12 +1014,13 @@ internal sealed partial class HeifEncoderCore
return checked((uint)Math.Max(1UL, scaledDuration / delay.Denominator)); return checked((uint)Math.Max(1UL, scaledDuration / delay.Denominator));
} }
private static uint GetSequenceTimescale<TPixel>(Image<TPixel> image) private static uint GetSequenceTimescale<TPixel>(Image<TPixel> image, int firstFrameIndex)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
uint timescale = DefaultSequenceTimescale; uint timescale = DefaultSequenceTimescale;
foreach (ImageFrame<TPixel> frame in image.Frames) for (int frameIndex = firstFrameIndex; frameIndex < image.Frames.Count; frameIndex++)
{ {
ImageFrame<TPixel> frame = image.Frames[frameIndex];
Rational delay = frame.Metadata.GetHeifMetadata().FrameDelay; Rational delay = frame.Metadata.GetHeifMetadata().FrameDelay;
if (delay.Numerator == 0 || delay.Denominator == 0) if (delay.Numerator == 0 || delay.Denominator == 0)
{ {
@ -1070,11 +1180,12 @@ internal sealed partial class HeifEncoderCore
private readonly struct HeifSequenceSampleInfo private readonly struct HeifSequenceSampleInfo
{ {
public HeifSequenceSampleInfo(long offset, int length, uint duration) public HeifSequenceSampleInfo(long offset, int length, uint duration, bool isSyncSample)
{ {
this.Offset = offset; this.Offset = offset;
this.Length = length; this.Length = length;
this.Duration = duration; this.Duration = duration;
this.IsSyncSample = isSyncSample;
} }
public long Offset { get; } public long Offset { get; }
@ -1082,6 +1193,8 @@ internal sealed partial class HeifEncoderCore
public int Length { get; } public int Length { get; }
public uint Duration { get; } public uint Duration { get; }
public bool IsSyncSample { get; }
} }
private readonly struct HeifSequenceEncoding private readonly struct HeifSequenceEncoding

623
src/ImageSharp/Formats/Heif/HeifEncoderCore.cs

@ -10,7 +10,6 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers;
using SixLabors.ImageSharp.Formats.Jpeg; using SixLabors.ImageSharp.Formats.Jpeg;
using SixLabors.ImageSharp.IO; using SixLabors.ImageSharp.IO;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata; using SixLabors.ImageSharp.Metadata;
using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.Metadata.Profiles.Cicp;
using SixLabors.ImageSharp.Metadata.Profiles.Icc; using SixLabors.ImageSharp.Metadata.Profiles.Icc;
@ -46,6 +45,32 @@ internal sealed partial class HeifEncoderCore
private const int MaximumCompactPropertyIndex = 0x7F; private const int MaximumCompactPropertyIndex = 0x7F;
private const ushort EssentialPropertyFlag = 0x8000; private const ushort EssentialPropertyFlag = 0x8000;
private const byte CompactEssentialPropertyFlag = 0x80; private const byte CompactEssentialPropertyFlag = 0x80;
private const uint HiddenImageItemFlag = 1;
/// <summary>
/// The version defined for the AVIF grid item payload.
/// </summary>
private const byte GridDescriptorVersion = 0;
/// <summary>
/// The largest row or column count representable by a grid descriptor.
/// </summary>
private const int MaximumGridAxisCellCount = byte.MaxValue + 1;
/// <summary>
/// The minimum width and height permitted for the first cell of an AVIF grid.
/// </summary>
private const int MinimumGridCellDimension = 64;
/// <summary>
/// The grid descriptor length when output dimensions use 32-bit fields.
/// </summary>
private const int LongGridDescriptorLength = 12;
/// <summary>
/// Selects 32-bit output dimensions in a grid descriptor.
/// </summary>
private const byte LargeGridDimensionsFlag = 1;
/// <summary> /// <summary>
/// The global configuration. /// The global configuration.
@ -81,24 +106,93 @@ internal sealed partial class HeifEncoderCore
Guard.NotNull(image, nameof(image)); Guard.NotNull(image, nameof(image));
Guard.NotNull(stream, nameof(stream)); Guard.NotNull(stream, nameof(stream));
switch (this.encoder.CompressionMethod)
{
case HeifCompressionMethod.LegacyJpeg:
break;
case HeifCompressionMethod.Av1:
if (image.Frames.Count > 1)
{
if (image.Width > ushort.MaxValue || image.Height > ushort.MaxValue)
{
throw new NotSupportedException("AV1 image-sequence dimensions cannot exceed 65535 pixels.");
}
}
break;
default:
throw new NotSupportedException($"HEIF compression method '{this.encoder.CompressionMethod}' is not supported.");
}
using ChunkedMemoryStream compressedPixels = new(this.configuration.MemoryAllocator); using ChunkedMemoryStream compressedPixels = new(this.configuration.MemoryAllocator);
if (this.encoder.CompressionMethod == HeifCompressionMethod.Av1 && image.Frames.Count > 1) if (this.encoder.CompressionMethod == HeifCompressionMethod.Av1 && image.Frames.Count > 1)
{ {
Av1EncodingSettings settings = this.ResolveAv1Encoding(image); Av1EncodingSettings settings = this.ResolveAv1Encoding(image);
int sampleCount = image.Frames.Count * (settings.HasAlpha ? 2 : 1); bool animateRootFrame = this.encoder.AnimateRootFrame
?? image.Metadata.GetHeifMetadata().AnimateRootFrame;
int firstFrameIndex = animateRootFrame ? 0 : 1;
int sequenceFrameCount = image.Frames.Count - firstFrameIndex;
int sampleCount = sequenceFrameCount * (settings.HasAlpha ? 2 : 1);
using IMemoryOwner<HeifSequenceSampleInfo> samplesOwner = using IMemoryOwner<HeifSequenceSampleInfo> samplesOwner =
this.configuration.MemoryAllocator.Allocate<HeifSequenceSampleInfo>(sampleCount); this.configuration.MemoryAllocator.Allocate<HeifSequenceSampleInfo>(sampleCount);
List<HeifItem> sequenceItems = new();
List<HeifItemLink> sequenceLinks = new();
if (!animateRootFrame)
{
Av1ImageItemEncoding primaryImage = this.CompressAv1ImageItem(
image.Frames.RootFrame,
compressedPixels,
settings,
cancellationToken);
this.WriteAv1ImageItems(
image,
compressedPixels,
settings,
primaryImage,
sequenceItems,
sequenceLinks);
}
Memory<HeifSequenceSampleInfo> samples = samplesOwner.Memory[..sampleCount]; Memory<HeifSequenceSampleInfo> samples = samplesOwner.Memory[..sampleCount];
HeifSequenceEncoding sequence = this.CompressAv1Sequence( HeifSequenceEncoding sequence = this.CompressAv1Sequence(
image, image,
compressedPixels, compressedPixels,
settings, settings,
samples, samples,
firstFrameIndex,
cancellationToken); cancellationToken);
if (animateRootFrame)
{
HeifSequenceSampleInfo colorSample = sequence.ColorTrack.Samples[0];
HeifSequenceTrackEncoding? alphaTrack = sequence.AlphaTrack;
Av1ImageItemEncoding primaryImage = new(
sequence.ColorTrack.Configuration,
colorSample.Offset,
colorSample.Length,
alphaTrack?.Configuration,
alphaTrack?.Samples[0].Offset ?? 0,
alphaTrack?.Samples[0].Length ?? 0);
// The primary image item and the first track sample describe the same sync sample. Sharing its
// extent matches libavif and avoids encoding or storing the root frame twice.
this.WriteAv1ImageItems(
image,
compressedPixels,
settings,
primaryImage,
sequenceItems,
sequenceLinks);
}
int fileTypeLength = this.WriteSequenceFileTypeBox(stream); int fileTypeLength = this.WriteSequenceFileTypeBox(stream);
this.WriteSequenceMovieBox(sequence, fileTypeLength, stream); int metadataLength = GetMetadataBoxLength(sequenceItems, sequenceLinks);
int movieLength = GetSequenceMovieBoxLength(sequence);
this.WriteMetadataBox(sequenceItems, sequenceLinks, fileTypeLength, movieLength, stream);
this.WriteSequenceMovieBox(sequence, fileTypeLength + metadataLength, stream);
this.WriteMediaDataBox(compressedPixels, stream); this.WriteMediaDataBox(compressedPixels, stream);
stream.Flush(); stream.Flush();
return; return;
@ -115,13 +209,11 @@ internal sealed partial class HeifEncoderCore
case HeifCompressionMethod.Av1: case HeifCompressionMethod.Av1:
this.CompressAv1Pixels(image, compressedPixels, items, links, cancellationToken); this.CompressAv1Pixels(image, compressedPixels, items, links, cancellationToken);
break; break;
default:
throw new NotSupportedException($"HEIF compression method '{this.encoder.CompressionMethod}' is not supported.");
} }
// Write out the generated header and pixels. // Write out the generated header and pixels.
long metadataBoxOffset = this.WriteFileTypeBox(stream); long metadataBoxOffset = this.WriteFileTypeBox(stream);
this.WriteMetadataBox(items, links, metadataBoxOffset, stream); this.WriteMetadataBox(items, links, metadataBoxOffset, 0, stream);
this.WriteMediaDataBox(compressedPixels, stream); this.WriteMediaDataBox(compressedPixels, stream);
stream.Flush(); stream.Flush();
} }
@ -229,8 +321,14 @@ internal sealed partial class HeifEncoderCore
/// <param name="items">The declared image and metadata items.</param> /// <param name="items">The declared image and metadata items.</param>
/// <param name="links">The typed relationships between items.</param> /// <param name="links">The typed relationships between items.</param>
/// <param name="metadataBoxOffset">The metadata box offset from the start of the encoded file.</param> /// <param name="metadataBoxOffset">The metadata box offset from the start of the encoded file.</param>
/// <param name="followingBoxLength">The number of bytes between this box and the media-data box.</param>
/// <param name="stream">The destination stream positioned after the file-type box.</param> /// <param name="stream">The destination stream positioned after the file-type box.</param>
private void WriteMetadataBox(List<HeifItem> items, List<HeifItemLink> links, long metadataBoxOffset, Stream stream) private void WriteMetadataBox(
List<HeifItem> items,
List<HeifItemLink> links,
long metadataBoxOffset,
int followingBoxLength,
Stream stream)
{ {
int metadataLength = GetMetadataBoxLength(items, links); int metadataLength = GetMetadataBoxLength(items, links);
using IMemoryOwner<byte> metadataOwner = this.configuration.MemoryAllocator.Allocate<byte>(metadataLength); using IMemoryOwner<byte> metadataOwner = this.configuration.MemoryAllocator.Allocate<byte>(metadataLength);
@ -254,7 +352,7 @@ internal sealed partial class HeifEncoderCore
bytesWritten += WriteItemLocationBox(memory, bytesWritten, items, 0); bytesWritten += WriteItemLocationBox(memory, bytesWritten, items, 0);
// The mdat payload immediately follows the completed meta box and its own eight-byte header. // The mdat payload immediately follows the completed meta box and its own eight-byte header.
long mediaDataOffset = checked(metadataBoxOffset + bytesWritten + BasicBoxHeaderLength); long mediaDataOffset = checked(metadataBoxOffset + bytesWritten + followingBoxLength + BasicBoxHeaderLength);
WriteItemLocationBox(memory, itemLocationOffset, items, mediaDataOffset); WriteItemLocationBox(memory, itemLocationOffset, items, mediaDataOffset);
buffer = memory[..bytesWritten]; buffer = memory[..bytesWritten];
@ -315,13 +413,21 @@ internal sealed partial class HeifEncoderCore
{ {
long propertyCount = 0; long propertyCount = 0;
long associationItemCount = 0; long associationItemCount = 0;
long associationPropertyCount = 0;
long propertyBytes = 0; long propertyBytes = 0;
foreach (HeifItem item in items) foreach (HeifItem item in items)
{ {
int itemPropertyCount = GetPropertyCount(item); HeifItem propertyItem = item.PropertySource ?? item;
propertyCount += itemPropertyCount; int itemPropertyCount = GetPropertyCount(propertyItem);
associationItemCount += itemPropertyCount == 0 ? 0 : 1; associationItemCount += itemPropertyCount == 0 ? 0 : 1;
associationPropertyCount += itemPropertyCount;
if (item.PropertySource is not null)
{
continue;
}
propertyCount += itemPropertyCount;
propertyBytes += item.Extent == default ? 0 : SpatialExtentPropertyBoxLength; propertyBytes += item.Extent == default ? 0 : SpatialExtentPropertyBoxLength;
if (item.ChannelBitDepths is not null) if (item.ChannelBitDepths is not null)
{ {
@ -348,7 +454,7 @@ internal sealed partial class HeifEncoderCore
long length = ItemPropertiesBoxFixedLength long length = ItemPropertiesBoxFixedLength
+ propertyBytes + propertyBytes
+ (associationItemCount * PropertyAssociationEntryFixedLength) + (associationItemCount * PropertyAssociationEntryFixedLength)
+ (propertyCount * associationSize); + (associationPropertyCount * associationSize);
return checked((int)length); return checked((int)length);
} }
@ -425,7 +531,12 @@ internal sealed partial class HeifEncoderCore
foreach (HeifItem item in items) foreach (HeifItem item in items)
{ {
int itemLengthOffset = bytesWritten; int itemLengthOffset = bytesWritten;
bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Infe, 2, 0); bytesWritten += WriteBoxHeader(
buffer[bytesWritten..],
Heif4CharCode.Infe,
2,
item.IsHidden ? HiddenImageItemFlag : 0);
BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], (ushort)item.Id); BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], (ushort)item.Id);
bytesWritten += 2; bytesWritten += 2;
BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], 0); BinaryPrimitives.WriteUInt16BigEndian(buffer[bytesWritten..], 0);
@ -499,17 +610,26 @@ internal sealed partial class HeifEncoderCore
// ipco order defines the one-based property indices written later in ipma. // ipco order defines the one-based property indices written later in ipma.
int ipcoLengthOffset = bytesWritten; int ipcoLengthOffset = bytesWritten;
bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipco); bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipco);
ushort nextPropertyIndex = 1;
foreach (HeifItem item in items) foreach (HeifItem item in items)
{ {
if (item.PropertySource is not null)
{
continue;
}
item.FirstPropertyIndex = nextPropertyIndex;
if (item.Extent != default) if (item.Extent != default)
{ {
bytesWritten += WriteSpatialExtentPropertyBox(memory, memoryOffset + bytesWritten, item); bytesWritten += WriteSpatialExtentPropertyBox(memory, memoryOffset + bytesWritten, item);
nextPropertyIndex++;
} }
byte[]? channelBitDepths = item.ChannelBitDepths; byte[]? channelBitDepths = item.ChannelBitDepths;
if (channelBitDepths is not null) if (channelBitDepths is not null)
{ {
bytesWritten += WritePixelInformationPropertyBox(memory, memoryOffset + bytesWritten, channelBitDepths); bytesWritten += WritePixelInformationPropertyBox(memory, memoryOffset + bytesWritten, channelBitDepths);
nextPropertyIndex++;
} }
else else
{ {
@ -521,6 +641,8 @@ internal sealed partial class HeifEncoderCore
memoryOffset + bytesWritten, memoryOffset + bytesWritten,
item.ChannelCount, item.ChannelCount,
uniformChannelBitDepth.Value); uniformChannelBitDepth.Value);
nextPropertyIndex++;
} }
} }
@ -528,39 +650,43 @@ internal sealed partial class HeifEncoderCore
if (codecConfiguration is not null) if (codecConfiguration is not null)
{ {
bytesWritten += WriteAv1CodecConfigurationPropertyBox(memory, memoryOffset + bytesWritten, codecConfiguration); bytesWritten += WriteAv1CodecConfigurationPropertyBox(memory, memoryOffset + bytesWritten, codecConfiguration);
nextPropertyIndex++;
} }
string? auxiliaryType = item.AuxiliaryType; string? auxiliaryType = item.AuxiliaryType;
if (auxiliaryType is not null) if (auxiliaryType is not null)
{ {
bytesWritten += WriteAuxiliaryTypePropertyBox(memory, memoryOffset + bytesWritten, auxiliaryType); bytesWritten += WriteAuxiliaryTypePropertyBox(memory, memoryOffset + bytesWritten, auxiliaryType);
nextPropertyIndex++;
} }
IccProfile? iccProfile = item.IccProfile; IccProfile? iccProfile = item.IccProfile;
if (iccProfile is not null) if (iccProfile is not null)
{ {
bytesWritten += WriteIccColorInformationPropertyBox(memory, memoryOffset + bytesWritten, item.GetIccProfileDataForWriting()); bytesWritten += WriteIccColorInformationPropertyBox(memory, memoryOffset + bytesWritten, item.GetIccProfileDataForWriting());
nextPropertyIndex++;
} }
CicpProfile? cicpProfile = item.CicpProfile; CicpProfile? cicpProfile = item.CicpProfile;
if (cicpProfile is not null) if (cicpProfile is not null)
{ {
bytesWritten += WriteColorInformationPropertyBox(memory, memoryOffset + bytesWritten, cicpProfile); bytesWritten += WriteColorInformationPropertyBox(memory, memoryOffset + bytesWritten, cicpProfile);
nextPropertyIndex++;
} }
} }
BinaryPrimitives.WriteUInt32BigEndian(buffer[ipcoLengthOffset..], (uint)(bytesWritten - ipcoLengthOffset)); BinaryPrimitives.WriteUInt32BigEndian(buffer[ipcoLengthOffset..], (uint)(bytesWritten - ipcoLengthOffset));
int propertyCount = 0; int propertyCount = nextPropertyIndex - 1;
int associationItemCount = 0; int associationItemCount = 0;
foreach (HeifItem item in items) foreach (HeifItem item in items)
{ {
int itemPropertyCount = GetPropertyCount(item); HeifItem propertyItem = item.PropertySource ?? item;
int itemPropertyCount = GetPropertyCount(propertyItem);
if (itemPropertyCount == 0) if (itemPropertyCount == 0)
{ {
continue; continue;
} }
propertyCount += itemPropertyCount;
associationItemCount++; associationItemCount++;
} }
@ -571,10 +697,10 @@ internal sealed partial class HeifEncoderCore
bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipma, 0, largePropertyIndex ? 1U : 0U); bytesWritten += WriteBoxHeader(buffer[bytesWritten..], Heif4CharCode.Ipma, 0, largePropertyIndex ? 1U : 0U);
BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)associationItemCount); BinaryPrimitives.WriteUInt32BigEndian(buffer[bytesWritten..], (uint)associationItemCount);
bytesWritten += 4; bytesWritten += 4;
ushort propertyIndex = 1;
foreach (HeifItem item in items) foreach (HeifItem item in items)
{ {
int itemPropertyCount = GetPropertyCount(item); HeifItem propertyItem = item.PropertySource ?? item;
int itemPropertyCount = GetPropertyCount(propertyItem);
if (itemPropertyCount == 0) if (itemPropertyCount == 0)
{ {
continue; continue;
@ -584,32 +710,33 @@ internal sealed partial class HeifEncoderCore
bytesWritten += 2; bytesWritten += 2;
buffer[bytesWritten++] = (byte)itemPropertyCount; buffer[bytesWritten++] = (byte)itemPropertyCount;
if (item.Extent != default) ushort propertyIndex = propertyItem.FirstPropertyIndex;
if (propertyItem.Extent != default)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false);
} }
if (item.ChannelBitDepths is not null || item.UniformChannelBitDepth is not null) if (propertyItem.ChannelBitDepths is not null || propertyItem.UniformChannelBitDepth is not null)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false);
} }
if (item.Av1CodecConfiguration is not null) if (propertyItem.Av1CodecConfiguration is not null)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, true); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, true);
} }
if (item.AuxiliaryType is not null) if (propertyItem.AuxiliaryType is not null)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false);
} }
if (item.IccProfile is not null) if (propertyItem.IccProfile is not null)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false);
} }
if (item.CicpProfile is not null) if (propertyItem.CicpProfile is not null)
{ {
WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false); WritePropertyAssociation(buffer, ref bytesWritten, propertyIndex++, largePropertyIndex, false);
} }
@ -918,68 +1045,396 @@ internal sealed partial class HeifEncoderCore
CancellationToken cancellationToken) CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
byte[]? exifData = null; Av1EncodingSettings settings = this.ResolveAv1Encoding(image);
uint tiffHeaderOffset = 0; if (image.Width > Av1Constants.MaxFrameDimension || image.Height > Av1Constants.MaxFrameDimension)
byte[]? xmpData = null; {
this.CompressAv1GridPixels(image, stream, settings, items, links, cancellationToken);
return;
}
Av1ImageItemEncoding encoding = this.CompressAv1ImageItem(
image.Frames.RootFrame,
stream,
settings,
cancellationToken);
this.WriteAv1ImageItems(image, stream, settings, encoding, items, links);
}
/// <summary>
/// Encodes a still image as independently coded AV1 cells referenced by one derived grid item.
/// </summary>
private void CompressAv1GridPixels<TPixel>(
Image<TPixel> image,
ChunkedMemoryStream stream,
Av1EncodingSettings settings,
List<HeifItem> items,
List<HeifItemLink> links,
CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel>
{
bool isSubsampledX = !settings.ColorConfig.IsMonochrome && settings.ColorConfig.SubSamplingX;
bool isSubsampledY = !settings.ColorConfig.IsMonochrome && settings.ColorConfig.SubSamplingY;
if ((isSubsampledX && (image.Width & 1) != 0) || (isSubsampledY && (image.Height & 1) != 0))
{
throw new NotSupportedException("AVIF grid output dimensions must be even along each subsampled chroma axis.");
}
int columns = GetGridCellCount(image.Width, Av1Constants.MaxFrameDimension);
int rows = GetGridCellCount(image.Height, Av1Constants.MaxFrameDimension);
if (columns > MaximumGridAxisCellCount || rows > MaximumGridAxisCellCount)
{
throw new NotSupportedException(
$"AVIF grids support at most {MaximumGridAxisCellCount} columns and rows.");
}
int cellWidth = GetGridCellSize(image.Width, columns, isSubsampledX);
int cellHeight = GetGridCellSize(image.Height, rows, isSubsampledY);
Size encodedCellSize = new(
Math.Max(cellWidth, MinimumGridCellDimension),
Math.Max(cellHeight, MinimumGridCellDimension));
long cellCount = (long)columns * rows;
long itemCount = 1 + cellCount;
if (settings.HasAlpha)
{
itemCount += 1 + cellCount;
}
if (!this.encoder.SkipMetadata) if (!this.encoder.SkipMetadata)
{ {
exifData = GetExifData(image.Metadata, out tiffHeaderOffset); itemCount += image.Metadata.ExifProfile is null ? 0 : 1;
byte[]? sourceXmpData = image.Metadata.XmpProfile?.Data; itemCount += image.Metadata.XmpProfile is null ? 0 : 1;
if (sourceXmpData is not null && sourceXmpData.Length > 0) }
if (itemCount > ushort.MaxValue)
{
throw new NotSupportedException(
$"The encoded AVIF grid requires {itemCount} items, but this container supports at most {ushort.MaxValue}.");
}
byte channelBitDepth = (byte)settings.BitDepth;
long descriptorOffset = stream.Length;
int descriptorLength = WriteGridDescriptor(stream, rows, columns, image.Size);
HeifItem colorGrid = new(Heif4CharCode.Grid, 1)
{
ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3,
UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3),
IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile,
CicpProfile = settings.ColorProfile
};
colorGrid.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
descriptorOffset,
descriptorLength));
colorGrid.SetExtent(image.Size);
items.Add(colorGrid);
HeifItemLink colorGridLink = new(Heif4CharCode.Dimg, colorGrid.Id);
links.Add(colorGridLink);
HeifItem? colorPropertySource = null;
ImageFrame<TPixel> rootFrame = image.Frames.RootFrame;
for (int row = 0; row < rows; row++)
{
int y = row * cellHeight;
int height = Math.Min(cellHeight, image.Height - y);
for (int column = 0; column < columns; column++)
{ {
xmpData = sourceXmpData; cancellationToken.ThrowIfCancellationRequested();
int x = column * cellWidth;
int width = Math.Min(cellWidth, image.Width - x);
Rectangle sourceRectangle = new(x, y, width, height);
long colorOffset = stream.Length;
ObuSequenceHeader colorHeader = Av1FrameEncoder.EncodeGridCell(
this.configuration,
rootFrame,
sourceRectangle,
encodedCellSize,
stream,
settings.ColorConfig,
settings.ColorQIndex,
this.encoder.Effort);
long colorLength = stream.Length - colorOffset;
HeifItem colorCell = new(Heif4CharCode.Av01, (uint)items.Count + 1)
{
IsHidden = true,
ChannelCount = colorGrid.ChannelCount,
UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = colorGrid.BitsPerPixel,
Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader),
IccProfile = colorGrid.IccProfile,
CicpProfile = settings.ColorProfile
};
colorCell.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
colorOffset,
colorLength));
colorCell.SetExtent(encodedCellSize);
ShareGridCellProperties(colorCell, ref colorPropertySource);
items.Add(colorCell);
colorGridLink.DestinationIds.Add(colorCell.Id);
} }
} }
Av1EncodingSettings settings = this.ResolveAv1Encoding(image); if (settings.HasAlpha)
{
descriptorOffset = stream.Length;
descriptorLength = WriteGridDescriptor(stream, rows, columns, image.Size);
HeifItem alphaGrid = new(Heif4CharCode.Grid, (uint)items.Count + 1)
{
ChannelCount = 1,
UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth,
AuxiliaryType = HeifConstants.AlphaAuxiliaryType
};
alphaGrid.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
descriptorOffset,
descriptorLength));
alphaGrid.SetExtent(image.Size);
items.Add(alphaGrid);
HeifItemLink alphaGridLink = new(Heif4CharCode.Dimg, alphaGrid.Id);
links.Add(alphaGridLink);
HeifItemLink alphaLink = new(Heif4CharCode.Auxl, alphaGrid.Id);
alphaLink.DestinationIds.Add(colorGrid.Id);
links.Add(alphaLink);
HeifItem? alphaPropertySource = null;
for (int row = 0; row < rows; row++)
{
int y = row * cellHeight;
int height = Math.Min(cellHeight, image.Height - y);
for (int column = 0; column < columns; column++)
{
cancellationToken.ThrowIfCancellationRequested();
int x = column * cellWidth;
int width = Math.Min(cellWidth, image.Width - x);
Rectangle sourceRectangle = new(x, y, width, height);
long alphaOffset = stream.Length;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlphaGridCell(
this.configuration,
rootFrame,
sourceRectangle,
encodedCellSize,
stream,
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
long alphaLength = stream.Length - alphaOffset;
HeifItem alphaCell = new(Heif4CharCode.Av01, (uint)items.Count + 1)
{
IsHidden = true,
ChannelCount = 1,
UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth,
Av1CodecConfiguration = new Av1CodecConfiguration(alphaHeader),
AuxiliaryType = HeifConstants.AlphaAuxiliaryType
};
alphaCell.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
alphaOffset,
alphaLength));
alphaCell.SetExtent(encodedCellSize);
ShareGridCellProperties(alphaCell, ref alphaPropertySource);
items.Add(alphaCell);
alphaGridLink.DestinationIds.Add(alphaCell.Id);
}
}
}
this.WriteMetadataItems(image, stream, colorGrid, items, links);
}
/// <summary>
/// Gets the minimum number of independently coded cells needed along one grid axis.
/// </summary>
/// <param name="dimension">The complete output dimension along the axis.</param>
/// <param name="maximumCellDimension">The largest permitted nominal cell dimension.</param>
private static int GetGridCellCount(int dimension, int maximumCellDimension)
=> (int)(((long)dimension + maximumCellDimension - 1) / maximumCellDimension);
/// <summary>
/// Gets the nominal cell size while preserving chroma alignment for every non-edge cell.
/// </summary>
private static int GetGridCellSize(int dimension, int cellCount, bool isSubsampled)
{
int cellSize = (int)(((long)dimension + cellCount - 1) / cellCount);
if (isSubsampled && (cellSize & 1) != 0)
{
cellSize++;
}
return cellSize;
}
/// <summary>
/// Writes the fixed grid item payload and returns its exact length.
/// </summary>
private static int WriteGridDescriptor(Stream stream, int rows, int columns, Size outputSize)
{
bool usesLargeDimensions = outputSize.Width > ushort.MaxValue || outputSize.Height > ushort.MaxValue;
Span<byte> descriptor = stackalloc byte[LongGridDescriptorLength];
int descriptorLength = 0;
descriptor[descriptorLength++] = GridDescriptorVersion;
descriptor[descriptorLength++] = usesLargeDimensions ? LargeGridDimensionsFlag : (byte)0;
descriptor[descriptorLength++] = (byte)(rows - 1);
descriptor[descriptorLength++] = (byte)(columns - 1);
if (usesLargeDimensions)
{
BinaryPrimitives.WriteUInt32BigEndian(descriptor[descriptorLength..], (uint)outputSize.Width);
descriptorLength += sizeof(uint);
BinaryPrimitives.WriteUInt32BigEndian(descriptor[descriptorLength..], (uint)outputSize.Height);
descriptorLength += sizeof(uint);
}
else
{
BinaryPrimitives.WriteUInt16BigEndian(descriptor[descriptorLength..], (ushort)outputSize.Width);
descriptorLength += sizeof(ushort);
BinaryPrimitives.WriteUInt16BigEndian(descriptor[descriptorLength..], (ushort)outputSize.Height);
descriptorLength += sizeof(ushort);
}
stream.Write(descriptor[..descriptorLength]);
return descriptorLength;
}
/// <summary>
/// Reuses the common property set emitted for the first cell in one grid plane.
/// </summary>
private static void ShareGridCellProperties(HeifItem item, ref HeifItem? source)
{
if (source is null)
{
source = item;
return;
}
// Every cell in one plane is coded to the same extent and configuration so current AVIF readers can
// share one property set. Only the source rectangle differs for cells clipped by the output canvas.
item.PropertySource = source;
}
/// <summary>
/// Encodes one frame as the color and optional alpha payloads used by a primary AV1 image item.
/// </summary>
private Av1ImageItemEncoding CompressAv1ImageItem<TPixel>(
ImageFrame<TPixel> frame,
ChunkedMemoryStream stream,
Av1EncodingSettings settings,
CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel>
{
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
long colorOffset = stream.Length;
ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode( ObuSequenceHeader colorHeader = Av1FrameEncoder.Encode(
this.configuration, this.configuration,
image.Frames.RootFrame, frame,
stream, stream,
settings.ColorConfig, settings.ColorConfig,
settings.ColorQIndex, settings.ColorQIndex,
this.encoder.Effort); this.encoder.Effort);
long colorLength = stream.Length; long colorLength = stream.Length - colorOffset;
Av1CodecConfiguration? alphaConfiguration = null;
long alphaOffset = 0;
long alphaLength = 0;
if (settings.HasAlpha)
{
cancellationToken.ThrowIfCancellationRequested();
alphaOffset = stream.Length;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha(
this.configuration,
frame,
stream,
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
alphaLength = stream.Length - alphaOffset;
alphaConfiguration = new Av1CodecConfiguration(alphaHeader);
}
return new Av1ImageItemEncoding(
new Av1CodecConfiguration(colorHeader),
colorOffset,
colorLength,
alphaConfiguration,
alphaOffset,
alphaLength);
}
/// <summary>
/// Declares a primary AV1 image item over existing payload extents and appends its associated metadata payloads.
/// </summary>
private void WriteAv1ImageItems<TPixel>(
Image<TPixel> image,
ChunkedMemoryStream stream,
Av1EncodingSettings settings,
Av1ImageItemEncoding encoding,
List<HeifItem> items,
List<HeifItemLink> links)
where TPixel : unmanaged, IPixel<TPixel>
{
byte channelBitDepth = (byte)settings.BitDepth; byte channelBitDepth = (byte)settings.BitDepth;
HeifItem colorItem = new(Heif4CharCode.Av01, 1) HeifItem colorItem = new(Heif4CharCode.Av01, 1)
{ {
ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3, ChannelCount = settings.ColorConfig.IsMonochrome ? 1 : 3,
UniformChannelBitDepth = channelBitDepth, UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3), BitsPerPixel = channelBitDepth * (settings.ColorConfig.IsMonochrome ? 1 : 3),
Av1CodecConfiguration = new Av1CodecConfiguration(colorHeader), Av1CodecConfiguration = encoding.ColorConfiguration,
IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile, IccProfile = this.encoder.SkipMetadata ? null : image.Metadata.IccProfile,
CicpProfile = settings.ColorProfile CicpProfile = settings.ColorProfile
}; };
colorItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, 0L, colorLength)); colorItem.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
encoding.ColorOffset,
encoding.ColorLength));
colorItem.SetExtent(image.Size); colorItem.SetExtent(image.Size);
items.Add(colorItem); items.Add(colorItem);
if (settings.HasAlpha) Av1CodecConfiguration? alphaConfiguration = encoding.AlphaConfiguration;
if (alphaConfiguration is not null)
{ {
cancellationToken.ThrowIfCancellationRequested();
long alphaOffset = stream.Length;
ObuSequenceHeader alphaHeader = Av1FrameEncoder.EncodeAlpha(
this.configuration,
image.Frames.RootFrame,
stream,
settings.AlphaConfig,
settings.AlphaQIndex,
this.encoder.Effort);
long alphaLength = stream.Length - alphaOffset;
HeifItem alphaItem = new(Heif4CharCode.Av01, 2) HeifItem alphaItem = new(Heif4CharCode.Av01, 2)
{ {
ChannelCount = 1, ChannelCount = 1,
UniformChannelBitDepth = channelBitDepth, UniformChannelBitDepth = channelBitDepth,
BitsPerPixel = channelBitDepth, BitsPerPixel = channelBitDepth,
Av1CodecConfiguration = new Av1CodecConfiguration(alphaHeader), Av1CodecConfiguration = alphaConfiguration,
AuxiliaryType = HeifConstants.AlphaAuxiliaryType AuxiliaryType = HeifConstants.AlphaAuxiliaryType
}; };
alphaItem.DataLocations.Add(new HeifLocation(HeifLocationOffsetOrigin.FileOffset, 0L, alphaOffset, alphaLength)); alphaItem.DataLocations.Add(
new HeifLocation(
HeifLocationOffsetOrigin.FileOffset,
0L,
encoding.AlphaOffset,
encoding.AlphaLength));
alphaItem.SetExtent(image.Size); alphaItem.SetExtent(image.Size);
items.Add(alphaItem); items.Add(alphaItem);
HeifItemLink alphaLink = new(Heif4CharCode.Auxl, alphaItem.Id); HeifItemLink alphaLink = new(Heif4CharCode.Auxl, alphaItem.Id);
@ -987,15 +1442,30 @@ internal sealed partial class HeifEncoderCore
links.Add(alphaLink); links.Add(alphaLink);
} }
this.WriteMetadataItems(image, stream, colorItem, items, links);
}
/// <summary>
/// Appends Exif and XMP payload items associated with the primary presentation item.
/// </summary>
private void WriteMetadataItems<TPixel>(
Image<TPixel> image,
ChunkedMemoryStream stream,
HeifItem primaryItem,
List<HeifItem> items,
List<HeifItemLink> links)
where TPixel : unmanaged, IPixel<TPixel>
{
if (this.encoder.SkipMetadata) if (this.encoder.SkipMetadata)
{ {
return; return;
} }
byte[]? exifData = GetExifData(image.Metadata, out uint tiffHeaderOffset);
if (exifData is not null) if (exifData is not null)
{ {
long exifOffset = stream.Length; long exifOffset = stream.Length;
Span<byte> offsetBuffer = stackalloc byte[4]; Span<byte> offsetBuffer = stackalloc byte[sizeof(uint)];
BinaryPrimitives.WriteUInt32BigEndian(offsetBuffer, tiffHeaderOffset); BinaryPrimitives.WriteUInt32BigEndian(offsetBuffer, tiffHeaderOffset);
stream.Write(offsetBuffer); stream.Write(offsetBuffer);
stream.Write(exifData); stream.Write(exifData);
@ -1010,15 +1480,16 @@ internal sealed partial class HeifEncoderCore
HeifLocationOffsetOrigin.FileOffset, HeifLocationOffsetOrigin.FileOffset,
0L, 0L,
exifOffset, exifOffset,
4L + exifData.Length)); sizeof(uint) + (long)exifData.Length));
items.Add(exifItem); items.Add(exifItem);
HeifItemLink exifLink = new(Heif4CharCode.Cdsc, exifItem.Id); HeifItemLink exifLink = new(Heif4CharCode.Cdsc, exifItem.Id);
exifLink.DestinationIds.Add(colorItem.Id); exifLink.DestinationIds.Add(primaryItem.Id);
links.Add(exifLink); links.Add(exifLink);
} }
if (xmpData is not null) byte[]? xmpData = image.Metadata.XmpProfile?.Data;
if (xmpData is not null && xmpData.Length > 0)
{ {
long xmpOffset = stream.Length; long xmpOffset = stream.Length;
stream.Write(xmpData); stream.Write(xmpData);
@ -1037,7 +1508,7 @@ internal sealed partial class HeifEncoderCore
items.Add(xmpItem); items.Add(xmpItem);
HeifItemLink xmpLink = new(Heif4CharCode.Cdsc, xmpItem.Id); HeifItemLink xmpLink = new(Heif4CharCode.Cdsc, xmpItem.Id);
xmpLink.DestinationIds.Add(colorItem.Id); xmpLink.DestinationIds.Add(primaryItem.Id);
links.Add(xmpLink); links.Add(xmpLink);
} }
} }
@ -1093,16 +1564,6 @@ internal sealed partial class HeifEncoderCore
CancellationToken cancellationToken) CancellationToken cancellationToken)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
if (this.encoder.Lossless)
{
throw new NotSupportedException("Legacy JPEG image items do not support lossless encoding.");
}
if (this.encoder.BitDepth is not null and not HeifBitDepth.Bit8)
{
throw new NotSupportedException("Legacy JPEG image items support only 8-bit component encoding.");
}
JpegColorType colorType = this.encoder.ChromaSubsampling switch JpegColorType colorType = this.encoder.ChromaSubsampling switch
{ {
null or HeifChromaSubsampling.Yuv420 => JpegColorType.YCbCrRatio420, null or HeifChromaSubsampling.Yuv420 => JpegColorType.YCbCrRatio420,
@ -1125,4 +1586,38 @@ internal sealed partial class HeifEncoderCore
// cannot return while its pooled item payload is still being produced. // cannot return while its pooled item payload is still being produced.
image.SaveAsJpegAsync(stream, encoder, cancellationToken).GetAwaiter().GetResult(); image.SaveAsJpegAsync(stream, encoder, cancellationToken).GetAwaiter().GetResult();
} }
/// <summary>
/// Describes the already-written color and optional alpha extents backing one AV1 image item.
/// </summary>
private readonly struct Av1ImageItemEncoding
{
public Av1ImageItemEncoding(
Av1CodecConfiguration colorConfiguration,
long colorOffset,
long colorLength,
Av1CodecConfiguration? alphaConfiguration,
long alphaOffset,
long alphaLength)
{
this.ColorConfiguration = colorConfiguration;
this.ColorOffset = colorOffset;
this.ColorLength = colorLength;
this.AlphaConfiguration = alphaConfiguration;
this.AlphaOffset = alphaOffset;
this.AlphaLength = alphaLength;
}
public Av1CodecConfiguration ColorConfiguration { get; }
public long ColorOffset { get; }
public long ColorLength { get; }
public Av1CodecConfiguration? AlphaConfiguration { get; }
public long AlphaOffset { get; }
public long AlphaLength { get; }
}
} }

15
src/ImageSharp/Formats/Heif/HeifItem.cs

@ -28,6 +28,21 @@ internal sealed class HeifItem(Heif4CharCode type, uint id)
/// </summary> /// </summary>
public Heif4CharCode Type { get; } = type; public Heif4CharCode Type { get; } = type;
/// <summary>
/// Gets or sets a value indicating whether this item is excluded from primary-item discovery.
/// </summary>
public bool IsHidden { get; set; }
/// <summary>
/// Gets or sets an earlier item whose identical property associations are reused by this item.
/// </summary>
public HeifItem? PropertySource { get; set; }
/// <summary>
/// Gets or sets the first one-based property index assigned while writing the property container.
/// </summary>
public ushort FirstPropertyIndex { get; set; }
/// <summary> /// <summary>
/// Gets or sets the name of this item. /// Gets or sets the name of this item.
/// </summary> /// </summary>

16
src/ImageSharp/Processing/Processors/Transforms/Linear/FlipProcessor{TPixel}.cs

@ -60,15 +60,23 @@ internal class FlipProcessor<TPixel> : ImageProcessor<TPixel>
/// <inheritdoc/> /// <inheritdoc/>
protected override void OnFrameApply(ImageFrame<TPixel> source) protected override void OnFrameApply(ImageFrame<TPixel> source)
=> Apply(this.definition.FlipMode, source, this.Configuration);
/// <summary>
/// Applies an exact axis-aligned reflection to an existing frame.
/// </summary>
/// <param name="flipMode">The reflection direction.</param>
/// <param name="source">The frame modified in place.</param>
/// <param name="configuration">The configuration controlling row parallelism and scratch allocation.</param>
internal static void Apply(FlipMode flipMode, ImageFrame<TPixel> source, Configuration configuration)
{ {
switch (this.definition.FlipMode) switch (flipMode)
{ {
// No default needed as we have already set the pixels.
case FlipMode.Vertical: case FlipMode.Vertical:
FlipX(source.PixelBuffer, this.Configuration); FlipX(source.PixelBuffer, configuration);
break; break;
case FlipMode.Horizontal: case FlipMode.Horizontal:
FlipY(source, this.Configuration); FlipY(source, configuration);
break; break;
} }
} }

33
src/ImageSharp/Processing/Processors/Transforms/Linear/RotateProcessor{TPixel}.cs

@ -103,25 +103,52 @@ internal class RotateProcessor<TPixel> : AffineTransformProcessor<TPixel>
if (MathF.Abs(degrees - 90) < Constants.Epsilon) if (MathF.Abs(degrees - 90) < Constants.Epsilon)
{ {
Rotate90(source, destination, configuration); ApplyQuarterTurn(RotateMode.Rotate90, source, destination, configuration);
return true; return true;
} }
if (MathF.Abs(degrees - 180) < Constants.Epsilon) if (MathF.Abs(degrees - 180) < Constants.Epsilon)
{ {
Rotate180(source, destination, configuration); ApplyQuarterTurn(RotateMode.Rotate180, source, destination, configuration);
return true; return true;
} }
if (MathF.Abs(degrees - 270) < Constants.Epsilon) if (MathF.Abs(degrees - 270) < Constants.Epsilon)
{ {
Rotate270(source, destination, configuration); ApplyQuarterTurn(RotateMode.Rotate270, source, destination, configuration);
return true; return true;
} }
return false; return false;
} }
/// <summary>
/// Applies an exact quarter-turn rotation between already allocated frames.
/// </summary>
/// <param name="rotation">The clockwise quarter-turn rotation.</param>
/// <param name="source">The source frame.</param>
/// <param name="destination">The destination frame with the rotated dimensions.</param>
/// <param name="configuration">The configuration controlling row parallelism.</param>
internal static void ApplyQuarterTurn(
RotateMode rotation,
ImageFrame<TPixel> source,
ImageFrame<TPixel> destination,
Configuration configuration)
{
switch (rotation)
{
case RotateMode.Rotate90:
Rotate90(source, destination, configuration);
break;
case RotateMode.Rotate180:
Rotate180(source, destination, configuration);
break;
case RotateMode.Rotate270:
Rotate270(source, destination, configuration);
break;
}
}
/// <summary> /// <summary>
/// Rotates the image 180 degrees clockwise at the centre point. /// Rotates the image 180 degrees clockwise at the centre point.
/// </summary> /// </summary>

20
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1BitStreamTests.cs

@ -204,6 +204,26 @@ public class Av1BitStreamTests
Assert.Equal(values, actuals); Assert.Equal(values, actuals);
} }
[Fact]
public void SignedReferenceSubexponentialMatchesFiniteRecentering()
{
const int ValueMagnitude = 5;
const int GroupBitCount = 3;
byte[] buffer = new byte[2];
Av1BitStreamWriter writer = new(buffer);
writer.WriteSignedReferenceSubexponential(-4, ValueMagnitude, GroupBitCount, -4);
writer.WriteSignedReferenceSubexponential(4, ValueMagnitude, GroupBitCount, -4);
writer.WriteSignedReferenceSubexponential(0, ValueMagnitude, GroupBitCount, 0);
writer.Flush();
Assert.Equal([0x1e, 0x00], buffer);
Av1BitStreamReader reader = new(buffer);
Assert.Equal(-4, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, -4));
Assert.Equal(4, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, -4));
Assert.Equal(0, reader.ReadSignedReferenceSubexponential(ValueMagnitude, GroupBitCount, 0));
}
[Theory] [Theory]
[InlineData(3)] [InlineData(3)]
[InlineData(4)] [InlineData(4)]

174
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

@ -5,8 +5,10 @@ using System.Buffers;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
@ -133,14 +135,16 @@ public class Av1CoefficientsEntropyTests
TilesInfo = new ObuTileGroupHeader() TilesInfo = new ObuTileGroupHeader()
}, },
FrameHeader = new ObuFrameHeader(), FrameHeader = new ObuFrameHeader(),
PreviousQIndex = [] PreviousQIndex = Memory<int>.Empty
}, },
SegmentationNeighborMap = Memory<byte>.Empty, SegmentationNeighborMap = Memory<byte>.Empty,
ModeInfoGrid = grid, ModeInfoGrid = grid,
ModeInfoAllocation = allocation, ModeInfoAllocation = allocation,
ModeInfoStride = 4, ModeInfoStride = 4,
Disallow4x4AllFrames = disallow4x4, Disallow4x4AllFrames = disallow4x4,
CdefPreset = [] CdefPreset = Memory<int>.Empty,
TileDataOffsets = Memory<int>.Empty,
TileDataLengths = Memory<int>.Empty
}; };
Point position = new(column, row); Point position = new(column, row);
@ -218,6 +222,137 @@ public class Av1CoefficientsEntropyTests
Assert.Equal(8, Unsafe.SizeOf<Av1MacroBlockModeInfo>()); Assert.Equal(8, Unsafe.SizeOf<Av1MacroBlockModeInfo>());
} }
/// <summary>
/// Verifies that segment, reference, filter, and flag updates preserve adjacent packed values.
/// </summary>
[Fact]
public void EncoderBlockModeInfoPackedFieldsRemainIndependent()
{
for (int segment = 0; segment < 8; segment++)
{
for (int reference = 0; reference < 8; reference++)
{
for (int vertical = 0; vertical < 4; vertical++)
{
for (int horizontal = 0; horizontal < 4; horizontal++)
{
Av1EncoderBlockModeInfo modeInfo = new()
{
Skip = true,
SkipMode = true,
UseIntraBlockCopy = true,
SegmentId = segment,
ReferenceFrame = (Av1ReferenceFrameType)reference,
VerticalInterpolationFilter = (Av1InterpolationFilter)vertical,
HorizontalInterpolationFilter = (Av1InterpolationFilter)horizontal
};
Assert.Equal(segment, modeInfo.SegmentId);
Assert.Equal((Av1ReferenceFrameType)reference, modeInfo.ReferenceFrame);
Assert.Equal((Av1InterpolationFilter)vertical, modeInfo.VerticalInterpolationFilter);
Assert.Equal((Av1InterpolationFilter)horizontal, modeInfo.HorizontalInterpolationFilter);
Assert.True(modeInfo.Skip);
Assert.True(modeInfo.SkipMode);
Assert.True(modeInfo.UseIntraBlockCopy);
// Overwrite every bit in each shared region after the adjacent value has been populated.
modeInfo.SegmentId = segment ^ 7;
Assert.Equal((Av1ReferenceFrameType)reference, modeInfo.ReferenceFrame);
modeInfo.ReferenceFrame = (Av1ReferenceFrameType)(reference ^ 7);
Assert.Equal(segment ^ 7, modeInfo.SegmentId);
modeInfo.Skip = false;
modeInfo.SkipMode = false;
modeInfo.UseIntraBlockCopy = false;
Assert.Equal((Av1InterpolationFilter)vertical, modeInfo.VerticalInterpolationFilter);
Assert.Equal((Av1InterpolationFilter)horizontal, modeInfo.HorizontalInterpolationFilter);
modeInfo.VerticalInterpolationFilter = (Av1InterpolationFilter)(vertical ^ 3);
modeInfo.HorizontalInterpolationFilter = (Av1InterpolationFilter)(horizontal ^ 3);
Assert.False(modeInfo.Skip);
Assert.False(modeInfo.SkipMode);
Assert.False(modeInfo.UseIntraBlockCopy);
}
}
}
}
}
/// <summary>
/// Verifies both filter directions for matching references, mismatches, and unavailable tile neighbors.
/// </summary>
[Theory]
[InlineData(false, false)]
[InlineData(false, true)]
[InlineData(true, false)]
[InlineData(true, true)]
public void EncoderInterpolationContextUsesTileNeighbors(bool aboveAvailable, bool leftAvailable)
{
// Rows are the above filter and columns the left filter; index three means a nonmatching reference.
ReadOnlySpan<int> expectedContexts = [0, 3, 3, 0, 3, 1, 3, 1, 3, 3, 2, 2, 0, 1, 2, 3];
Av1MacroBlockModeInfo[] allocation = new Av1MacroBlockModeInfo[3];
int[] grid = new int[9];
grid[1] = 0;
grid[3] = 1;
grid[4] = 2;
Av1MacroBlockD macroBlock = CreateMacroBlock();
macroBlock.ModeInfoStride = 3;
macroBlock.IsUpAvailable = aboveAvailable;
macroBlock.IsLeftAvailable = leftAvailable;
macroBlock.SetModeInfoGrid(grid, allocation, 4);
Av1EncoderBlockModeInfo current = new() { ReferenceFrame = Av1ReferenceFrameType.Last };
for (int above = 0; above < 4; above++)
{
for (int left = 0; left < 4; left++)
{
allocation[0].Block.ReferenceFrame = above == 3 ? Av1ReferenceFrameType.Golden : Av1ReferenceFrameType.Last;
allocation[0].Block.VerticalInterpolationFilter = (Av1InterpolationFilter)(above % 3);
allocation[0].Block.HorizontalInterpolationFilter = (Av1InterpolationFilter)((above + 1) % 3);
allocation[1].Block.ReferenceFrame = left == 3 ? Av1ReferenceFrameType.Golden : Av1ReferenceFrameType.Last;
allocation[1].Block.VerticalInterpolationFilter = (Av1InterpolationFilter)(left % 3);
allocation[1].Block.HorizontalInterpolationFilter = (Av1InterpolationFilter)((left + 1) % 3);
int aboveVertical = aboveAvailable ? above : 3;
int leftVertical = leftAvailable ? left : 3;
int aboveHorizontal = aboveVertical == 3 ? 3 : (above + 1) % 3;
int leftHorizontal = leftVertical == 3 ? 3 : (left + 1) % 3;
Assert.Equal(expectedContexts[(aboveVertical * 4) + leftVertical], Av1SymbolContextHelper.GetSwitchableInterpolationContext(current, macroBlock, 0));
Assert.Equal(8 + expectedContexts[(aboveHorizontal * 4) + leftHorizontal], Av1SymbolContextHelper.GetSwitchableInterpolationContext(current, macroBlock, 1));
}
}
}
/// <summary>
/// Verifies which inter modes signal filters and distinguishes residual skip from compound skip mode.
/// </summary>
[Theory]
[InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.GlobalMotionVector, false)]
[InlineData(Av1GlobalMotionType.Translation, Av1PredictionMode.GlobalMotionVector, true)]
[InlineData(Av1GlobalMotionType.RotationZoom, Av1PredictionMode.GlobalMotionVector, false)]
[InlineData(Av1GlobalMotionType.Affine, Av1PredictionMode.GlobalMotionVector, false)]
[InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.NewMotionVector, true)]
[InlineData(Av1GlobalMotionType.Identity, Av1PredictionMode.NearestMotionVector, true)]
public void EncoderInterpolationSyntaxMatchesModeEligibility(int globalType, int predictionMode, bool expected)
{
ObuFrameHeader frameHeader = new() { InterpolationFilter = Av1InterpolationFilter.Switchable };
frameHeader.GetGlobalMotionParameters()[0].Type = (Av1GlobalMotionType)globalType;
Av1EncoderBlockModeInfo modeInfo = new()
{
BlockSize = Av1BlockSize.Block8x8,
ReferenceFrame = Av1ReferenceFrameType.Last,
Mode = (Av1PredictionMode)predictionMode
};
Assert.Equal(expected, Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo));
modeInfo.Skip = true;
Assert.Equal(expected, Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo));
modeInfo.BlockSize = Av1BlockSize.Block4x8;
Assert.True(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo));
modeInfo.SkipMode = true;
Assert.False(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo));
modeInfo.SkipMode = false;
frameHeader.InterpolationFilter = Av1InterpolationFilter.Regular;
Assert.False(Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo));
}
[Fact] [Fact]
public void EncoderSuperblockWorkspaceUsesOneExactSizeOwner() public void EncoderSuperblockWorkspaceUsesOneExactSizeOwner()
{ {
@ -231,9 +366,9 @@ public class Av1CoefficientsEntropyTests
{ {
allocation = Assert.Single(allocator.AllocationLog); allocation = Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
Assert.Equal(typeof(Av1EncoderBlockStruct), allocation.ElementType); Assert.Equal(typeof(byte), allocation.ElementType);
Assert.Equal(AllocationOptions.None, allocation.AllocationOptions); Assert.Equal(AllocationOptions.None, allocation.AllocationOptions);
Assert.Equal(Av1EncoderSuperblockWorkspace.StorageLength, allocation.Length); Assert.Equal(Av1EncoderSuperblockWorkspace.StorageByteLength, allocation.Length);
Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumFinalBlockCount, workspace.FinalBlocks.Length); Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumFinalBlockCount, workspace.FinalBlocks.Length);
Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumPartitionCount, workspace.PartitionTypes.Length); Assert.Equal(Av1EncoderSuperblockWorkspace.MaximumPartitionCount, workspace.PartitionTypes.Length);
Assert.Equal(Av1EncoderBlockStruct.StorageSize, Unsafe.SizeOf<Av1EncoderBlockStruct>()); Assert.Equal(Av1EncoderBlockStruct.StorageSize, Unsafe.SizeOf<Av1EncoderBlockStruct>());
@ -295,39 +430,34 @@ public class Av1CoefficientsEntropyTests
} }
[Fact] [Fact]
public void EncoderPaletteMapsUseOneLazyExactSizeOwner() public void EncoderSuperblockWorkspaceExposesReusablePaletteMaps()
{ {
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone(); Configuration configuration = Configuration.Default.Clone();
configuration.MemoryAllocator = allocator; configuration.MemoryAllocator = allocator;
TestMemoryAllocator.AllocationRequest[] allocations; TestMemoryAllocator.AllocationRequest allocation;
using (Av1EncoderSuperblockWorkspace workspace = new(configuration)) using (Av1EncoderSuperblockWorkspace workspace = new(configuration))
{ {
Assert.Single(allocator.AllocationLog); allocation = Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
Av1EncoderPaletteMapBuffer maps = workspace.GetPaletteMaps(); Av1EncoderPaletteMapBuffer maps = workspace.GetPaletteMaps();
Assert.Same(maps, workspace.GetPaletteMaps()); Assert.Same(maps, workspace.GetPaletteMaps());
allocations = allocator.AllocationLog.ToArray(); Assert.Single(allocator.AllocationLog);
Assert.Equal(2, allocations.Length);
Assert.Equal(typeof(byte), allocations[1].ElementType);
Assert.Equal(Av1EncoderPaletteMapBuffer.StorageLength, allocations[1].Length);
Assert.Equal(AllocationOptions.None, allocations[1].AllocationOptions);
Buffer2DRegion<byte> luma = maps.GetMap(Av1PlaneType.Y, 64, 64); Buffer2DRegion<byte> luma = maps.GetMap(Av1PlaneType.Y, 64, 64);
Buffer2DRegion<byte> chroma = maps.GetMap(Av1PlaneType.Uv, 32, 32); Buffer2DRegion<byte> chroma = maps.GetMap(Av1PlaneType.Uv, 32, 32);
luma.DangerousGetRowSpan(0)[0] = 3; luma.DangerousGetRowSpan(0)[0] = 3;
chroma.DangerousGetRowSpan(0)[0] = 5; chroma.DangerousGetRowSpan(0)[0] = 5;
Assert.Equal(3, luma.DangerousGetRowSpan(0)[0]); Av1EncoderPaletteMapBuffer retainedMaps = workspace.GetPaletteMaps();
Assert.Equal(5, chroma.DangerousGetRowSpan(0)[0]); Assert.Equal(3, retainedMaps.GetMap(Av1PlaneType.Y, 64, 64).DangerousGetRowSpan(0)[0]);
Assert.Equal(5, retainedMaps.GetMap(Av1PlaneType.Uv, 32, 32).DangerousGetRowSpan(0)[0]);
} }
Assert.Equal(2, allocator.ReturnLog.Count); TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog);
Assert.Equal( Assert.Equal(allocation.AllocationId, returned.AllocationId);
allocations.Select(x => x.AllocationId).Order(),
allocator.ReturnLog.Select(x => x.AllocationId).Order());
} }
[Fact] [Fact]
@ -841,7 +971,7 @@ public class Av1CoefficientsEntropyTests
skip: false, skip: false,
modeInfoPosition: new Point(20, 4)); modeInfoPosition: new Point(20, 4));
Assert.Equal(new[] { -1, 3, -1, -1 }, picture.CdefPreset[0]); Assert.True(picture.CdefPreset.Span.SequenceEqual([-1, 3, -1, -1]));
} }
[Fact] [Fact]
@ -1431,13 +1561,15 @@ public class Av1CoefficientsEntropyTests
TilesInfo = tiles TilesInfo = tiles
}, },
FrameHeader = frameHeader, FrameHeader = frameHeader,
PreviousQIndex = [] PreviousQIndex = Memory<int>.Empty
}, },
SegmentationNeighborMap = Memory<byte>.Empty, SegmentationNeighborMap = Memory<byte>.Empty,
ModeInfoGrid = modeInfoGrid, ModeInfoGrid = modeInfoGrid,
ModeInfoAllocation = modeInfoAllocation, ModeInfoAllocation = modeInfoAllocation,
ModeInfoStride = modeInfoColumnCount, ModeInfoStride = modeInfoColumnCount,
CdefPreset = [[-1, -1, -1, -1]] CdefPreset = new int[] { -1, -1, -1, -1 },
TileDataOffsets = Memory<int>.Empty,
TileDataLengths = Memory<int>.Empty
}; };
} }

282
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

@ -3,9 +3,11 @@
using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Formats.Heif.Components; using SixLabors.ImageSharp.Formats.Heif.Components;
@ -26,6 +28,42 @@ public class Av1EncoderFrameTests
private const int Yuv422 = (int)Av1ColorFormat.Yuv422; private const int Yuv422 = (int)Av1ColorFormat.Yuv422;
private const int Yuv444 = (int)Av1ColorFormat.Yuv444; private const int Yuv444 = (int)Av1ColorFormat.Yuv444;
[Fact]
public void EncodeUsesMultipleTilesWhenSingleTileWidthLimitIsExceeded()
{
const int Width = Av1Constants.MaxTileWidth + 1;
const int SuperblockSize = 1 << (Av1Constants.MaxSuperBlockSizeLog2 - 1);
const int Height = SuperblockSize;
int superblockColumns = (Width + SuperblockSize - 1) / SuperblockSize;
int secondTileStart = ((superblockColumns + 1) / 2) * SuperblockSize;
using Image<L8> source = new(Width, Height, new L8(128));
source[0, 0] = new L8(1);
source[secondTileStart - 1, 0] = new L8(17);
source[secondTileStart, 0] = new L8(241);
source[Width - 1, Height - 1] = new L8(255);
using MemoryStream stream = new();
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv400);
Av1FrameEncoder.Encode(
Configuration.Default,
source.Frames.RootFrame,
stream,
colorConfig,
qIndex: 0,
effort: 0);
using Av1Decoder decoder = new(Configuration.Default);
using Image<L8> decoded = decoder.Decode<L8>(stream.ToArray());
Assert.Equal(2, decoder.FrameHeader.TilesInfo.TileColumnCount);
Assert.Equal(1, decoder.FrameHeader.TilesInfo.TileRowCount);
Assert.Equal(source.Size, decoded.Size);
Assert.Equal(source[0, 0], decoded[0, 0]);
Assert.Equal(source[secondTileStart - 1, 0], decoded[secondTileStart - 1, 0]);
Assert.Equal(source[secondTileStart, 0], decoded[secondTileStart, 0]);
Assert.Equal(source[Width - 1, Height - 1], decoded[Width - 1, Height - 1]);
}
[Theory] [Theory]
[InlineData(8, 8, false, EightBit, Yuv400)] [InlineData(8, 8, false, EightBit, Yuv400)]
[InlineData(8, 8, true, EightBit, Yuv400)] [InlineData(8, 8, true, EightBit, Yuv400)]
@ -153,6 +191,250 @@ public class Av1EncoderFrameTests
} }
} }
[Theory]
[InlineData(false)]
[InlineData(true)]
public void EncodeSequenceFrameWritesNonReducedHeaderConsumedByProductionDecoder(bool encodeAlpha)
{
const int Width = 16;
const int Height = 16;
using Image<Rgba32> source = new(Width, Height, new Rgba32(48, 96, 192));
using MemoryStream stream = new();
ObuColorConfig colorConfig = encodeAlpha
? CreateColorConfig(Av1BitDepth.EightBit)
: CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420);
using Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha
? Av1FrameEncoder.CreateAlphaSequenceEncoder(
Configuration.Default,
Width,
Height,
colorConfig,
qIndex: 37,
effort: 5)
: Av1FrameEncoder.CreateColorSequenceEncoder(
Configuration.Default,
Width,
Height,
colorConfig,
qIndex: 37,
effort: 5);
encoder.EncodeKeyFrame(source.Frames.RootFrame, stream);
ObuSequenceHeader encodedHeader = encoder.SequenceHeader;
byte[] payload = stream.ToArray();
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
ObuSequenceHeader decodedHeader = decoder.SequenceHeader;
Assert.False(encodedHeader.IsStillPicture);
Assert.False(encodedHeader.IsReducedStillPictureHeader);
Assert.Equal(encodeAlpha, encodedHeader.ColorConfig.IsMonochrome);
Assert.NotNull(decodedHeader);
Assert.False(decodedHeader.IsStillPicture);
Assert.False(decodedHeader.IsReducedStillPictureHeader);
Assert.Equal(new Size(Width, Height), decoded.Size);
}
/// <summary>
/// Verifies retained reference reconstruction and effort-dependent filter signaling through production sequence decoding.
/// </summary>
[Theory]
[InlineData(5, false, false)]
[InlineData(7, false, false)]
[InlineData(8, true, false)]
[InlineData(9, true, true)]
public void SequenceEncoderUsesRetainedReconstructionForInterFrame(int effort, bool switchableFilters, bool dualFilters)
{
const int Width = 16;
const int Height = 16;
Rgba32 sourceColor = new(48, 96, 192);
using Image<Rgba32> source = new(Width, Height, sourceColor);
using MemoryStream firstSample = new();
using MemoryStream secondSample = new();
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420);
using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder(
Configuration.Default,
Width,
Height,
colorConfig,
qIndex: 37,
effort);
encoder.EncodeKeyFrame(source.Frames.RootFrame, firstSample);
encoder.EncodeInterFrame(source.Frames.RootFrame, secondSample);
// Retain the exact two-sample elementary stream for independent reference-decoder acceptance.
string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.SequenceEncoderUsesRetainedReconstructionForInterFrame));
using (FileStream output = File.Create(Path.Combine(outputDirectory, $"effort-{effort}.obu")))
{
firstSample.Position = 0;
firstSample.CopyTo(output);
secondSample.Position = 0;
secondSample.CopyTo(output);
}
using Av1Decoder decoder = new(Configuration.Default);
using ImageFrame<Rgba32> decodedFirst = decoder.DecodeSequenceFrame<Rgba32>(
firstSample.ToArray(),
null,
null);
using ImageFrame<Rgba32> decodedSecond = decoder.DecodeSequenceFrame<Rgba32>(
secondSample.ToArray(),
null,
null);
ObuFrameHeader frameHeader = decoder.FrameHeader;
Assert.Equal(ObuFrameType.InterFrame, frameHeader.FrameType);
Assert.False(frameHeader.SegmentationParameters.Enabled);
Assert.False(frameHeader.AllowScreenContentTools);
Assert.False(frameHeader.ForceIntegerMotionVector);
Assert.Equal(effort >= 8, frameHeader.AllowHighPrecisionMotionVector);
Assert.Equal(37, frameHeader.QuantizationParameters.BaseQIndex);
Assert.Equal(switchableFilters ? Av1InterpolationFilter.Switchable : Av1InterpolationFilter.Regular, frameHeader.InterpolationFilter);
Assert.Equal(dualFilters, decoder.SequenceHeader.EnableDualFilter);
for (int y = 0; y < Height; y++)
{
Assert.Equal(
decodedFirst.PixelBuffer.DangerousGetRowSpan(y),
decodedSecond.PixelBuffer.DangerousGetRowSpan(y));
}
}
[Fact]
public void SequenceEncoderWritesSelectedGlobalTranslation()
{
const int Width = 64;
const int Height = 64;
const int HorizontalOffset = 4;
using Image<Rgba32> first = new(Width, Height);
using Image<Rgba32> second = new(Width, Height);
for (int y = 0; y < Height; y++)
{
Span<Rgba32> firstRow = first.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
for (int x = 0; x < Width; x++)
{
byte value = (byte)(((x * 37) + (y * 53) + ((x * y) * 11)) & byte.MaxValue);
firstRow[x] = new Rgba32(value, value, value);
}
}
for (int y = 0; y < Height; y++)
{
ReadOnlySpan<Rgba32> firstRow = first.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
Span<Rgba32> secondRow = second.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
for (int x = 0; x < Width; x++)
{
secondRow[x] = firstRow[Math.Min(x + HorizontalOffset, Width - 1)];
}
}
using MemoryStream firstSample = new();
using MemoryStream secondSample = new();
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv420);
using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder(
Configuration.Default,
Width,
Height,
colorConfig,
qIndex: 4,
effort: 6);
encoder.EncodeKeyFrame(first.Frames.RootFrame, firstSample);
encoder.EncodeInterFrame(second.Frames.RootFrame, secondSample);
using Av1Decoder decoder = new(Configuration.Default);
using ImageFrame<Rgba32> decodedFirst = decoder.DecodeSequenceFrame<Rgba32>(
firstSample.ToArray(),
null,
null);
using ImageFrame<Rgba32> decodedSecond = decoder.DecodeSequenceFrame<Rgba32>(
secondSample.ToArray(),
null,
null);
ObuFrameHeader frameHeader = decoder.FrameHeader;
Av1GlobalMotionParameters globalMotion = frameHeader.GetGlobalMotionParameters()[0];
Av1MotionVector vector = globalMotion.GetMotionVector(
frameHeader.AllowHighPrecisionMotionVector,
Av1BlockSize.Block8x8,
default,
frameHeader.ForceIntegerMotionVector);
Assert.Equal(Av1GlobalMotionType.RotationZoom, globalMotion.Type);
Assert.False(frameHeader.AllowScreenContentTools);
Assert.False(frameHeader.ForceIntegerMotionVector);
Assert.Equal(0, vector.Row);
Assert.Equal(HorizontalOffset * 8, vector.Column);
Assert.Equal(first.Size, decodedFirst.Size);
Assert.Equal(second.Size, decodedSecond.Size);
}
[Theory]
[InlineData(false, EightBit, Yuv420, 384)]
[InlineData(false, TwelveBit, Yuv444, 288)]
[InlineData(true, EightBit, Yuv400, 192)]
[InlineData(true, TwelveBit, Yuv400, 192)]
public void SequenceEncoderReusesAllocatorOwnedRowStorage(
bool encodeAlpha,
int bitDepthValue,
int colorFormatValue,
int expectedRowStorageLength)
{
const int Width = 64;
const int Height = 64;
Av1BitDepth bitDepth = (Av1BitDepth)bitDepthValue;
Av1ColorFormat colorFormat = (Av1ColorFormat)colorFormatValue;
using Image<Rgba64> source = new(
Width,
Height,
new Rgba64(ushort.MaxValue, 32768, 16384, 49152));
TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone();
configuration.MemoryAllocator = allocator;
ObuColorConfig colorConfig = CreateColorConfig(bitDepth, colorFormat);
TestMemoryAllocator.AllocationRequest rowStorage;
int allocationCount;
using (Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha
? Av1FrameEncoder.CreateAlphaSequenceEncoder(
configuration,
Width,
Height,
colorConfig,
qIndex: 37,
effort: 6)
: Av1FrameEncoder.CreateColorSequenceEncoder(
configuration,
Width,
Height,
colorConfig,
qIndex: 37,
effort: 6))
{
rowStorage = Assert.Single(
allocator.AllocationLog,
allocation => allocation.ElementType == typeof(float));
allocationCount = allocator.AllocationLog.Count;
using MemoryStream output = new(256 * 1024);
encoder.EncodeKeyFrame(source.Frames.RootFrame, output);
encoder.EncodeInterFrame(source.Frames.RootFrame, output);
// Fixed sequence geometry lets libaom retain its frame-sized compressor data. The ImageSharp
// sequence encoder must likewise perform every sample conversion and coding pass without another rent.
Assert.Equal(allocationCount, allocator.AllocationLog.Count);
}
Assert.Equal(expectedRowStorageLength, rowStorage.Length);
Assert.Contains(
allocator.ReturnLog,
returned => returned.AllocationId == rowStorage.AllocationId);
}
[Theory] [Theory]
[InlineData(TenBit)] [InlineData(TenBit)]
[InlineData(TwelveBit)] [InlineData(TwelveBit)]

187
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs

@ -54,10 +54,13 @@ public class Av1EncoderModeInfoBufferTests
public unsafe void PictureBufferPacksAllPictureStateIntoTwoAllocatorOwners( public unsafe void PictureBufferPacksAllPictureStateIntoTwoAllocatorOwners(
bool allowScreenContentTools, bool allowScreenContentTools,
bool allowIntraBlockCopy, bool allowIntraBlockCopy,
int expectedStateStorageLength) int expectedContextStorageLength)
{ {
const int Width = 16; const int Width = 16;
const int Height = 16; const int Height = 16;
// Four CDEF presets, the preceding quantizer, and two payload bounds follow the context regions.
const int TileStateStorageLength = 7 * sizeof(int);
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone(); Configuration configuration = Configuration.Default.Clone();
@ -108,7 +111,7 @@ public class Av1EncoderModeInfoBufferTests
Assert.Equal(6_144, allocations[0].Length); Assert.Equal(6_144, allocations[0].Length);
Assert.Equal(AllocationOptions.Clean, allocations[0].AllocationOptions); Assert.Equal(AllocationOptions.Clean, allocations[0].AllocationOptions);
Assert.Equal(typeof(byte), allocations[1].ElementType); Assert.Equal(typeof(byte), allocations[1].ElementType);
Assert.Equal(expectedStateStorageLength, allocations[1].Length); Assert.Equal(expectedContextStorageLength + TileStateStorageLength, allocations[1].Length);
Assert.Equal(AllocationOptions.Clean, allocations[1].AllocationOptions); Assert.Equal(AllocationOptions.Clean, allocations[1].AllocationOptions);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
@ -122,6 +125,29 @@ public class Av1EncoderModeInfoBufferTests
Assert.Equal(16, picture.CbDcSignLevelCoefficientNeighbors[0].Top.Length); Assert.Equal(16, picture.CbDcSignLevelCoefficientNeighbors[0].Top.Length);
Assert.Equal(32, picture.TransformFunctionContexts[0].Left.Length); Assert.Equal(32, picture.TransformFunctionContexts[0].Left.Length);
Assert.Equal(32, picture.TransformFunctionContexts[0].Top.Length); Assert.Equal(32, picture.TransformFunctionContexts[0].Top.Length);
Assert.Equal(4, picture.CdefPreset.Length);
Assert.Equal(1, picture.Parent.PreviousQIndex.Length);
Assert.Equal(1, picture.TileDataOffsets.Length);
Assert.Equal(1, picture.TileDataLengths.Length);
// Exact offsets prove that all four typed views occupy the trailing region of the same owner,
// without gaps, overlapping fields, or a separate allocation hidden behind a memory manager.
fixed (byte* state = picture.SegmentationNeighborMap.Span)
{
fixed (int* cdef = picture.CdefPreset.Span,
quantizer = picture.Parent.PreviousQIndex.Span,
offsets = picture.TileDataOffsets.Span,
lengths = picture.TileDataLengths.Span)
{
Assert.Equal((nuint)0, (nuint)cdef % (nuint)sizeof(int));
Assert.Equal(expectedContextStorageLength, (byte*)cdef - state);
Assert.Equal(4, quantizer - cdef);
Assert.Equal(1, offsets - quantizer);
Assert.Equal(1, lengths - offsets);
Assert.Equal(allocations[1].Length, (byte*)(lengths + 1) - state);
}
}
if (allowScreenContentTools) if (allowScreenContentTools)
{ {
Av1NeighborArrayUnit<Av1EncoderPaletteInfo> paletteContext = Assert.Single(picture.PaletteContexts); Av1NeighborArrayUnit<Av1EncoderPaletteInfo> paletteContext = Assert.Single(picture.PaletteContexts);
@ -176,6 +202,157 @@ public class Av1EncoderModeInfoBufferTests
allocator.ReturnLog.Select(x => x.AllocationId).Order()); allocator.ReturnLog.Select(x => x.AllocationId).Order());
} }
[Fact]
public void InterPictureBufferExposesPackedMotionVectorStorage()
{
const int Width = 16;
const int Height = 16;
ObuColorConfig colorConfig = new()
{
IsMonochrome = false,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
ObuTileGroupHeader tiles = new()
{
TileColumnCount = 1,
TileRowCount = 1
};
tiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2;
tiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2;
ObuSequenceHeader sequenceHeader = new() { ColorConfig = colorConfig };
ObuFrameHeader frameHeader = new()
{
FrameType = ObuFrameType.InterFrame,
ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2,
ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2,
TilesInfo = tiles
};
using Av1EncoderPictureBuffer buffer = new(
Configuration.Default,
sequenceHeader,
frameHeader,
Width,
Height,
disallow4x4AllFrames: true);
Av1PictureControlSet picture = buffer.Picture;
Assert.Equal(256, picture.DisplacementVectors.Length);
Assert.Equal(0, picture.IntraBlockCopySearch.OriginWidth);
Point position = new(2, 2);
Av1MotionVector vector = new(-32, 40);
picture.MapModeInfoBlock(position, Av1BlockSize.Block8x8);
picture.SetDisplacementVector(position, vector);
Assert.Equal(vector, picture.GetDisplacementVector(new Point(3, 3)));
}
[Fact]
public void PictureBufferResetReusesStorageAndRestoresFrameState()
{
const int Width = 16;
const int Height = 16;
const int InitialQIndex = 37;
const int NextQIndex = 91;
TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone();
configuration.MemoryAllocator = allocator;
ObuColorConfig colorConfig = new()
{
IsMonochrome = false,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
ObuTileGroupHeader initialTiles = new()
{
TileColumnCount = 1,
TileRowCount = 1
};
initialTiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2;
initialTiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2;
ObuSequenceHeader sequenceHeader = new()
{
Use128x128Superblock = true,
ColorConfig = colorConfig
};
ObuFrameHeader initialFrameHeader = new()
{
FrameType = ObuFrameType.KeyFrame,
ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2,
ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2,
TilesInfo = initialTiles
};
initialFrameHeader.QuantizationParameters.BaseQIndex = InitialQIndex;
using Av1EncoderPictureBuffer buffer = new(
configuration,
sequenceHeader,
initialFrameHeader,
Width,
Height,
disallow4x4AllFrames: true,
allocateScreenContentState: true,
allocateMotionVectorState: true,
allocateIntraBlockCopySearch: true);
Av1PictureControlSet picture = buffer.Picture;
int allocationCount = allocator.AllocationLog.Count;
picture.ModeInfoGrid.Span[0] = 7;
picture.ModeInfoAllocation.Span[0].Block.Mode = Av1PredictionMode.Paeth;
picture.SegmentationNeighborMap.Span[0] = 3;
picture.PartitionContexts[0].Left[0] = new Av1PartitionContext(5, 7);
picture.TransformFunctionContexts[0].Top[0] = 8;
picture.PaletteContexts[0].Left[0].PaletteSizes[0] = 2;
picture.DisplacementVectors.Span[0] = new Av1EncoderDisplacementVector { Row = -8, Column = 16 };
picture.CdefPreset.Span[0] = 2;
picture.Parent.PreviousQIndex.Span[0] = InitialQIndex + 1;
picture.TileDataOffsets.Span[0] = 11;
picture.TileDataLengths.Span[0] = 13;
ObuTileGroupHeader nextTiles = new()
{
TileColumnCount = 1,
TileRowCount = 1
};
nextTiles.TileColumnStartModeInfo[1] = Width >> Av1Constants.ModeInfoSizeLog2;
nextTiles.TileRowStartModeInfo[1] = Height >> Av1Constants.ModeInfoSizeLog2;
ObuFrameHeader nextFrameHeader = new()
{
FrameType = ObuFrameType.InterFrame,
ModeInfoColumnCount = Width >> Av1Constants.ModeInfoSizeLog2,
ModeInfoRowCount = Height >> Av1Constants.ModeInfoSizeLog2,
TilesInfo = nextTiles
};
nextFrameHeader.QuantizationParameters.BaseQIndex = NextQIndex;
buffer.Reset(nextFrameHeader);
Assert.Equal(allocationCount, allocator.AllocationLog.Count);
Assert.Empty(allocator.ReturnLog);
Assert.Equal(0, picture.ModeInfoGrid.Span[0]);
Assert.Equal(Av1PredictionMode.DC, picture.ModeInfoAllocation.Span[0].Block.Mode);
Assert.Equal(0, picture.SegmentationNeighborMap.Span[0]);
Assert.Equal(default, picture.PartitionContexts[0].Left[0]);
Assert.Equal(Av1Constants.MaxTransformSize, picture.TransformFunctionContexts[0].Top[0]);
Assert.Equal(0, picture.PaletteContexts[0].Left[0].PaletteSizes[0]);
Assert.Equal(default, picture.DisplacementVectors.Span[0]);
Assert.Equal(-1, picture.CdefPreset.Span[0]);
Assert.Equal(NextQIndex, picture.Parent.PreviousQIndex.Span[0]);
Assert.Equal(0, picture.TileDataOffsets.Span[0]);
Assert.Equal(0, picture.TileDataLengths.Span[0]);
Assert.Same(nextFrameHeader, picture.Parent.FrameHeader);
Assert.Same(nextTiles, picture.Parent.Common.TilesInfo);
}
[Theory] [Theory]
[InlineData(false, 2, 3, 98)] [InlineData(false, 2, 3, 98)]
[InlineData(true, 2, 2, 17)] [InlineData(true, 2, 2, 17)]
@ -248,14 +425,16 @@ public class Av1EncoderModeInfoBufferTests
TilesInfo = tiles TilesInfo = tiles
}, },
FrameHeader = frameHeader, FrameHeader = frameHeader,
PreviousQIndex = [] PreviousQIndex = Memory<int>.Empty
}, },
SegmentationNeighborMap = Memory<byte>.Empty, SegmentationNeighborMap = Memory<byte>.Empty,
ModeInfoGrid = buffer.Grid, ModeInfoGrid = buffer.Grid,
ModeInfoAllocation = buffer.Allocation, ModeInfoAllocation = buffer.Allocation,
ModeInfoStride = buffer.ModeInfoStride, ModeInfoStride = buffer.ModeInfoStride,
Disallow4x4AllFrames = buffer.Disallow4x4AllFrames, Disallow4x4AllFrames = buffer.Disallow4x4AllFrames,
CdefPreset = [] CdefPreset = Memory<int>.Empty,
TileDataOffsets = Memory<int>.Empty,
TileDataLengths = Memory<int>.Empty
}; };
} }
} }

218
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs

@ -155,6 +155,27 @@ public class Av1EntropyTests
encoder.GetTransformBlockSkipCost(true, TransformSize, SkipContext)); encoder.GetTransformBlockSkipCost(true, TransformSize, SkipContext));
} }
[Fact]
public void SymbolEncoderResetRestoresNormativeDistributionState()
{
const byte TopContext = 0;
const byte LeftContext = 0;
const Av1PredictionMode LumaMode = Av1PredictionMode.DC;
using Av1SymbolEncoder encoder = new(Configuration.Default, 64, BaseQIndex, updateCdf: true);
int initialCost = encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext);
for (int i = 0; i < 8; i++)
{
encoder.WriteLumaMode(LumaMode, TopContext, LeftContext);
}
Assert.NotEqual(initialCost, encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext));
encoder.Reset();
Assert.Equal(initialCost, encoder.GetLumaModeCost(LumaMode, TopContext, LeftContext));
}
[Fact] [Fact]
public void BlockSkipDecisionUsesAdaptedRatesForEmptyTransforms() public void BlockSkipDecisionUsesAdaptedRatesForEmptyTransforms()
{ {
@ -246,7 +267,8 @@ public class Av1EntropyTests
macroBlock, macroBlock,
Av1BlockSize.Block8x8, Av1BlockSize.Block8x8,
Mode, Mode,
angleDelta)); angleDelta,
isIntraFrame: true));
} }
/// <summary> /// <summary>
@ -698,6 +720,48 @@ public class Av1EntropyTests
long expected) long expected)
=> Assert.Equal(expected, Av1RateDistortion.GetCost(rateMultiplier, rate, distortion)); => Assert.Equal(expected, Av1RateDistortion.GetCost(rateMultiplier, rate, distortion));
/// <summary>
/// Verifies fixed curve samples, interpolation, error categories, native quantizer normalization, and skip selection.
/// </summary>
[Theory]
[InlineData(Av1BlockSize.Block4x4, 16L, 16, 8, Av1BitDepth.EightBit, 1, 13243, 17L)]
[InlineData(Av1BlockSize.Block8x8, 64L, 64, 8, Av1BitDepth.EightBit, 1, 45715, 66L)]
[InlineData(Av1BlockSize.Block16x16, 256L, 256, 8, Av1BitDepth.EightBit, 1, 154928, 265L)]
[InlineData(Av1BlockSize.Block32x32, 1024L, 1024, 8, Av1BitDepth.EightBit, 1, 410224, 1061L)]
[InlineData(Av1BlockSize.Block8x8, 96L, 64, 8, Av1BitDepth.EightBit, 1, 53253, 71L)]
[InlineData(Av1BlockSize.Block8x8, 1024L, 64, 8, Av1BitDepth.EightBit, 1, 96672, 95L)]
[InlineData(Av1BlockSize.Block8x8, 1056L, 64, 8, Av1BitDepth.EightBit, 1, 97253, 95L)]
[InlineData(Av1BlockSize.Block8x8, 64L, 64, 32, Av1BitDepth.TenBit, 1, 45715, 66L)]
[InlineData(Av1BlockSize.Block8x8, 64L, 64, 128, Av1BitDepth.TwelveBit, 1, 45715, 66L)]
[InlineData(Av1BlockSize.Block8x8, 64L, 64, 8, Av1BitDepth.EightBit, 1000000, 0, 1024L)]
[InlineData(Av1BlockSize.Block8x8, 0L, 64, 8, Av1BitDepth.EightBit, 1, 0, 0L)]
[InlineData(Av1BlockSize.Block128x128, 1L, 16384, 21387, Av1BitDepth.TwelveBit, 1, 0, 16L)]
public void PredictionErrorModelMatchesReferenceCurveSamples(
int blockSize,
long squaredError,
int sampleCount,
int acQuantizer,
int bitDepth,
int rateMultiplier,
int expectedRate,
long expectedDistortion)
{
// Expectations come from the published curve samples and cubic polynomial, not from an encode/decode
// round trip. Unit normalized error and unit quantizer hit rate column 31 in each block-size category.
Av1RateDistortion.ModelPredictionError(
(Av1BlockSize)blockSize,
squaredError,
sampleCount,
acQuantizer,
(Av1BitDepth)bitDepth,
rateMultiplier,
out int rate,
out long distortion);
Assert.Equal(expectedRate, rate);
Assert.Equal(expectedDistortion, distortion);
}
[Theory] [Theory]
[InlineData(1, 8191, 100, 100)] [InlineData(1, 8191, 100, 100)]
[InlineData(1, 8192, 100, 101)] [InlineData(1, 8192, 100, 101)]
@ -744,6 +808,19 @@ public class Av1EntropyTests
int expected) int expected)
=> Assert.Equal(expected, Av1RateDistortion.GetKeyFrameRateMultiplier(qIndex, (Av1BitDepth)bitDepth)); => Assert.Equal(expected, Av1RateDistortion.GetKeyFrameRateMultiplier(qIndex, (Av1BitDepth)bitDepth));
[Theory]
[InlineData(0, 0, 51)]
[InlineData(0, 1, 3)]
[InlineData(0, 2, 1)]
[InlineData(255, 0, 9_288_598)]
[InlineData(255, 1, 20_049_918)]
[InlineData(255, 2, 63_036_850)]
public void InterFrameRateMultiplierMatchesCurrentLibaom(
int qIndex,
int bitDepth,
int expected)
=> Assert.Equal(expected, Av1RateDistortion.GetInterFrameRateMultiplier(qIndex, (Av1BitDepth)bitDepth));
[Fact] [Fact]
public void SymbolWriterMatchesCurrentLibaomCarryRegression() public void SymbolWriterMatchesCurrentLibaomCarryRegression()
{ {
@ -782,7 +859,7 @@ public class Av1EntropyTests
} }
[Fact] [Fact]
public void SymbolWriterExposesExistingOutputAllocationWithoutCopy() public void SymbolWriterResetReusesExistingOutputAllocation()
{ {
const int bufferLength = 257; const int bufferLength = 257;
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
@ -805,14 +882,46 @@ public class Av1EntropyTests
Assert.Equal(63, encoded.Span[0]); Assert.Equal(63, encoded.Span[0]);
Assert.Single(allocator.AllocationLog); Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
int firstLength = length;
writer.Reset(firstLength);
writer.WriteBoolean(false, 16_384);
writer.WriteBoolean(false, 16_384);
writer.WriteBoolean(true, 512);
writer.WriteBoolean(false, 8_192);
encoded = writer.Exit(out length);
Assert.Equal(2, length);
Assert.Equal(length, encoded.Length);
Assert.Equal(63, encoded.Span[0]);
ReadOnlyMemory<byte> output = writer.GetOutput(firstLength + length);
Assert.True(output.Span[..firstLength].SequenceEqual(output.Span[firstLength..]));
Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog);
writer.Reset();
writer.WriteBoolean(false, 16_384);
writer.WriteBoolean(false, 16_384);
writer.WriteBoolean(true, 512);
writer.WriteBoolean(false, 8_192);
encoded = writer.Exit(out length);
Assert.Equal(2, length);
Assert.Equal(length, encoded.Length);
Assert.Equal(63, encoded.Span[0]);
Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog);
} }
TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog); TestMemoryAllocator.ReturnRequest returned = Assert.Single(allocator.ReturnLog);
Assert.Equal(allocation.AllocationId, returned.AllocationId); Assert.Equal(allocation.AllocationId, returned.AllocationId);
} }
/// <summary>
/// Verifies bounded coefficient scratch is rented at construction and reused by costing, coding, and frame resets.
/// </summary>
[Fact] [Fact]
public void SymbolEncoderRentsCoefficientScratchOnlyForNonzeroBlocks() public void SymbolEncoderReusesConstructorOwnedCoefficientScratchAcrossFrames()
{ {
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
@ -822,54 +931,10 @@ public class Av1EntropyTests
using (Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex, updateCdf: true)) using (Av1SymbolEncoder encoder = new(configuration, 64, BaseQIndex, updateCdf: true))
{ {
TestMemoryAllocator.AllocationRequest outputScratch = Assert.Single(allocator.AllocationLog);
Assert.Equal(typeof(byte), outputScratch.ElementType);
int emptyContext = encoder.WriteCoefficients(
Av1TransformSize.Size4x4,
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
0,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(0, emptyContext);
Assert.Single(allocator.AllocationLog);
coefficients[0] = 1;
_ = encoder.GetCoefficientCost(
Av1TransformSize.Size4x4,
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
1,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(3, allocator.AllocationLog.Count); Assert.Equal(3, allocator.AllocationLog.Count);
TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[0];
encoder.WriteCoefficients( TestMemoryAllocator.AllocationRequest contextScratch = allocator.AllocationLog[1];
Av1TransformSize.Size4x4, TestMemoryAllocator.AllocationRequest outputScratch = allocator.AllocationLog[2];
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
1,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(3, allocator.AllocationLog.Count);
TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1];
TestMemoryAllocator.AllocationRequest contextScratch = allocator.AllocationLog[2];
int maximumTransformDimension = Av1Constants.MaxTransformSize / 2; int maximumTransformDimension = Av1Constants.MaxTransformSize / 2;
int expectedLevelLength = int expectedLevelLength =
(Av1Constants.TransformPadHorizontal + maximumTransformDimension) * (Av1Constants.TransformPadHorizontal + maximumTransformDimension) *
@ -880,6 +945,59 @@ public class Av1EntropyTests
Assert.Equal(AllocationOptions.Clean, levelScratch.AllocationOptions); Assert.Equal(AllocationOptions.Clean, levelScratch.AllocationOptions);
Assert.Equal(typeof(sbyte), contextScratch.ElementType); Assert.Equal(typeof(sbyte), contextScratch.ElementType);
Assert.Equal(maximumTransformDimension * maximumTransformDimension, contextScratch.Length); Assert.Equal(maximumTransformDimension * maximumTransformDimension, contextScratch.Length);
Assert.Equal(typeof(byte), outputScratch.ElementType);
Assert.Equal(64, outputScratch.Length);
// Exercise both an empty and a coded transform on each side of a frame reset. The second pass must
// reuse every constructor-owned buffer even after nonzero levels and probability updates exist.
for (int frame = 0; frame < 2; frame++)
{
coefficients.Clear();
int emptyContext = encoder.WriteCoefficients(
Av1TransformSize.Size4x4,
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
0,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(0, emptyContext);
Assert.Equal(3, allocator.AllocationLog.Count);
coefficients[0] = 1;
_ = encoder.GetCoefficientCost(
Av1TransformSize.Size4x4,
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
1,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
encoder.WriteCoefficients(
Av1TransformSize.Size4x4,
Av1TransformType.DctDct,
Av1PredictionMode.DC,
coefficients,
Av1ComponentType.Luminance,
default,
1,
false,
Av1FilterIntraMode.DC,
usesInterTransformSet: false);
encoder.Exit(out _);
encoder.Reset();
Assert.Equal(3, allocator.AllocationLog.Count);
Assert.Empty(allocator.ReturnLog);
}
} }
Assert.Equal(3, allocator.ReturnLog.Count); Assert.Equal(3, allocator.ReturnLog.Count);

44
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InterpolationFilterEntropyTests.cs

@ -114,6 +114,50 @@ public class Av1InterpolationFilterEntropyTests
} }
} }
/// <summary>
/// Verifies the encoder's context selection, read-only costing, and adaptive output against independently seeded distributions.
/// </summary>
[Theory]
[MemberData(nameof(GetContexts))]
public void EncoderUsesRequestedContextAndLiveCosts(int context)
{
ReadOnlySpan<Av1InterpolationFilter> filters =
[
Av1InterpolationFilter.Sharp,
Av1InterpolationFilter.Smooth,
Av1InterpolationFilter.Regular,
Av1InterpolationFilter.Sharp,
Av1InterpolationFilter.Regular,
Av1InterpolationFilter.Smooth,
];
// The constructor converts forward thresholds to inverse CDF storage. Supply the published forward
// values directly, independently of the production context factory.
Av1Distribution distribution = new(
ForwardThresholds[context * 2],
ForwardThresholds[(context * 2) + 1]);
using Av1SymbolWriter expectedWriter = new(Configuration.Default, 64, updateCdf: true);
using Av1SymbolEncoder encoder = new(Configuration.Default, 64, qIndex: 0, updateCdf: true);
foreach (Av1InterpolationFilter filter in filters)
{
int expectedCost = Av1ProbabilityCost.GetSymbolCost(distribution, (int)filter);
Assert.Equal(expectedCost, encoder.GetSwitchableInterpolationFilterCost(filter, context));
Assert.Equal(expectedCost, encoder.GetSwitchableInterpolationFilterCost(filter, context));
expectedWriter.WriteSymbol((int)filter, distribution);
encoder.WriteSwitchableInterpolationFilter(filter, context);
}
using IMemoryOwner<byte> expected = expectedWriter.Exit();
using IMemoryOwner<byte> actual = encoder.Exit();
Assert.Equal(expected.Memory.Span, actual.Memory.Span);
Av1SymbolDecoder decoder = new(Configuration.Default, actual.Memory.Span, 0, updateCdf: true);
foreach (Av1InterpolationFilter filter in filters)
{
Assert.Equal(filter, decoder.ReadSwitchableInterpolationFilter(context));
}
}
/// <summary> /// <summary>
/// Verifies all sixteen combinations of reference type, direction, and contributing neighbor filter state. /// Verifies all sixteen combinations of reference type, direction, and contributing neighbor filter state.
/// </summary> /// </summary>

224
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

@ -8,12 +8,15 @@ using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.Inter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Tests.Memory;
namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
@ -23,6 +26,168 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
[Trait("Format", "Avif")] [Trait("Format", "Avif")]
public class Av1IntraSuperblockEncoderTests public class Av1IntraSuperblockEncoderTests
{ {
/// <summary>
/// Verifies non-regular filter selection, retained reconstruction, and allocation-free inter tile coding.
/// </summary>
[Theory]
[InlineData((int)Av1InterpolationFilter.Smooth, false)]
[InlineData((int)Av1InterpolationFilter.Sharp, false)]
[InlineData((int)Av1InterpolationFilter.Smooth, true)]
[InlineData((int)Av1InterpolationFilter.Sharp, true)]
public void ProductionTileSelectsNonRegularInterpolation(int filterValue, bool dualFilter)
{
const int Width = 32;
const int Height = 8;
const int TargetColumn = 8;
const int BlockWidth = 8;
const int QIndex = 37;
const int TileBufferLength = 4096;
Av1InterpolationFilter filter = (Av1InterpolationFilter)filterValue;
int effort = dualFilter ? 9 : 8;
ReadOnlySpan<byte> referencePeriod = [128, 184, 208, 184, 128, 72, 48, 72];
// These are fixed half-sample responses of the reference's eight-tap smooth and sharp kernels.
// The horizontal pass rounds first by three bits and then by four; edge samples are replicated.
// Keeping the results literal avoids using the predictor under test to manufacture its own target.
ReadOnlySpan<byte> targetRow = filter == Av1InterpolationFilter.Smooth
? [159, 189, 190, 154, 102, 66, 66, 102, 154, 190, 190, 154, 102, 66, 66, 102,
154, 190, 190, 154, 102, 66, 66, 102, 154, 190, 190, 154, 102, 67, 61, 69]
: [153, 204, 200, 158, 98, 55, 55, 98, 158, 202, 202, 158, 98, 55, 55, 98,
158, 202, 202, 158, 98, 55, 55, 98, 158, 202, 202, 158, 100, 53, 59, 75];
TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone();
configuration.MemoryAllocator = allocator;
ObuColorConfig colorConfig = new()
{
IsMonochrome = true,
ColorRange = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
using Image<L8> referenceImage = new(Width, Height);
using Av1EncoderFrameBuffer<byte> reference = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0);
using Av1EncoderFrameBuffer<byte> source = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0);
using Av1EncoderFrameBuffer<byte> reconstruction = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0);
for (int y = 0; y < Height; y++)
{
Span<L8> pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
Span<byte> referenceRow = reference.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
for (int x = 0; x < Width; x++)
{
byte sample = referencePeriod[x % referencePeriod.Length];
referenceRow[x] = sample;
pixels[x] = new L8(sample);
}
targetRow.CopyTo(source.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y));
}
reference.Frame.ExtendBorders();
source.Frame.ExtendBorders();
ClearPlane(reconstruction.Luma);
// A lossless key frame gives an independent decoder exactly the reference samples used by tile search.
using MemoryStream firstSample = new();
using Av1FrameEncoder.SequenceEncoder keyEncoder = Av1FrameEncoder.CreateColorSequenceEncoder(
configuration,
Width,
Height,
colorConfig,
qIndex: 0,
effort);
keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample);
ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader;
using Av1EncoderModeInfoBuffer modeInfo = new(configuration, Width, Height, disallow4x4AllFrames: true);
Av1PictureControlSet template = CreatePicture(modeInfo, colorConfig, use128x128Superblock: false, QIndex);
ObuFrameHeader frameHeader = template.Parent.FrameHeader;
frameHeader.FrameType = ObuFrameType.InterFrame;
frameHeader.ShowFrame = true;
frameHeader.ErrorResilientMode = true;
frameHeader.RefreshFrameFlags = byte.MaxValue;
frameHeader.DisableFrameEndUpdateCdf = true;
frameHeader.ReferenceMode = ObuReferenceMode.SingleReference;
frameHeader.InterpolationFilter = Av1InterpolationFilter.Switchable;
frameHeader.AllowHighPrecisionMotionVector = true;
frameHeader.TransformMode = Av1TransformMode.Select;
frameHeader.FrameSize.FrameWidth = Width;
frameHeader.FrameSize.FrameHeight = Height;
frameHeader.FrameSize.SuperResolutionUpscaledWidth = Width;
frameHeader.FrameSize.RenderWidth = Width;
frameHeader.FrameSize.RenderHeight = Height;
frameHeader.TilesInfo.HasUniformTileSpacing = true;
Av1QuantizationLookup.UpdateFrameQuantizationState(frameHeader);
using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true);
using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration);
using Av1EncoderBlockWorkspace blockWorkspace = new(configuration);
using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true);
Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace);
int allocationCount = allocator.AllocationLog.Count;
Av1TileEncoder tileWriter = new(
symbolEncoder,
source.Frame,
reference.Frame,
reconstruction.Frame,
picture.Picture,
coefficients,
tileWorkspace,
blockWorkspace,
effort);
Assert.Equal(allocationCount, allocator.AllocationLog.Count);
Point targetPosition = new(TargetColumn >> Av1Constants.ModeInfoSizeLog2, 0);
ref Av1MacroBlockModeInfo targetMode = ref picture.Picture.GetMacroBlockModeInfo(targetPosition);
Assert.Equal(Av1ReferenceFrameType.Last, targetMode.Block.ReferenceFrame);
Assert.Equal(filter, targetMode.Block.HorizontalInterpolationFilter);
Assert.Equal(dualFilter ? Av1InterpolationFilter.Regular : filter, targetMode.Block.VerticalInterpolationFilter);
Assert.Equal(4, picture.Picture.GetDisplacementVector(targetPosition).Column);
Assert.Equal(0, picture.Picture.GetDisplacementVector(targetPosition).Row);
for (int y = 0; y < Height; y++)
{
Assert.Equal(
targetRow.Slice(TargetColumn, BlockWidth),
reconstruction.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y).Slice(TargetColumn, BlockWidth));
}
using MemoryStream secondSample = new();
using ObuWriter obuWriter = new(configuration);
obuWriter.WriteFrame(secondSample, sequenceHeader, frameHeader, tileWriter);
using Av1Decoder decoder = new(configuration);
using Av1FrameBuffer<byte> decodedFirst = decoder.DecodeFrameBuffer(firstSample.ToArray(), null, null, out _);
using Av1FrameBuffer<byte> decodedSecond = decoder.DecodeFrameBuffer(secondSample.ToArray(), null, null, out _);
// Preserve both the production stream and every managed reconstructed luma sample for exact libaom comparison.
string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.ProductionTileSelectsNonRegularInterpolation));
string outputName = $"{filter}-{dualFilter}";
using FileStream output = File.Create(Path.Combine(outputDirectory, outputName + ".obu"));
firstSample.Position = 0;
firstSample.CopyTo(output);
secondSample.Position = 0;
secondSample.CopyTo(output);
using FileStream rawOutput = File.Create(Path.Combine(outputDirectory, outputName + ".managed.yuv"));
for (int y = 0; y < Height; y++)
{
ReadOnlySpan<byte> expected = reference.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
ReadOnlySpan<byte> actual = decodedFirst.DeriveBlockPointer(Av1Plane.Y, new Point(0, y), 0, 0, out _)[..Width];
Assert.Equal(expected, actual);
rawOutput.Write(actual);
}
for (int y = 0; y < Height; y++)
{
ReadOnlySpan<byte> expected = reconstruction.Frame.View.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
ReadOnlySpan<byte> actual = decodedSecond.DeriveBlockPointer(Av1Plane.Y, new Point(0, y), 0, 0, out _)[..Width];
Assert.Equal(expected, actual);
rawOutput.Write(actual);
}
}
/// <summary> /// <summary>
/// Gets the normative eight-sample weights used to build independent smooth-mode fixtures. /// Gets the normative eight-sample weights used to build independent smooth-mode fixtures.
/// </summary> /// </summary>
@ -259,7 +424,7 @@ public class Av1IntraSuperblockEncoderTests
tilePicture.Picture, tilePicture.Picture,
512); 512);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
tileSymbolEncoder, tileSymbolEncoder,
source.Frame, source.Frame,
tileReconstruction.Frame, tileReconstruction.Frame,
@ -413,7 +578,7 @@ public class Av1IntraSuperblockEncoderTests
livePicture.Picture, livePicture.Picture,
256); 256);
Av1IntraTileWriter liveTileWriter = new( Av1TileEncoder liveTileWriter = new(
liveSymbolEncoder, liveSymbolEncoder,
source.Frame, source.Frame,
liveReconstruction.Frame, liveReconstruction.Frame,
@ -541,7 +706,7 @@ public class Av1IntraSuperblockEncoderTests
tilePicture.Picture, tilePicture.Picture,
256); 256);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
tileSymbolEncoder, tileSymbolEncoder,
source.Frame, source.Frame,
tileReconstruction.Frame, tileReconstruction.Frame,
@ -749,7 +914,7 @@ public class Av1IntraSuperblockEncoderTests
32, 32,
224, 224,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -770,7 +935,7 @@ public class Av1IntraSuperblockEncoderTests
512, 512,
3584, 3584,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -791,7 +956,7 @@ public class Av1IntraSuperblockEncoderTests
48, 48,
208, 208,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -816,7 +981,7 @@ public class Av1IntraSuperblockEncoderTests
64, 64,
192, 192,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -837,7 +1002,7 @@ public class Av1IntraSuperblockEncoderTests
1024, 1024,
3072, 3072,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -946,7 +1111,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
256); 256);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -1162,7 +1327,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
512); 512);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -1279,7 +1444,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
512); 512);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -1442,7 +1607,7 @@ public class Av1IntraSuperblockEncoderTests
pilotPicture.Picture, pilotPicture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter pilotWriter = createWriter( Av1TileEncoder pilotWriter = createWriter(
pilotSymbolEncoder, pilotSymbolEncoder,
pilotSource.Frame, pilotSource.Frame,
pilotReconstruction.Frame, pilotReconstruction.Frame,
@ -1549,7 +1714,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter tileWriter = createWriter( Av1TileEncoder tileWriter = createWriter(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -1792,7 +1957,7 @@ public class Av1IntraSuperblockEncoderTests
pilotPicture.Picture, pilotPicture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter pilotWriter = createWriter( Av1TileEncoder pilotWriter = createWriter(
pilotSymbolEncoder, pilotSymbolEncoder,
pilotSource.Frame, pilotSource.Frame,
pilotReconstruction.Frame, pilotReconstruction.Frame,
@ -1954,7 +2119,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter tileWriter = createWriter( Av1TileEncoder tileWriter = createWriter(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -2158,7 +2323,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
2048); 2048);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -2183,7 +2348,7 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
static value => (byte)value, static value => (byte)value,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -2198,7 +2363,7 @@ public class Av1IntraSuperblockEncoderTests
12, 12,
static value => (ushort)(value << 4), static value => (ushort)(value << 4),
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) => static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter( new Av1TileEncoder(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -2303,7 +2468,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter tileWriter = createTileWriter( Av1TileEncoder tileWriter = createTileWriter(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -2436,7 +2601,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
4096); 4096);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -2550,7 +2715,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
4096); 4096);
Av1IntraTileWriter tileWriter = new( Av1TileEncoder tileWriter = new(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -2631,8 +2796,8 @@ public class Av1IntraSuperblockEncoderTests
frameHeader.TilesInfo.HasUniformTileSpacing = true; frameHeader.TilesInfo.HasUniformTileSpacing = true;
using MemoryStream stream = new(); using MemoryStream stream = new();
new ObuWriter().WriteAll( using ObuWriter obuWriter = new(Configuration.Default);
Configuration.Default, obuWriter.WriteSequenceFrame(
stream, stream,
sequenceHeader, sequenceHeader,
frameHeader, frameHeader,
@ -2689,14 +2854,16 @@ public class Av1IntraSuperblockEncoderTests
FrameSize = new ObuFrameSize() FrameSize = new ObuFrameSize()
}, },
FrameHeader = frameHeader, FrameHeader = frameHeader,
PreviousQIndex = [qIndex] PreviousQIndex = new int[] { qIndex }
}, },
SegmentationNeighborMap = new byte[modeInfo.ModeInfoColumnCount * modeInfo.ModeInfoRowCount], SegmentationNeighborMap = new byte[modeInfo.ModeInfoColumnCount * modeInfo.ModeInfoRowCount],
ModeInfoGrid = modeInfo.Grid, ModeInfoGrid = modeInfo.Grid,
ModeInfoAllocation = modeInfo.Allocation, ModeInfoAllocation = modeInfo.Allocation,
ModeInfoStride = modeInfo.ModeInfoStride, ModeInfoStride = modeInfo.ModeInfoStride,
Disallow4x4AllFrames = modeInfo.Disallow4x4AllFrames, Disallow4x4AllFrames = modeInfo.Disallow4x4AllFrames,
CdefPreset = [[-1, -1, -1, -1]] CdefPreset = new int[] { -1, -1, -1, -1 },
TileDataOffsets = Memory<int>.Empty,
TileDataLengths = Memory<int>.Empty
}; };
} }
@ -2813,7 +2980,7 @@ public class Av1IntraSuperblockEncoderTests
picture.Picture, picture.Picture,
TileBufferLength); TileBufferLength);
Av1IntraTileWriter tileWriter = createTileWriter( Av1TileEncoder tileWriter = createTileWriter(
symbolEncoder, symbolEncoder,
source.Frame, source.Frame,
reconstruction.Frame, reconstruction.Frame,
@ -3044,7 +3211,7 @@ public class Av1IntraSuperblockEncoderTests
updateCdf: !frameHeader.DisableCdfUpdate); updateCdf: !frameHeader.DisableCdfUpdate);
} }
private delegate Av1IntraTileWriter TileWriterFactory<TSample>( private delegate Av1TileEncoder TileWriterFactory<TSample>(
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
Av1EncoderFrame<TSample> source, Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reconstruction, Av1EncoderFrame<TSample> reconstruction,
@ -3143,7 +3310,8 @@ public class Av1IntraSuperblockEncoderTests
macroBlock, macroBlock,
Av1BlockSize.Block8x8, Av1BlockSize.Block8x8,
Av1PredictionMode.DC, Av1PredictionMode.DC,
0); 0,
isIntraFrame: true);
modeInfo.Block = new Av1EncoderBlockModeInfo modeInfo.Block = new Av1EncoderBlockModeInfo
{ {

31
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1MotionVectorEntropyTests.cs

@ -112,6 +112,37 @@ public class Av1MotionVectorEntropyTests
Assert.True(decoder.ReadDrl(1)); Assert.True(decoder.ReadDrl(1));
} }
[Theory]
[InlineData((int)Av1MotionVectorPrecision.Integer, 16, -24)]
[InlineData((int)Av1MotionVectorPrecision.QuarterSample, 6, -10)]
[InlineData((int)Av1MotionVectorPrecision.EighthSample, 11, -17)]
public void WriteMotionVectorRoundTripsRequestedPrecision(
int precisionValue,
int rowDelta,
int columnDelta)
{
Av1MotionVectorPrecision precision = (Av1MotionVectorPrecision)precisionValue;
Av1MotionVector reference = new(27, -11);
Av1MotionVector value = reference + new Av1MotionVector(rowDelta, columnDelta);
Av1MotionVectorContext writerContext = new();
Av1Distribution trailingDistribution = Av1DefaultDistributions.Drl[1];
using Av1SymbolWriter writer = new(Configuration.Default, 32, updateCdf: true);
writerContext.Write(writer, value, reference, precision);
writer.WriteSymbol(true, trailingDistribution);
using IMemoryOwner<byte> encoded = writer.Exit();
Av1FrameEntropyContext decoderContext = new(0);
Av1SymbolDecoder decoder = new(
Configuration.Default,
encoded.Memory.Span,
decoderContext,
updateCdf: true);
Assert.Equal(value, decoder.ReadMotionVector(reference, precision));
Assert.True(decoder.ReadDrl(1));
}
/// <summary> /// <summary>
/// Verifies that normal and displacement motion vectors never share adaptive distribution state. /// Verifies that normal and displacement motion vectors never share adaptive distribution state.
/// </summary> /// </summary>

140
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1ResidualBuilderTests.cs

@ -15,6 +15,128 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
[Trait("Format", "Avif")] [Trait("Format", "Avif")]
public class Av1ResidualBuilderTests public class Av1ResidualBuilderTests
{ {
/// <summary>
/// Verifies precision limits, independent strides, candidate order, and exact final-row bounds.
/// </summary>
[Theory]
[InlineData(255, false)]
[InlineData(255, true)]
public void ByteSearchMetricsMatchKnownMoments(int maximum, bool negative)
{
const int SourceStride = 13;
const int PredictionStride = 17;
const int SourceOffset = 1;
const int PredictionOffset = 3;
byte[] sourceBuffer = new byte[SourceOffset + (7 * SourceStride) + 8];
byte[] predictionBuffer = new byte[PredictionOffset + (7 * PredictionStride) + 11];
sourceBuffer.AsSpan().Fill((byte)maximum);
predictionBuffer.AsSpan().Fill((byte)maximum);
Span<byte> source = sourceBuffer.AsSpan(SourceOffset);
Span<byte> prediction = predictionBuffer.AsSpan(PredictionOffset);
int[] sums = [-1, 0, 0, 0, 0, -1];
// The last source row contains exactly eight samples, and the four prediction windows require
// exactly eleven. Distinct nonzero padding catches accidental participation of neighboring rows.
for (int row = 0; row < 8; row++)
{
source.Slice(row * SourceStride, 8).Fill((byte)(negative ? 0 : maximum));
prediction.Slice(row * PredictionStride, 11).Fill((byte)(negative ? maximum : 0));
}
Assert.Equal(64 * maximum, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride));
Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out int sum, out int squaredSum);
Assert.Equal((negative ? -64 : 64) * maximum, sum);
Assert.Equal(64 * maximum * maximum, squaredSum);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4));
Assert.Equal(new[] { -1, 64 * maximum, 64 * maximum, 64 * maximum, 64 * maximum, -1 }, sums);
// Source (2*y + x) and prediction (y + 2*x) give residual (y - x) * step, exercising both signs
// and distinct rows. Across the 8x8 square its signed sum is zero, absolute sum 168, and squared sum 672.
// Offsetting prediction by 1, 2, or 3 columns gives absolute sums 198, 276, and 386 respectively.
int step = maximum / 32;
for (int row = 0; row < 8; row++)
{
for (int column = 0; column < 8; column++)
{
source[(row * SourceStride) + column] = (byte)(((2 * row) + column) * step);
}
for (int column = 0; column < 11; column++)
{
prediction[(row * PredictionStride) + column] = (byte)((row + (2 * column)) * step);
}
}
Assert.Equal(168 * step, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride));
Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out sum, out squaredSum);
Assert.Equal(0, sum);
Assert.Equal(672 * step * step, squaredSum);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4));
Assert.Equal(new[] { -1, 168 * step, 198 * step, 276 * step, 386 * step, -1 }, sums);
}
/// <summary>
/// Verifies precision limits, independent strides, candidate order, and exact final-row bounds.
/// </summary>
[Theory]
[InlineData(1023, false)]
[InlineData(1023, true)]
[InlineData(4095, false)]
[InlineData(4095, true)]
public void HighBitDepthSearchMetricsMatchKnownMoments(int maximum, bool negative)
{
const int SourceStride = 13;
const int PredictionStride = 17;
const int SourceOffset = 1;
const int PredictionOffset = 3;
ushort[] sourceBuffer = new ushort[SourceOffset + (7 * SourceStride) + 8];
ushort[] predictionBuffer = new ushort[PredictionOffset + (7 * PredictionStride) + 11];
sourceBuffer.AsSpan().Fill((ushort)maximum);
predictionBuffer.AsSpan().Fill((ushort)maximum);
Span<ushort> source = sourceBuffer.AsSpan(SourceOffset);
Span<ushort> prediction = predictionBuffer.AsSpan(PredictionOffset);
int[] sums = [-1, 0, 0, 0, 0, -1];
// The last source row contains exactly eight samples, and the four prediction windows require
// exactly eleven. Distinct nonzero padding catches accidental participation of neighboring rows.
for (int row = 0; row < 8; row++)
{
source.Slice(row * SourceStride, 8).Fill((ushort)(negative ? 0 : maximum));
prediction.Slice(row * PredictionStride, 11).Fill((ushort)(negative ? maximum : 0));
}
Assert.Equal(64 * maximum, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride));
Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out int sum, out int squaredSum);
Assert.Equal((negative ? -64 : 64) * maximum, sum);
Assert.Equal(64 * maximum * maximum, squaredSum);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4));
Assert.Equal(new[] { -1, 64 * maximum, 64 * maximum, 64 * maximum, 64 * maximum, -1 }, sums);
// Source (2*y + x) and prediction (y + 2*x) give residual (y - x) * step, exercising both signs
// and distinct rows. Across the 8x8 square its signed sum is zero, absolute sum 168, and squared sum 672.
// Offsetting prediction by 1, 2, or 3 columns gives absolute sums 198, 276, and 386 respectively.
int step = maximum / 32;
for (int row = 0; row < 8; row++)
{
for (int column = 0; column < 8; column++)
{
source[(row * SourceStride) + column] = (ushort)(((2 * row) + column) * step);
}
for (int column = 0; column < 11; column++)
{
prediction[(row * PredictionStride) + column] = (ushort)((row + (2 * column)) * step);
}
}
Assert.Equal(168 * step, Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride));
Av1ResidualBuilder.GetMoments8x8(source, SourceStride, prediction, PredictionStride, out sum, out squaredSum);
Assert.Equal(0, sum);
Assert.Equal(672 * step * step, squaredSum);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, SourceStride, prediction, PredictionStride, sums.AsSpan(1, 4));
Assert.Equal(new[] { -1, 168 * step, 198 * step, 276 * step, 386 * step, -1 }, sums);
}
private const HwIntrinsics ResidualConfigurations = private const HwIntrinsics ResidualConfigurations =
HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic; HwIntrinsics.AllowAll | HwIntrinsics.DisableAVX512F | HwIntrinsics.DisableAVX | HwIntrinsics.DisableHWIntrinsic;
@ -133,6 +255,14 @@ public class Av1ResidualBuilderTests
width, width,
height); height);
int[] candidateSums = new int[4];
_ = Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, width, prediction, width);
_ = Av1ResidualBuilder.SumAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, width, prediction, width, candidateSums);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width, candidateSums);
Av1ResidualBuilder.GetMoments8x8(source, width, prediction, width, out _, out _);
Av1ResidualBuilder.GetMoments8x8(highBitDepthSource, width, highBitDepthPrediction, width, out _, out _);
long sum = 0; long sum = 0;
long before = GC.GetAllocatedBytesForCurrentThread(); long before = GC.GetAllocatedBytesForCurrentThread();
for (int iteration = 0; iteration < 32; iteration++) for (int iteration = 0; iteration < 32; iteration++)
@ -147,6 +277,16 @@ public class Av1ResidualBuilderTests
width, width,
width, width,
height); height);
sum += Av1ResidualBuilder.SumAbsoluteDifferences8x8(source, width, prediction, width);
sum += Av1ResidualBuilder.SumAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width);
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(source, width, prediction, width, candidateSums);
sum += candidateSums[0];
Av1ResidualBuilder.SumFourAbsoluteDifferences8x8(highBitDepthSource, width, highBitDepthPrediction, width, candidateSums);
sum += candidateSums[3];
Av1ResidualBuilder.GetMoments8x8(source, width, prediction, width, out int byteSum, out int byteSquares);
Av1ResidualBuilder.GetMoments8x8(highBitDepthSource, width, highBitDepthPrediction, width, out int wordSum, out int wordSquares);
sum += byteSum + byteSquares + wordSum + wordSquares;
} }
long allocated = GC.GetAllocatedBytesForCurrentThread() - before; long allocated = GC.GetAllocatedBytesForCurrentThread() - before;

77
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1SingleReferenceEntropyTests.cs

@ -14,6 +14,11 @@ namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
[Trait("Format", "Avif")] [Trait("Format", "Avif")]
public class Av1SingleReferenceEntropyTests public class Av1SingleReferenceEntropyTests
{ {
/// <summary>
/// The quantizer index used to initialize an otherwise unrelated tile entropy encoder.
/// </summary>
private const int BaseQIndex = 128;
/// <summary> /// <summary>
/// Verifies all eighteen normative single-reference distributions against the reference decoder's forward Q15 defaults. /// Verifies all eighteen normative single-reference distributions against the reference decoder's forward Q15 defaults.
/// </summary> /// </summary>
@ -121,6 +126,41 @@ public class Av1SingleReferenceEntropyTests
} }
} }
/// <summary>
/// Verifies that the production writer emits every single-reference branch consumed by the decoder.
/// </summary>
/// <param name="referenceFrameValue">The encoded reference-frame label.</param>
[Theory]
[InlineData((int)Av1ReferenceFrameType.Last)]
[InlineData((int)Av1ReferenceFrameType.Last2)]
[InlineData((int)Av1ReferenceFrameType.Last3)]
[InlineData((int)Av1ReferenceFrameType.Golden)]
[InlineData((int)Av1ReferenceFrameType.Backward)]
[InlineData((int)Av1ReferenceFrameType.Alternate2)]
[InlineData((int)Av1ReferenceFrameType.Alternate)]
public void SingleReferenceWriterRoundTripsEveryReference(int referenceFrameValue)
{
Av1ReferenceFrameType referenceFrame = (Av1ReferenceFrameType)referenceFrameValue;
InlineArray8<byte> referenceCountStorage = default;
Span<byte> referenceCounts = referenceCountStorage;
referenceCounts[(int)Av1ReferenceFrameType.Last] = 5;
referenceCounts[(int)Av1ReferenceFrameType.Last2] = 1;
referenceCounts[(int)Av1ReferenceFrameType.Last3] = 2;
referenceCounts[(int)Av1ReferenceFrameType.Golden] = 2;
referenceCounts[(int)Av1ReferenceFrameType.Backward] = 3;
referenceCounts[(int)Av1ReferenceFrameType.Alternate2] = 3;
referenceCounts[(int)Av1ReferenceFrameType.Alternate] = 6;
using Av1SymbolEncoder encoder = new(Configuration.Default, 8, BaseQIndex, updateCdf: true);
encoder.WriteSingleReference(referenceFrame, referenceCounts);
using IMemoryOwner<byte> encoded = encoder.Exit();
Av1SymbolDecoder decoder = new(Configuration.Default, encoded.Memory.Span, BaseQIndex, updateCdf: true);
Av1ReferenceFrameType decodedReference = ReadSingleReference(ref decoder, referenceCounts);
Assert.Equal(referenceFrame, decodedReference);
}
/// <summary> /// <summary>
/// Verifies one-pass neighbor collection, compound-neighbor votes, clearing, and intra-neighbor exclusion. /// Verifies one-pass neighbor collection, compound-neighbor votes, clearing, and intra-neighbor exclusion.
/// </summary> /// </summary>
@ -314,6 +354,43 @@ public class Av1SingleReferenceEntropyTests
_ => decoder.ReadSingleReferenceIsAlternate2(context), _ => decoder.ReadSingleReferenceIsAlternate2(context),
}; };
/// <summary>
/// Reads one complete single-reference branch through the production semantic entry points.
/// </summary>
private static Av1ReferenceFrameType ReadSingleReference(
ref Av1SymbolDecoder decoder,
scoped ReadOnlySpan<byte> referenceCounts)
{
int context = Av1SymbolContextHelper.GetSingleReferenceBackwardContext(referenceCounts);
if (decoder.ReadSingleReferenceIsBackward(context))
{
context = Av1SymbolContextHelper.GetSingleReferenceAlternateContext(referenceCounts);
if (decoder.ReadSingleReferenceIsAlternate(context))
{
return Av1ReferenceFrameType.Alternate;
}
context = Av1SymbolContextHelper.GetSingleReferenceAlternate2Context(referenceCounts);
return decoder.ReadSingleReferenceIsAlternate2(context)
? Av1ReferenceFrameType.Alternate2
: Av1ReferenceFrameType.Backward;
}
context = Av1SymbolContextHelper.GetSingleReferenceLast3OrGoldenContext(referenceCounts);
if (decoder.ReadSingleReferenceIsLast3OrGolden(context))
{
context = Av1SymbolContextHelper.GetSingleReferenceGoldenContext(referenceCounts);
return decoder.ReadSingleReferenceIsGolden(context)
? Av1ReferenceFrameType.Golden
: Av1ReferenceFrameType.Last3;
}
context = Av1SymbolContextHelper.GetSingleReferenceLast2Context(referenceCounts);
return decoder.ReadSingleReferenceIsLast2(context)
? Av1ReferenceFrameType.Last2
: Av1ReferenceFrameType.Last;
}
/// <summary> /// <summary>
/// Creates decoded block-mode state with the requested primary and secondary reference labels. /// Creates decoded block-mode state with the requested primary and secondary reference labels.
/// </summary> /// </summary>

4
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs

@ -614,8 +614,8 @@ public class Av1TransformBlockEncoderTests
Av1EncoderModeDecisionWorkspace<ushort> modeWorkspace = workspace.GetModeDecisionWorkspace<ushort>(); Av1EncoderModeDecisionWorkspace<ushort> modeWorkspace = workspace.GetModeDecisionWorkspace<ushort>();
Av1EncoderPaletteWorkspace<ushort> paletteWorkspace = modeWorkspace.Palette; Av1EncoderPaletteWorkspace<ushort> paletteWorkspace = modeWorkspace.Palette;
Av1EncoderIntraBlockCopyWorkspace<ushort> intraBlockCopyWorkspace = Av1EncoderInterPredictionWorkspace<ushort> intraBlockCopyWorkspace =
workspace.GetIntraBlockCopyWorkspace<ushort>(); workspace.GetInterPredictionWorkspace<ushort>();
Assert.Equal( Assert.Equal(
(2 * Av1Constants.MaxTransformSize) + 1, (2 * Av1Constants.MaxTransformSize) + 1,

105
tests/ImageSharp.Tests/Formats/Heif/Av1/ObuFrameHeaderTests.cs

@ -129,8 +129,8 @@ public class ObuFrameHeaderTests
MemoryStream encoded = new(); MemoryStream encoded = new();
// Act 2 // Act 2
ObuWriter obuWriter = new(); using ObuWriter obuWriter = new(Configuration.Default);
obuWriter.WriteAll(Configuration.Default, encoded, obuReader.SequenceHeader, obuReader.FrameHeader, tileStub); obuWriter.WriteSequenceFrame(encoded, obuReader.SequenceHeader, obuReader.FrameHeader, tileStub);
// Assert // Assert
byte[] encodedArray = encoded.ToArray(); byte[] encodedArray = encoded.ToArray();
@ -157,8 +157,8 @@ public class ObuFrameHeaderTests
MemoryStream encoded = new(); MemoryStream encoded = new();
// Act 2 // Act 2
ObuWriter obuWriter = new(); using ObuWriter obuWriter = new(Configuration.Default);
obuWriter.WriteAll(Configuration.Default, encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader, tileStub); obuWriter.WriteSequenceFrame(encoded, obuReader1.SequenceHeader, obuReader1.FrameHeader, tileStub);
// Assign 2 // Assign 2
Span<byte> encodedBuffer = encoded.ToArray(); Span<byte> encodedBuffer = encoded.ToArray();
@ -456,8 +456,8 @@ public class ObuFrameHeaderTests
tileStub.ReadTile(emptyTile, 0); tileStub.ReadTile(emptyTile, 0);
using MemoryStream stream = new(); using MemoryStream stream = new();
ObuWriter writer = new(); using ObuWriter writer = new(Configuration.Default);
writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, tileStub); writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub);
byte[] bitStream = stream.ToArray(); byte[] bitStream = stream.ToArray();
int sequenceObuOffset = DefaultTemporalDelimiterBitStream.Length; int sequenceObuOffset = DefaultTemporalDelimiterBitStream.Length;
@ -639,10 +639,9 @@ public class ObuFrameHeaderTests
{ {
// Arrange // Arrange
using MemoryStream stream = new(2); using MemoryStream stream = new(2);
ObuWriter obuWriter = new();
// Act // Act
obuWriter.WriteAll(Configuration.Default, stream, null, null, null); ObuWriter.WriteTemporalDelimiter(stream);
byte[] actual = stream.GetBuffer(); byte[] actual = stream.GetBuffer();
// Assert // Assert
@ -655,10 +654,9 @@ public class ObuFrameHeaderTests
// Arrange // Arrange
using MemoryStream stream = new(10); using MemoryStream stream = new(10);
ObuSequenceHeader input = GetDefaultSequenceHeader(); ObuSequenceHeader input = GetDefaultSequenceHeader();
ObuWriter obuWriter = new();
// Act // Act
obuWriter.WriteAll(Configuration.Default, stream, input, null, null); ObuWriter.WriteSequenceHeader(Configuration.Default, stream, input);
byte[] buffer = stream.GetBuffer(); byte[] buffer = stream.GetBuffer();
// Assert // Assert
@ -667,6 +665,68 @@ public class ObuFrameHeaderTests
Assert.Equal(DefaultSequenceHeaderBitStream, actual); Assert.Equal(DefaultSequenceHeaderBitStream, actual);
} }
[Fact]
public void WriteNonReducedSequenceHeaderPreservesTimingAndDecoderModel()
{
ObuSequenceHeader input = GetDefaultSequenceHeader();
input.IsStillPicture = false;
input.IsReducedStillPictureHeader = false;
input.TimingInfoPresentFlag = true;
input.TimingInfo = new ObuTimingInfo
{
NumUnitsInDisplayTick = 1001,
TimeScale = 60000,
EqualPictureInterval = true,
NumTicksPerPicture = 2
};
input.DecoderModelInfoPresentFlag = true;
input.DecoderModelInfo = new ObuDecoderModelInfo
{
BufferDelayLength = 12,
NumUnitsInDecodingTick = 1000,
BufferRemovalTimeLength = 10,
FramePresentationTimeLength = 9
};
input.InitialDisplayDelayPresentFlag = true;
ObuOperatingPoint inputOperatingPoint = input.OperatingPoint[0];
inputOperatingPoint.SequenceLevelIndex = Av1Constants.SequenceTierMinimumLevelIndex;
inputOperatingPoint.SequenceTier = 1;
inputOperatingPoint.IsDecoderModelInfoPresent = true;
inputOperatingPoint.DecoderBufferDelay = 137;
inputOperatingPoint.EncoderBufferDelay = 211;
inputOperatingPoint.LowDelayMode = true;
inputOperatingPoint.IsInitialDisplayDelayPresent = true;
inputOperatingPoint.InitialDisplayDelay = 4;
using MemoryStream stream = new();
ObuWriter.WriteSequenceHeader(Configuration.Default, stream, input);
byte[] bitStream = stream.ToArray();
Av1BitStreamReader reader = new(bitStream);
ObuReader obuReader = new();
obuReader.ReadAll(ref reader, bitStream.Length, () => new Av1TileDecoderStub());
ObuSequenceHeader output = obuReader.SequenceHeader;
ObuTimingInfo outputTiming = output.GetTimingInfo();
ObuDecoderModelInfo outputDecoderModel = output.GetDecoderModelInfo();
ObuOperatingPoint outputOperatingPoint = output.OperatingPoint[0];
Assert.False(output.IsStillPicture);
Assert.False(output.IsReducedStillPictureHeader);
Assert.Equal(input.TimingInfo.NumUnitsInDisplayTick, outputTiming.NumUnitsInDisplayTick);
Assert.Equal(input.TimingInfo.TimeScale, outputTiming.TimeScale);
Assert.Equal(input.TimingInfo.NumTicksPerPicture, outputTiming.NumTicksPerPicture);
Assert.Equal(input.DecoderModelInfo.BufferDelayLength, outputDecoderModel.BufferDelayLength);
Assert.Equal(input.DecoderModelInfo.NumUnitsInDecodingTick, outputDecoderModel.NumUnitsInDecodingTick);
Assert.Equal(input.DecoderModelInfo.BufferRemovalTimeLength, outputDecoderModel.BufferRemovalTimeLength);
Assert.Equal(input.DecoderModelInfo.FramePresentationTimeLength, outputDecoderModel.FramePresentationTimeLength);
Assert.Equal(inputOperatingPoint.SequenceTier, outputOperatingPoint.SequenceTier);
Assert.Equal(inputOperatingPoint.DecoderBufferDelay, outputOperatingPoint.DecoderBufferDelay);
Assert.Equal(inputOperatingPoint.EncoderBufferDelay, outputOperatingPoint.EncoderBufferDelay);
Assert.Equal(inputOperatingPoint.LowDelayMode, outputOperatingPoint.LowDelayMode);
Assert.Equal(inputOperatingPoint.InitialDisplayDelay, outputOperatingPoint.InitialDisplayDelay);
}
/// <summary> /// <summary>
/// Verifies that the combined frame OBU declares exactly the payload bytes emitted by the writer. /// Verifies that the combined frame OBU declares exactly the payload bytes emitted by the writer.
/// </summary> /// </summary>
@ -680,10 +740,10 @@ public class ObuFrameHeaderTests
Av1TileDecoderStub tileStub = new(); Av1TileDecoderStub tileStub = new();
byte[] tileData = [0x80]; byte[] tileData = [0x80];
tileStub.ReadTile(tileData, 0); tileStub.ReadTile(tileData, 0);
ObuWriter obuWriter = new(); using ObuWriter obuWriter = new(Configuration.Default);
// Act // Act
obuWriter.WriteAll(Configuration.Default, stream, sequenceInput, frameInput, tileStub); obuWriter.WriteSequenceFrame(stream, sequenceInput, frameInput, tileStub);
byte[] bitStream = stream.ToArray(); byte[] bitStream = stream.ToArray();
// Assert // Assert
@ -706,10 +766,10 @@ public class ObuFrameHeaderTests
} }
/// <summary> /// <summary>
/// Verifies that the OBU writer streams an encoded tile from its owning buffer without renting a second payload-sized buffer. /// Verifies that the OBU writer reuses header scratch and streams encoded tiles without a payload-sized copy.
/// </summary> /// </summary>
[Fact] [Fact]
public void WriteFrameStreamsTilePayloadWithoutRentingPayloadCopy() public void WriterReusesHeaderScratchAndStreamsTilePayloadWithoutCopy()
{ {
const int TilePayloadLength = 64 * 1024; const int TilePayloadLength = 64 * 1024;
@ -725,8 +785,13 @@ public class ObuFrameHeaderTests
tileStub.ReadTile(tileData, 0); tileStub.ReadTile(tileData, 0);
using MemoryStream stream = new(); using MemoryStream stream = new();
ObuWriter writer = new(); using (ObuWriter writer = new(configuration))
writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileStub); {
writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub);
// A second complete write must reuse the same bounded header owner retained by the writer.
writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub);
}
TestMemoryAllocator.AllocationRequest headerScratch = Assert.Single(allocator.AllocationLog); TestMemoryAllocator.AllocationRequest headerScratch = Assert.Single(allocator.AllocationLog);
Assert.Equal(typeof(byte), headerScratch.ElementType); Assert.Equal(typeof(byte), headerScratch.ElementType);
@ -760,8 +825,8 @@ public class ObuFrameHeaderTests
sourceTiles.ReadTile([0x80], 1); sourceTiles.ReadTile([0x80], 1);
using MemoryStream stream = new(); using MemoryStream stream = new();
ObuWriter writer = new(); using ObuWriter writer = new(Configuration.Default);
writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, sourceTiles); writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, sourceTiles);
byte[] bitStream = stream.ToArray(); byte[] bitStream = stream.ToArray();
Assert.Equal([0x00, 0x80, 0x80], bitStream[^3..]); Assert.Equal([0x00, 0x80, 0x80], bitStream[^3..]);
@ -936,8 +1001,8 @@ public class ObuFrameHeaderTests
tileStub.ReadTile([0x80], 1); tileStub.ReadTile([0x80], 1);
using MemoryStream stream = new(); using MemoryStream stream = new();
ObuWriter writer = new(); using ObuWriter writer = new(Configuration.Default);
writer.WriteAll(Configuration.Default, stream, sequenceHeader, frameHeader, tileStub); writer.WriteSequenceFrame(stream, sequenceHeader, frameHeader, tileStub);
return stream.ToArray(); return stream.ToArray();
} }

476
tests/ImageSharp.Tests/Formats/Heif/HeifEncoderTests.cs

@ -4,14 +4,13 @@
using System.Buffers; using System.Buffers;
using System.Buffers.Binary; using System.Buffers.Binary;
using System.Text; using System.Text;
using SixLabors.ImageSharp.ColorProfiles;
using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Heif; using SixLabors.ImageSharp.Formats.Heif;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.Metadata;
using SixLabors.ImageSharp.Metadata.Profiles.Cicp; using SixLabors.ImageSharp.Metadata.Profiles.Cicp;
using SixLabors.ImageSharp.Metadata.Profiles.Exif; using SixLabors.ImageSharp.Metadata.Profiles.Exif;
using SixLabors.ImageSharp.Metadata.Profiles.Icc; using SixLabors.ImageSharp.Metadata.Profiles.Icc;
@ -171,7 +170,7 @@ public class HeifEncoderTests
} }
[Fact] [Fact]
public void LegacyJpegRejectsLosslessEncoding() public void LegacyJpegIgnoresLosslessOption()
{ {
using Image<Rgba32> image = new(1, 1); using Image<Rgba32> image = new(1, 1);
using MemoryStream stream = new(); using MemoryStream stream = new();
@ -181,14 +180,19 @@ public class HeifEncoderTests
Lossless = true Lossless = true
}; };
Assert.Throws<NotSupportedException>(() => image.Save(stream, encoder)); image.Save(stream, encoder);
Assert.Equal(0, stream.Length); stream.Position = 0;
using Image<Rgba32> decoded = Image.Load<Rgba32>(stream);
Assert.Equal(image.Size, decoded.Size);
Assert.Equal(
HeifCompressionMethod.LegacyJpeg,
decoded.Metadata.GetHeifMetadata().CompressionMethod);
} }
[Theory] [Theory]
[InlineData(HeifBitDepth.Bit10)] [InlineData(HeifBitDepth.Bit10)]
[InlineData(HeifBitDepth.Bit12)] [InlineData(HeifBitDepth.Bit12)]
public void LegacyJpegRejectsHighBitDepth(HeifBitDepth bitDepth) public void LegacyJpegNormalizesHighBitDepthToEightBit(HeifBitDepth bitDepth)
{ {
using Image<Rgba32> image = new(1, 1); using Image<Rgba32> image = new(1, 1);
using MemoryStream stream = new(); using MemoryStream stream = new();
@ -198,8 +202,310 @@ public class HeifEncoderTests
BitDepth = bitDepth BitDepth = bitDepth
}; };
Assert.Throws<NotSupportedException>(() => image.Save(stream, encoder)); image.Save(stream, encoder);
Assert.Equal(0, stream.Length); stream.Position = 0;
using Image<Rgba32> decoded = Image.Load<Rgba32>(stream);
Assert.Equal(HeifBitDepth.Bit8, decoded.Metadata.GetHeifMetadata().BitDepth);
}
[Fact]
public void LegacyJpegEncodesRootFrameFromImageSequence()
{
using Image<Rgba32> image = new(1, 1);
image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[0] = new Rgba32(255, 255, 255);
image.Frames.AddFrame(image.Frames.RootFrame);
image.Frames[1].PixelBuffer.DangerousGetRowSpan(0)[0] = new Rgba32(0, 0, 0);
using MemoryStream stream = new();
HeifEncoder encoder = new()
{
CompressionMethod = HeifCompressionMethod.LegacyJpeg,
Quality = 100
};
image.Save(stream, encoder);
stream.Position = 0;
using Image<Rgba32> decoded = Image.Load<Rgba32>(stream);
Assert.Single(decoded.Frames);
Assert.Equal(new Rgba32(255, 255, 255), decoded.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(0)[0]);
}
[Fact]
public void Av1ImageSequencePreservesSeparateRootFrame()
{
const int width = 8;
const int height = 8;
using Image<Rgb24> image = new(width, height);
for (int row = 0; row < height; row++)
{
image.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row).Fill(new Rgb24(255, 255, 255));
}
image.Frames.AddFrame(image.Frames.RootFrame);
for (int row = 0; row < height; row++)
{
image.Frames[1].PixelBuffer.DangerousGetRowSpan(row).Fill(new Rgb24(0, 0, 0));
}
image.Frames[1].Metadata.GetHeifMetadata().FrameDelay = new Rational(1, 20);
using MemoryStream stream = new();
HeifEncoder encoder = new()
{
CompressionMethod = HeifCompressionMethod.Av1,
AnimateRootFrame = false,
Lossless = true,
Effort = 0
};
image.Save(stream, encoder);
byte[] file = stream.ToArray();
Span<byte> fileType = GetTopLevelBox(file, Heif4CharCode.Ftyp);
Assert.Equal(Heif4CharCode.Miaf, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[^sizeof(uint)..]));
Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(GetMetadataChild(file, Heif4CharCode.Pitm)[12..]));
stream.Position = 0;
using Image<Rgb24> decoded = Image.Load<Rgb24>(stream);
Assert.Equal(2, decoded.Frames.Count);
Assert.False(decoded.Metadata.GetHeifMetadata().AnimateRootFrame);
Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded));
Assert.Equal(
image.Frames[1].Metadata.GetHeifMetadata().FrameDelay,
decoded.Frames[1].Metadata.GetHeifMetadata().FrameDelay);
}
[Fact]
public void GridDecoderAcceptsSmallerRightAndBottomColorAndAlphaCells()
{
const int tileWidth = 64;
const int tileHeight = 64;
const int outputWidth = 96;
const int outputHeight = 96;
Size[] tileSizes =
[
new(tileWidth, tileHeight),
new(outputWidth - tileWidth, tileHeight),
new(tileWidth, outputHeight - tileHeight),
new(outputWidth - tileWidth, outputHeight - tileHeight)
];
Rgba32[] tileColors =
[
new(32, 32, 32),
new(64, 64, 64),
new(96, 96, 96),
new(128, 128, 128)
];
ObuColorConfig colorConfig = new()
{
IsColorDescriptionPresent = true,
ColorPrimaries = ObuColorPrimaries.Bt709,
TransferCharacteristics = ObuTransferCharacteristics.Srgb,
MatrixCoefficients = ObuMatrixCoefficients.Bt709,
ColorRange = true,
IsMonochrome = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
List<HeifItem> items = [];
Dictionary<uint, byte[]> payloads = [];
HeifItem gridItem = new(Heif4CharCode.Grid, 1);
gridItem.SetExtent(new Size(outputWidth, outputHeight));
items.Add(gridItem);
HeifItemLink gridLink = new(Heif4CharCode.Dimg, gridItem.Id);
for (int tileIndex = 0; tileIndex < tileSizes.Length; tileIndex++)
{
Size tileSize = tileSizes[tileIndex];
using Image<Rgba32> tile = new(tileSize.Width, tileSize.Height, tileColors[tileIndex]);
using MemoryStream payload = new();
ObuSequenceHeader header = Av1FrameEncoder.Encode(
Configuration.Default,
tile.Frames.RootFrame,
payload,
colorConfig,
qIndex: 0,
effort: 0);
uint itemId = (uint)tileIndex + 2;
HeifItem tileItem = new(Heif4CharCode.Av01, itemId)
{
Av1CodecConfiguration = new Av1CodecConfiguration(header)
};
tileItem.SetExtent(tileSize);
items.Add(tileItem);
gridLink.DestinationIds.Add(itemId);
payloads.Add(itemId, payload.ToArray());
}
List<HeifItemLink> links = [gridLink];
GridHeifItemDecoder<Rgba32> decoder = new(items, links, ReadItem);
Span<byte> descriptor = [0, 0, 1, 1, 0, outputWidth, 0, outputHeight];
using Image<Rgba32> result = decoder.DecodeItemData(
new DecoderOptions { Configuration = Configuration.Default },
gridItem,
descriptor,
null,
TestContext.Current.CancellationToken);
for (int y = 0; y < outputHeight; y++)
{
for (int x = 0; x < outputWidth; x++)
{
int tileIndex = (y < tileHeight ? 0 : 2) + (x < tileWidth ? 0 : 1);
Assert.Equal(tileColors[tileIndex], result[x, y]);
}
}
Rgba32 opaqueColor = new(7, 11, 13);
using Image<Rgba32> alphaResult = new(outputWidth, outputHeight, opaqueColor);
decoder.DecodeAlphaItemData(
new DecoderOptions { Configuration = Configuration.Default },
gridItem,
descriptor,
alphaResult.Frames.RootFrame,
alphaResult.Size,
new Rectangle(Point.Empty, alphaResult.Size),
false,
TestContext.Current.CancellationToken);
for (int y = 0; y < outputHeight; y++)
{
for (int x = 0; x < outputWidth; x++)
{
int tileIndex = (y < tileHeight ? 0 : 2) + (x < tileWidth ? 0 : 1);
Rgba32 expected = opaqueColor;
expected.A = tileColors[tileIndex].R;
Assert.Equal(expected, alphaResult[x, y]);
}
}
IMemoryOwner<byte> ReadItem(HeifItem item)
{
byte[] payload = payloads[item.Id];
IMemoryOwner<byte> owner = Configuration.Default.MemoryAllocator.Allocate<byte>(payload.Length);
payload.CopyTo(owner.Memory.Span);
return owner;
}
}
[Fact]
public void GridDecoderRejectsCellSmallerThanMiafMinimum()
{
ObuColorConfig colorConfig = new()
{
IsColorDescriptionPresent = true,
ColorPrimaries = ObuColorPrimaries.Bt709,
TransferCharacteristics = ObuTransferCharacteristics.Srgb,
MatrixCoefficients = ObuMatrixCoefficients.Bt709,
ColorRange = true,
IsMonochrome = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
InvalidImageContentException exception = Assert.Throws<InvalidImageContentException>(
() =>
{
using Image<Rgba32> decoded = DecodeSingleCellGrid(63, 64, colorConfig);
});
Assert.Contains("grid cells must be at least 64 samples", exception.Message, StringComparison.Ordinal);
}
/// <summary>
/// Verifies grid dimensions preserve chroma alignment for AV1's 4:2:2 and 4:2:0 layouts.
/// </summary>
[Theory]
[InlineData(65, 64, true, false)]
[InlineData(65, 64, true, true)]
[InlineData(64, 65, true, true)]
public void GridDecoderRejectsOddSubsampledDimension(
int width,
int height,
bool subsamplingX,
bool subsamplingY)
{
ObuColorConfig colorConfig = new()
{
IsColorDescriptionPresent = true,
ColorPrimaries = ObuColorPrimaries.Bt709,
TransferCharacteristics = ObuTransferCharacteristics.Srgb,
MatrixCoefficients = ObuMatrixCoefficients.Bt709,
ColorRange = true,
BitDepth = Av1BitDepth.EightBit,
SubSamplingX = subsamplingX,
SubSamplingY = subsamplingY
};
InvalidImageContentException exception = Assert.Throws<InvalidImageContentException>(
() =>
{
using Image<Rgba32> decoded = DecodeSingleCellGrid(width, height, colorConfig);
});
Assert.Contains("must be even", exception.Message, StringComparison.Ordinal);
}
[Fact]
public void Av1OversizedStillImageWritesAndDecodesGrid()
{
const int width = 65537;
using Image<Rgb24> image = new(width, 1);
image[0, 0] = new Rgb24(1, 2, 3);
image[32768, 0] = new Rgb24(11, 13, 17);
image[32769, 0] = new Rgb24(19, 23, 29);
image[width - 1, 0] = new Rgb24(31, 37, 41);
// Identity-matrix 4:4:4 makes the lossless AV1 cells preserve the packed RGB channels exactly.
image.Metadata.CicpProfile = new CicpProfile(1, 13, 0, true);
using MemoryStream stream = new();
HeifEncoder encoder = new()
{
CompressionMethod = HeifCompressionMethod.Av1,
Lossless = true,
Effort = 0
};
image.Save(stream, encoder);
byte[] file = stream.ToArray();
Assert.Equal(
[0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1],
GetItemPayload(file, 1).ToArray());
using Av1Decoder firstCellDecoder = new(Configuration.Default);
using Image<Rgb24> firstCell = firstCellDecoder.Decode<Rgb24>(GetItemPayload(file, 2));
using Av1Decoder secondCellDecoder = new(Configuration.Default);
using Image<Rgb24> secondCell = secondCellDecoder.Decode<Rgb24>(GetItemPayload(file, 3));
// AVIF requires the first grid cell to be at least 64 samples on both axes. The derived grid trims
// the replicated right and bottom edges back to the presentation encoded in its descriptor.
Assert.Equal(new Size(32769, 64), firstCell.Size);
Assert.Equal(new Size(32769, 64), secondCell.Size);
Assert.Equal(image[32768, 0], firstCell[32768, 0]);
Assert.Equal(image[32769, 0], secondCell[0, 0]);
Assert.Equal(image[width - 1, 0], secondCell[32767, 0]);
Assert.Equal(secondCell[32767, 0], secondCell[32768, 0]);
Assert.Equal(secondCell[0, 0], secondCell[0, 63]);
Assert.Equal(1U, GetItemInfoFlags(file, 2));
Assert.Equal(1U, GetItemInfoFlags(file, 3));
ReadOnlySpan<byte> references = GetMetadataChild(file, Heif4CharCode.Iref);
const int FirstReferenceOffset = 12;
Assert.Equal(
Heif4CharCode.Dimg,
(Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(references[(FirstReferenceOffset + 4)..]));
Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 8)..]));
Assert.Equal(2, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 10)..]));
Assert.Equal(2, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 12)..]));
Assert.Equal(3, BinaryPrimitives.ReadUInt16BigEndian(references[(FirstReferenceOffset + 14)..]));
stream.Position = 0;
using Image<Rgb24> decoded = Image.Load<Rgb24>(stream);
Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded));
} }
[Fact] [Fact]
@ -335,9 +641,13 @@ public class HeifEncoderTests
=> (ushort)(((sample * (long)ushort.MaxValue) + (maximum / 2)) / maximum); => (ushort)(((sample * (long)ushort.MaxValue) + (maximum / 2)) / maximum);
[Theory] [Theory]
[InlineData((ushort)0)] [InlineData((ushort)0, null, (ushort)0)]
[InlineData((ushort)3)] [InlineData((ushort)3, null, (ushort)3)]
public void Av1LosslessImageSequencePreservesFramesTimingAndAlpha(ushort repeatCount) [InlineData((ushort)3, (ushort)7, (ushort)7)]
public void Av1LosslessImageSequencePreservesFramesTimingAndAlpha(
ushort metadataRepeatCount,
ushort? encoderRepeatCount,
ushort expectedRepeatCount)
{ {
const int width = 8; const int width = 8;
const int height = 8; const int height = 8;
@ -376,24 +686,36 @@ public class HeifEncoderTests
image.Metadata.ExifProfile = exifProfile; image.Metadata.ExifProfile = exifProfile;
byte[] xmpData = Encoding.UTF8.GetBytes("<xmp>ImageSharp AV1 sequence</xmp>"); byte[] xmpData = Encoding.UTF8.GetBytes("<xmp>ImageSharp AV1 sequence</xmp>");
image.Metadata.XmpProfile = new XmpProfile(xmpData); image.Metadata.XmpProfile = new XmpProfile(xmpData);
image.Metadata.GetHeifMetadata().RepeatCount = repeatCount; image.Metadata.GetHeifMetadata().RepeatCount = metadataRepeatCount;
using MemoryStream stream = new(); using MemoryStream stream = new();
HeifEncoder encoder = new() HeifEncoder encoder = new()
{ {
CompressionMethod = HeifCompressionMethod.Av1, CompressionMethod = HeifCompressionMethod.Av1,
Lossless = true, Lossless = true,
Effort = 0 Effort = 0,
RepeatCount = encoderRepeatCount
}; };
image.Save(stream, encoder); image.Save(stream, encoder);
byte[] file = stream.ToArray(); byte[] file = stream.ToArray();
Assert.Equal((uint)Heif4CharCode.Avis, BinaryPrimitives.ReadUInt32BigEndian(file.AsSpan(8))); Assert.Equal((uint)Heif4CharCode.Avis, BinaryPrimitives.ReadUInt32BigEndian(file.AsSpan(8)));
Assert.Equal(1, BinaryPrimitives.ReadUInt16BigEndian(GetMetadataChild(file, Heif4CharCode.Pitm)[12..]));
using (Av1Decoder sampleDecoder = new(Configuration.Default))
using (Image<Rgba32> decodedSample = sampleDecoder.Decode<Rgba32>(GetItemPayload(file, 1)))
{
ObuSequenceHeader sampleHeader = sampleDecoder.SequenceHeader;
Assert.NotNull(sampleHeader);
Assert.False(sampleHeader.IsStillPicture);
Assert.False(sampleHeader.IsReducedStillPictureHeader);
}
stream.Position = 0; stream.Position = 0;
DecoderOptions preserveOptions = new() { ColorProfileHandling = ColorProfileHandling.Preserve }; DecoderOptions preserveOptions = new() { ColorProfileHandling = ColorProfileHandling.Preserve };
using Image<Rgba32> decoded = Image.Load<Rgba32>(preserveOptions, stream); using Image<Rgba32> decoded = Image.Load<Rgba32>(preserveOptions, stream);
Assert.Equal(frameCount, decoded.Frames.Count); Assert.Equal(frameCount, decoded.Frames.Count);
Assert.Equal(repeatCount, decoded.Metadata.GetHeifMetadata().RepeatCount); Assert.Equal(expectedRepeatCount, decoded.Metadata.GetHeifMetadata().RepeatCount);
Assert.True(decoded.Metadata.GetHeifMetadata().AnimateRootFrame);
Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded)); Assert.Empty(ImageComparer.Exact.CompareImages(image, decoded));
Assert.Equal( Assert.Equal(
IccTestDataProfiles.ProfileRandomArray, IccTestDataProfiles.ProfileRandomArray,
@ -483,7 +805,7 @@ public class HeifEncoderTests
image.Save(stream, encoder); image.Save(stream, encoder);
byte[] file = stream.ToArray(); byte[] file = stream.ToArray();
ReadOnlySpan<byte> fileType = file.AsSpan(0, 28); ReadOnlySpan<byte> fileType = GetTopLevelBox(file, Heif4CharCode.Ftyp);
Assert.Equal(28, BinaryPrimitives.ReadInt32BigEndian(fileType)); Assert.Equal(28, BinaryPrimitives.ReadInt32BigEndian(fileType));
Assert.Equal(Heif4CharCode.Ftyp, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[4..])); Assert.Equal(Heif4CharCode.Ftyp, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[4..]));
Assert.Equal(Heif4CharCode.Avif, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[8..])); Assert.Equal(Heif4CharCode.Avif, (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(fileType[8..]));
@ -997,6 +1319,49 @@ public class HeifEncoderTests
ipmaPayload[8..].ToArray()); ipmaPayload[8..].ToArray());
} }
[Fact]
public void ItemPropertiesReuseAnEarlierIdenticalCellPropertySet()
{
ObuSequenceHeader sequenceHeader = new()
{
SequenceProfile = ObuSequenceProfile.Main,
OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }],
ColorConfig = new ObuColorConfig
{
BitDepth = Av1BitDepth.EightBit,
SubSamplingX = false,
SubSamplingY = false
}
};
HeifItem firstCell = new(Heif4CharCode.Av01, 1)
{
ChannelBitDepths = [8, 8, 8],
Av1CodecConfiguration = new Av1CodecConfiguration(sequenceHeader)
};
firstCell.SetExtent(new Size(64, 64));
HeifItem secondCell = new(Heif4CharCode.Av01, 2)
{
PropertySource = firstCell
};
secondCell.SetExtent(firstCell.Extent);
List<HeifItem> items = [firstCell, secondCell];
int expectedLength = HeifEncoderCore.GetItemPropertiesBoxLength(items);
using IMemoryOwner<byte> owner = Configuration.Default.MemoryAllocator.Allocate<byte>(expectedLength);
Span<byte> propertyBox = owner.Memory.Span[..expectedLength];
int length = HeifEncoderCore.WriteItemPropertiesBox(propertyBox, 0, items);
Assert.Equal(92, length);
const int IpcoOffset = 8;
int ipmaOffset = IpcoOffset + BinaryPrimitives.ReadInt32BigEndian(propertyBox[IpcoOffset..]);
ReadOnlySpan<byte> ipmaPayload = propertyBox[(ipmaOffset + 8)..];
Assert.Equal(
[0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 3, 1, 2, 0x83, 0, 2, 3, 1, 2, 0x83],
ipmaPayload.ToArray());
}
[Fact] [Fact]
public void Av1ItemPropertiesWriteIccBeforeCicpAndExcludeMetadataItemsFromAssociations() public void Av1ItemPropertiesWriteIccBeforeCicpAndExcludeMetadataItemsFromAssociations()
{ {
@ -1188,6 +1553,87 @@ public class HeifEncoderTests
throw new InvalidImageContentException($"The encoded file has no payload for item {itemId}."); throw new InvalidImageContentException($"The encoded file has no payload for item {itemId}.");
} }
private static uint GetItemInfoFlags(Span<byte> file, ushort itemId)
{
ReadOnlySpan<byte> itemInformation = GetMetadataChild(file, Heif4CharCode.Iinf);
int entryOffset = 14;
while (entryOffset < itemInformation.Length)
{
int entrySize = BinaryPrimitives.ReadInt32BigEndian(itemInformation[entryOffset..]);
ReadOnlySpan<byte> entry = itemInformation.Slice(entryOffset, entrySize);
ushort currentItemId = BinaryPrimitives.ReadUInt16BigEndian(entry[12..]);
if (currentItemId == itemId)
{
return (uint)((entry[9] << 16) | (entry[10] << 8) | entry[11]);
}
entryOffset += entrySize;
}
throw new InvalidImageContentException($"The encoded file has no item-information entry for item {itemId}.");
}
private static Image<Rgba32> DecodeSingleCellGrid(int width, int height, ObuColorConfig colorConfig)
{
using Image<Rgba32> tile = new(width, height, new Rgba32(127, 127, 127));
using MemoryStream payloadStream = new();
ObuSequenceHeader header = Av1FrameEncoder.Encode(
Configuration.Default,
tile.Frames.RootFrame,
payloadStream,
colorConfig,
qIndex: 0,
effort: 0);
byte[] payload = payloadStream.ToArray();
HeifItem gridItem = new(Heif4CharCode.Grid, 1);
gridItem.SetExtent(new Size(width, height));
HeifItem tileItem = new(Heif4CharCode.Av01, 2)
{
Av1CodecConfiguration = new Av1CodecConfiguration(header)
};
tileItem.SetExtent(new Size(width, height));
HeifItemLink gridLink = new(Heif4CharCode.Dimg, gridItem.Id);
gridLink.DestinationIds.Add(tileItem.Id);
GridHeifItemDecoder<Rgba32> decoder = new([gridItem, tileItem], [gridLink], ReadItem);
byte[] descriptor = new byte[8];
BinaryPrimitives.WriteUInt16BigEndian(descriptor.AsSpan(4), (ushort)width);
BinaryPrimitives.WriteUInt16BigEndian(descriptor.AsSpan(6), (ushort)height);
return decoder.DecodeItemData(
new DecoderOptions { Configuration = Configuration.Default },
gridItem,
descriptor,
null,
TestContext.Current.CancellationToken);
IMemoryOwner<byte> ReadItem(HeifItem item)
{
Assert.Equal(tileItem.Id, item.Id);
IMemoryOwner<byte> owner = Configuration.Default.MemoryAllocator.Allocate<byte>(payload.Length);
payload.CopyTo(owner.Memory.Span);
return owner;
}
}
private static Span<byte> GetTopLevelBox(Span<byte> file, Heif4CharCode requestedType)
{
int offset = 0;
while (offset < file.Length)
{
int boxSize = BinaryPrimitives.ReadInt32BigEndian(file[offset..]);
Heif4CharCode boxType = (Heif4CharCode)BinaryPrimitives.ReadUInt32BigEndian(file[(offset + sizeof(uint))..]);
if (boxType == requestedType)
{
return file.Slice(offset, boxSize);
}
offset += boxSize;
}
throw new InvalidImageContentException($"The encoded file has no {requestedType} top-level box.");
}
private static Span<byte> GetMetadataChild(Span<byte> file, Heif4CharCode childType) private static Span<byte> GetMetadataChild(Span<byte> file, Heif4CharCode childType)
{ {
int offset = 0; int offset = 0;

Loading…
Cancel
Save