Browse Source

Integrate AV1 motion search with retained frame decisions

pull/2633/head
James Jackson-South 4 weeks ago
parent
commit
e94848c356
  1. 1139
      HEIF_IMPLEMENTATION_PLAN.md
  2. 22
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs
  3. 11
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs
  4. 22
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs
  5. 135
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  6. 3
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  7. 235
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs
  8. 621
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs
  9. 22
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs
  10. 175
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs
  11. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs
  12. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs
  13. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs
  14. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs
  15. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs
  16. 2
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs
  17. 276
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterBase.cs
  18. 141
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterEncoder.cs
  19. 26
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderDisplacementVector.cs
  20. 50
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs
  21. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderTransformBlockState.cs
  22. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EntropyCodingContext.cs
  23. 52
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs
  24. 26
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs
  25. 101
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.BlockEncoding.cs
  26. 612
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs
  27. 24
      src/ImageSharp/Formats/Heif/HeifEncoder.cs
  28. 6
      src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs
  29. 13
      tests/ImageSharp.Benchmarks/Codecs/Heif/Av1SequenceEncoderBenchmarks.cs
  30. 169
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs
  31. 84
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs
  32. 108
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs
  33. 640
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs
  34. 16
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs

1139
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

22
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs

@ -39,6 +39,7 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
private const int ResidualStorageLength = MaximumResidualCount / 2; private const int ResidualStorageLength = MaximumResidualCount / 2;
private const int MotionSearchSiteCount = 6; private const int MotionSearchSiteCount = 6;
private const int MotionSearchPredictionSampleCount = 128 * (128 + 8);
private const int MotionSearchSiteStorageOffset = StorageLength + Av1MotionVectorCosts.StorageLength; private const int MotionSearchSiteStorageOffset = StorageLength + Av1MotionVectorCosts.StorageLength;
private const int TransformCoefficientOffset = ResidualStorageLength; private const int TransformCoefficientOffset = ResidualStorageLength;
private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount; private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount;
@ -80,9 +81,12 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
InterPredictionCoefficientStorageLength; InterPredictionCoefficientStorageLength;
private const int ModeDecisionStorageLength = Av1EncoderModeDecisionWorkspace<ushort>.StorageLength; private const int ModeDecisionStorageLength = Av1EncoderModeDecisionWorkspace<ushort>.StorageLength;
private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > InterPredictionStorageLength private const int InterSearchStorageLength =
InterPredictionStorageLength + (MotionSearchPredictionSampleCount * sizeof(ushort) / sizeof(int));
private const int SharedModeDecisionStorageLength = ModeDecisionStorageLength > InterSearchStorageLength
? ModeDecisionStorageLength ? ModeDecisionStorageLength
: InterPredictionStorageLength; : InterSearchStorageLength;
private const int PartitionContextStorageOffset = private const int PartitionContextStorageOffset =
InterPredictionSampleStorageOffset + SharedModeDecisionStorageLength; InterPredictionSampleStorageOffset + SharedModeDecisionStorageLength;
@ -186,6 +190,20 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
public Av1MotionVectorCosts GetMotionVectorCosts(Av1MotionVectorPrecision precision) public Av1MotionVectorCosts GetMotionVectorCosts(Av1MotionVectorPrecision precision)
=> new(this.owner.Memory.Span.Slice(StorageLength, Av1MotionVectorCosts.StorageLength), precision); => new(this.owner.Memory.Span.Slice(StorageLength, Av1MotionVectorCosts.StorageLength), precision);
/// <summary>
/// Borrows prediction samples for motion search while retaining the selected inter reconstruction.
/// </summary>
/// <typeparam name="TSample">The frame's unsigned sample storage type.</typeparam>
/// <returns>The reusable search prediction span.</returns>
public Span<TSample> GetMotionSearchPrediction<TSample>()
where TSample : unmanaged
{
// Intra trials have finished before inter search starts. Reuse their storage beyond the live inter
// candidate buffers; the extra eight rows accommodate separable filtering of a 128x128 prediction.
int offset = InterPredictionSampleStorageOffset + InterPredictionStorageLength;
return MemoryMarshal.Cast<int, TSample>(this.owner.Memory.Span[offset..])[..MotionSearchPredictionSampleCount];
}
/// <summary> /// <summary>
/// Gets the retained full-pixel search geometry for the reference plane's current stride. /// Gets the retained full-pixel search geometry for the reference plane's current stride.
/// </summary> /// </summary>

11
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrame.cs

@ -184,22 +184,23 @@ internal readonly struct Av1EncoderFrame<TSample>
Av1Math.AlignPowerOf2(height, CodedDimensionAlignmentLog2)); Av1Math.AlignPowerOf2(height, CodedDimensionAlignmentLog2));
/// <summary> /// <summary>
/// Calculates the physical dimensions required for an all-intra component plane. /// Calculates the physical dimensions required for a bordered component plane.
/// </summary> /// </summary>
/// <param name="width">The visible luma width.</param> /// <param name="width">The visible luma width.</param>
/// <param name="height">The visible luma height.</param> /// <param name="height">The visible luma height.</param>
/// <param name="subsamplingX">The plane's horizontal subsampling shift.</param> /// <param name="subsamplingX">The plane's horizontal subsampling shift.</param>
/// <param name="subsamplingY">The plane's vertical subsampling shift.</param> /// <param name="subsamplingY">The plane's vertical subsampling shift.</param>
/// <param name="lumaBorder">The border width and height in luma samples.</param>
/// <returns>The physical plane dimensions, including its complete border and row padding.</returns> /// <returns>The physical plane dimensions, including its complete border and row padding.</returns>
public static Size GetPlaneBufferSize(int width, int height, int subsamplingX, int subsamplingY) public static Size GetPlaneBufferSize(int width, int height, int subsamplingX, int subsamplingY, int lumaBorder)
{ {
Size codedSize = GetCodedSize(width, height); Size codedSize = GetCodedSize(width, height);
// libaom aligns the complete luma row before deriving a subsampled plane's stride. // Align the complete luma row before deriving a subsampled plane's stride.
// Aligning chroma independently would produce a different physical layout for narrow or odd-sized frames. // Aligning chroma independently would produce a different physical layout for narrow or odd-sized frames.
int lumaStride = Av1Math.AlignPowerOf2(codedSize.Width + (2 * LumaBorder), LumaStrideAlignmentLog2); int lumaStride = Av1Math.AlignPowerOf2(codedSize.Width + (2 * lumaBorder), LumaStrideAlignmentLog2);
int planeStride = lumaStride >> subsamplingX; int planeStride = lumaStride >> subsamplingX;
int planeBorderHeight = LumaBorder >> subsamplingY; int planeBorderHeight = lumaBorder >> subsamplingY;
return new Size(planeStride, (codedSize.Height >> subsamplingY) + (2 * planeBorderHeight)); return new Size(planeStride, (codedSize.Height >> subsamplingY) + (2 * planeBorderHeight));
} }

22
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderFrameBuffer.cs

@ -15,7 +15,7 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary> /// <summary>
/// The byte boundary used by libaom for SIMD-accessible component planes. /// The byte boundary used for SIMD-accessible component planes.
/// </summary> /// </summary>
private const int PlaneAlignmentBytes = 32; private const int PlaneAlignmentBytes = 32;
@ -34,6 +34,7 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
/// <param name="colorFormat">The native luma and chroma sampling layout.</param> /// <param name="colorFormat">The native luma and chroma sampling layout.</param>
/// <param name="chromaPositionX">The horizontal chroma position in half-luma-sample units.</param> /// <param name="chromaPositionX">The horizontal chroma position in half-luma-sample units.</param>
/// <param name="chromaPositionY">The vertical chroma position in half-luma-sample units.</param> /// <param name="chromaPositionY">The vertical chroma position in half-luma-sample units.</param>
/// <param name="lumaBorder">The border width and height in luma samples.</param>
public Av1EncoderFrameBuffer( public Av1EncoderFrameBuffer(
Configuration configuration, Configuration configuration,
int width, int width,
@ -41,16 +42,17 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
int bitDepth, int bitDepth,
Av1ColorFormat colorFormat, Av1ColorFormat colorFormat,
int chromaPositionX, int chromaPositionX,
int chromaPositionY) int chromaPositionY,
int lumaBorder)
{ {
int subsamplingX = colorFormat is Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422 ? 1 : 0; int subsamplingX = colorFormat is Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422 ? 1 : 0;
int subsamplingY = colorFormat == Av1ColorFormat.Yuv420 ? 1 : 0; int subsamplingY = colorFormat == Av1ColorFormat.Yuv420 ? 1 : 0;
Size codedSize = Av1EncoderFrame<TSample>.GetCodedSize(width, height); Size codedSize = Av1EncoderFrame<TSample>.GetCodedSize(width, height);
Size lumaSize = Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, 0, 0); Size lumaSize = Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, 0, 0, lumaBorder);
int lumaElementCount = checked(lumaSize.Width * lumaSize.Height); int lumaElementCount = checked(lumaSize.Width * lumaSize.Height);
Size chromaSize = colorFormat == Av1ColorFormat.Yuv400 Size chromaSize = colorFormat == Av1ColorFormat.Yuv400
? Size.Empty ? Size.Empty
: Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY); : Av1EncoderFrame<TSample>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY, lumaBorder);
int chromaElementCount = checked(chromaSize.Width * chromaSize.Height); int chromaElementCount = checked(chromaSize.Width * chromaSize.Height);
int planeAlignment = Math.Max(PlaneAlignmentBytes / Unsafe.SizeOf<TSample>(), 1); int planeAlignment = Math.Max(PlaneAlignmentBytes / Unsafe.SizeOf<TSample>(), 1);
@ -60,8 +62,8 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
? lumaElementCount ? lumaElementCount
: checked(chromaRedOffset + chromaElementCount); : checked(chromaRedOffset + chromaElementCount);
// Libaom keeps the three component planes in one 32-byte-aligned frame allocation. The non-owning // Component planes share one frame allocation; their offsets preserve the 32-byte plane alignment.
// Buffer2D views preserve ImageSharp's row API without introducing separate plane rents or copies. // Non-owning Buffer2D views expose rows without introducing separate plane rents or copies.
IMemoryOwner<TSample> owner = configuration.MemoryAllocator.Allocate<TSample>(storageLength); IMemoryOwner<TSample> owner = configuration.MemoryAllocator.Allocate<TSample>(storageLength);
Memory<TSample> storage = owner.Memory; Memory<TSample> storage = owner.Memory;
Buffer2D<TSample> luma = Buffer2D<TSample>.WrapMemory( Buffer2D<TSample> luma = Buffer2D<TSample>.WrapMemory(
@ -72,8 +74,8 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
this.Luma = luma; this.Luma = luma;
Buffer2DRegion<TSample> lumaRegion = luma.GetRegion( Buffer2DRegion<TSample> lumaRegion = luma.GetRegion(
Av1EncoderFrame<TSample>.LumaBorder, lumaBorder,
Av1EncoderFrame<TSample>.LumaBorder, lumaBorder,
codedSize.Width, codedSize.Width,
codedSize.Height); codedSize.Height);
@ -94,8 +96,8 @@ internal sealed class Av1EncoderFrameBuffer<TSample> : IDisposable
this.ChromaBlue = chromaBlue; this.ChromaBlue = chromaBlue;
this.ChromaRed = chromaRed; this.ChromaRed = chromaRed;
int chromaBorderX = Av1EncoderFrame<TSample>.LumaBorder >> subsamplingX; int chromaBorderX = lumaBorder >> subsamplingX;
int chromaBorderY = Av1EncoderFrame<TSample>.LumaBorder >> subsamplingY; int chromaBorderY = lumaBorder >> subsamplingY;
int codedChromaWidth = codedSize.Width >> subsamplingX; int codedChromaWidth = codedSize.Width >> subsamplingX;
int codedChromaHeight = codedSize.Height >> subsamplingY; int codedChromaHeight = codedSize.Height >> subsamplingY;
chromaBlueRegion = chromaBlue.GetRegion( chromaBlueRegion = chromaBlue.GetRegion(

135
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs

@ -272,8 +272,9 @@ internal static class Av1FrameEncoder
int height, int height,
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort) int effort,
=> CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, false); HeifEncodingSpeed speed)
=> CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, speed, false);
/// <summary> /// <summary>
/// Creates an encoder that retains reconstructed alpha frames for prediction by later samples in the sequence. /// Creates an encoder that retains reconstructed alpha frames for prediction by later samples in the sequence.
@ -284,8 +285,9 @@ internal static class Av1FrameEncoder
int height, int height,
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort) int effort,
=> CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, true); HeifEncodingSpeed speed)
=> CreateSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, speed, true);
private static ObuSequenceHeader Encode<TPixel>( private static ObuSequenceHeader Encode<TPixel>(
Configuration configuration, Configuration configuration,
@ -359,14 +361,15 @@ internal static class Av1FrameEncoder
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort, int effort,
HeifEncodingSpeed speed,
bool encodeAlpha) bool encodeAlpha)
{ {
if (colorConfig.BitDepth == Av1BitDepth.EightBit) if (colorConfig.BitDepth == Av1BitDepth.EightBit)
{ {
return new ByteSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, encodeAlpha); return new ByteSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, speed, encodeAlpha);
} }
return new HighBitDepthSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, encodeAlpha); return new HighBitDepthSequenceEncoder(configuration, width, height, colorConfig, qIndex, effort, speed, encodeAlpha);
} }
private static ObuSequenceHeader CreateSequenceHeader( private static ObuSequenceHeader CreateSequenceHeader(
@ -622,7 +625,8 @@ internal static class Av1FrameEncoder
ByteSampleBitDepth, ByteSampleBitDepth,
colorFormat, colorFormat,
chromaPositionX: CenteredChromaSamplePosition, chromaPositionX: CenteredChromaSamplePosition,
chromaPositionY: CenteredChromaSamplePosition); chromaPositionY: CenteredChromaSamplePosition,
lumaBorder: Av1EncoderFrame<byte>.LumaBorder);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
configuration, configuration,
@ -631,7 +635,8 @@ internal static class Av1FrameEncoder
ByteSampleBitDepth, ByteSampleBitDepth,
colorFormat, colorFormat,
chromaPositionX: CenteredChromaSamplePosition, chromaPositionX: CenteredChromaSamplePosition,
chromaPositionY: CenteredChromaSamplePosition); chromaPositionY: CenteredChromaSamplePosition,
lumaBorder: Av1EncoderFrame<byte>.LumaBorder);
using Av1EncoderCoefficientBuffer coefficients = new( using Av1EncoderCoefficientBuffer coefficients = new(
configuration, configuration,
@ -651,7 +656,7 @@ internal static class Av1FrameEncoder
using ObuWriter obuWriter = new(configuration); using ObuWriter obuWriter = new(configuration);
PrepareFrame( bool isScreenContent = PrepareFrame(
configuration, configuration,
image, image,
sourceRectangle, sourceRectangle,
@ -670,6 +675,7 @@ internal static class Av1FrameEncoder
source.Frame.Height, source.Frame.Height,
disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9); disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9);
picture.Picture.Parent.IsScreenContent = isScreenContent;
Encode( Encode(
obuWriter, obuWriter,
stream, stream,
@ -709,7 +715,8 @@ internal static class Av1FrameEncoder
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaPositionX: CenteredChromaSamplePosition, chromaPositionX: CenteredChromaSamplePosition,
chromaPositionY: CenteredChromaSamplePosition); chromaPositionY: CenteredChromaSamplePosition,
lumaBorder: Av1EncoderFrame<ushort>.LumaBorder);
using Av1EncoderFrameBuffer<ushort> reconstruction = new( using Av1EncoderFrameBuffer<ushort> reconstruction = new(
configuration, configuration,
@ -718,7 +725,8 @@ internal static class Av1FrameEncoder
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaPositionX: CenteredChromaSamplePosition, chromaPositionX: CenteredChromaSamplePosition,
chromaPositionY: CenteredChromaSamplePosition); chromaPositionY: CenteredChromaSamplePosition,
lumaBorder: Av1EncoderFrame<ushort>.LumaBorder);
using Av1EncoderCoefficientBuffer coefficients = new( using Av1EncoderCoefficientBuffer coefficients = new(
configuration, configuration,
@ -738,7 +746,7 @@ internal static class Av1FrameEncoder
using ObuWriter obuWriter = new(configuration); using ObuWriter obuWriter = new(configuration);
PrepareFrame( bool isScreenContent = PrepareFrame(
configuration, configuration,
image, image,
sourceRectangle, sourceRectangle,
@ -757,6 +765,7 @@ internal static class Av1FrameEncoder
source.Frame.Height, source.Frame.Height,
disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9); disallow4x4AllFrames: !frameHeader.CodedLossless && effort < 9);
picture.Picture.Parent.IsScreenContent = isScreenContent;
Encode( Encode(
obuWriter, obuWriter,
stream, stream,
@ -777,7 +786,7 @@ internal static class Av1FrameEncoder
/// <summary> /// <summary>
/// Converts one source frame and resolves every content-dependent coding tool before picture-state allocation. /// Converts one source frame and resolves every content-dependent coding tool before picture-state allocation.
/// </summary> /// </summary>
private static void PrepareFrame<TPixel>( private static bool PrepareFrame<TPixel>(
Configuration configuration, Configuration configuration,
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
Rectangle sourceRectangle, Rectangle sourceRectangle,
@ -797,7 +806,7 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig, sequenceHeader.ColorConfig,
encodeAlpha); encodeAlpha);
ConfigureFrameTools( return ConfigureFrameTools(
source, source,
reference, reference,
sequenceHeader, sequenceHeader,
@ -808,7 +817,7 @@ internal static class Av1FrameEncoder
/// <summary> /// <summary>
/// Converts one sequence sample through its retained row workspace before resolving frame coding tools. /// Converts one sequence sample through its retained row workspace before resolving frame coding tools.
/// </summary> /// </summary>
private static void PrepareFrame<TPixel>( private static bool PrepareFrame<TPixel>(
Configuration configuration, Configuration configuration,
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
Rectangle sourceRectangle, Rectangle sourceRectangle,
@ -827,7 +836,7 @@ internal static class Av1FrameEncoder
source, source,
conversionWorkspace); conversionWorkspace);
ConfigureFrameTools( return ConfigureFrameTools(
source, source,
reference, reference,
sequenceHeader, sequenceHeader,
@ -838,7 +847,7 @@ internal static class Av1FrameEncoder
/// <summary> /// <summary>
/// Resolves the eight-bit frame tools whose syntax depends on the converted source samples. /// Resolves the eight-bit frame tools whose syntax depends on the converted source samples.
/// </summary> /// </summary>
private static void ConfigureFrameTools( private static bool ConfigureFrameTools(
Av1EncoderFrame<byte> source, Av1EncoderFrame<byte> source,
Av1EncoderFrame<byte> reference, Av1EncoderFrame<byte> reference,
ObuSequenceHeader sequenceHeader, ObuSequenceHeader sequenceHeader,
@ -852,32 +861,28 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig.BitDepth, sequenceHeader.ColorConfig.BitDepth,
effort); effort);
bool allowScreenContentTools = false; bool isScreenContent = Av1ScreenContentDetector.Detect(
bool allowIntraBlockCopy = false; source,
if (effort >= 5) out bool allowScreenContentTools,
{ out bool allowIntraBlockCopy);
// Lower effort levels never search palette or intra-block-copy modes, so scanning the complete
// luma plane cannot affect their bitstream decisions.
Av1ScreenContentDetector.Detect(
source,
out allowScreenContentTools,
out allowIntraBlockCopy);
}
frameHeader.AllowScreenContentTools = allowScreenContentTools; frameHeader.AllowScreenContentTools = effort >= 5 && allowScreenContentTools;
// The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible // The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible
// 4x4 transforms, so palette remains available while this incompatible candidate is omitted. // 4x4 transforms, so palette remains available while this incompatible candidate is omitted.
frameHeader.AllowIntraBlockCopy = frameHeader.AllowIntraBlockCopy =
frameHeader.IsIntra && frameHeader.IsIntra &&
!frameHeader.CodedLossless && !frameHeader.CodedLossless &&
frameHeader.AllowScreenContentTools &&
allowIntraBlockCopy; allowIntraBlockCopy;
return isScreenContent;
} }
/// <summary> /// <summary>
/// Converts one high-bit-depth source frame and resolves every content-dependent coding tool before picture-state allocation. /// Converts one high-bit-depth source frame and resolves every content-dependent coding tool before picture-state allocation.
/// </summary> /// </summary>
private static void PrepareFrame<TPixel>( private static bool PrepareFrame<TPixel>(
Configuration configuration, Configuration configuration,
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
Rectangle sourceRectangle, Rectangle sourceRectangle,
@ -897,7 +902,7 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig, sequenceHeader.ColorConfig,
encodeAlpha); encodeAlpha);
ConfigureFrameTools( return ConfigureFrameTools(
source, source,
reference, reference,
sequenceHeader, sequenceHeader,
@ -908,7 +913,7 @@ internal static class Av1FrameEncoder
/// <summary> /// <summary>
/// Converts one high-bit-depth sequence sample through retained row storage before resolving frame coding tools. /// Converts one high-bit-depth sequence sample through retained row storage before resolving frame coding tools.
/// </summary> /// </summary>
private static void PrepareFrame<TPixel>( private static bool PrepareFrame<TPixel>(
Configuration configuration, Configuration configuration,
ImageFrame<TPixel> image, ImageFrame<TPixel> image,
Rectangle sourceRectangle, Rectangle sourceRectangle,
@ -927,7 +932,7 @@ internal static class Av1FrameEncoder
source, source,
conversionWorkspace); conversionWorkspace);
ConfigureFrameTools( return ConfigureFrameTools(
source, source,
reference, reference,
sequenceHeader, sequenceHeader,
@ -938,7 +943,7 @@ internal static class Av1FrameEncoder
/// <summary> /// <summary>
/// Resolves the high-bit-depth frame tools whose syntax depends on the converted source samples. /// Resolves the high-bit-depth frame tools whose syntax depends on the converted source samples.
/// </summary> /// </summary>
private static void ConfigureFrameTools( private static bool ConfigureFrameTools(
Av1EncoderFrame<ushort> source, Av1EncoderFrame<ushort> source,
Av1EncoderFrame<ushort> reference, Av1EncoderFrame<ushort> reference,
ObuSequenceHeader sequenceHeader, ObuSequenceHeader sequenceHeader,
@ -952,26 +957,22 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig.BitDepth, sequenceHeader.ColorConfig.BitDepth,
effort); effort);
bool allowScreenContentTools = false; bool isScreenContent = Av1ScreenContentDetector.Detect(
bool allowIntraBlockCopy = false; source,
if (effort >= 5) out bool allowScreenContentTools,
{ out bool allowIntraBlockCopy);
// Lower effort levels never search palette or intra-block-copy modes, so scanning the complete
// luma plane cannot affect their bitstream decisions.
Av1ScreenContentDetector.Detect(
source,
out allowScreenContentTools,
out allowIntraBlockCopy);
}
frameHeader.AllowScreenContentTools = allowScreenContentTools; frameHeader.AllowScreenContentTools = effort >= 5 && allowScreenContentTools;
// The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible // The current intra-block-copy search owns one 8x8 transform. Lossless coding requires reversible
// 4x4 transforms, so palette remains available while this incompatible candidate is omitted. // 4x4 transforms, so palette remains available while this incompatible candidate is omitted.
frameHeader.AllowIntraBlockCopy = frameHeader.AllowIntraBlockCopy =
frameHeader.IsIntra && frameHeader.IsIntra &&
!frameHeader.CodedLossless && !frameHeader.CodedLossless &&
frameHeader.AllowScreenContentTools &&
allowIntraBlockCopy; allowIntraBlockCopy;
return isScreenContent;
} }
private static void Encode( private static void Encode(
@ -1166,7 +1167,7 @@ internal static class Av1FrameEncoder
int effortShift = effort - MinimumGlobalMotionSearchEffort; int effortShift = effort - MinimumGlobalMotionSearchEffort;
int searchRadius = Math.Min( int searchRadius = Math.Min(
MinimumGlobalMotionSearchRadius << effortShift, MinimumGlobalMotionSearchRadius << effortShift,
Av1EncoderFrame<TSample>.LumaBorder); Math.Min(referenceLuma.Bounds.X, referenceLuma.Bounds.Y));
Point bestOffset = default; Point bestOffset = default;
long bestAnalysisError = GetGlobalMotionSquaredError<TSample, TOperator>( long bestAnalysisError = GetGlobalMotionSquaredError<TSample, TOperator>(
@ -1499,6 +1500,7 @@ internal static class Av1FrameEncoder
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort, int effort,
HeifEncodingSpeed speed,
bool encodeAlpha, bool encodeAlpha,
bool usesHighBitDepth) bool usesHighBitDepth)
{ {
@ -1548,10 +1550,11 @@ internal static class Av1FrameEncoder
width, width,
height); height);
this.PictureBuffer.Picture.Parent.EncodingSpeed = speed;
this.SuperblockWorkspace = new Av1EncoderSuperblockWorkspace(configuration); this.SuperblockWorkspace = new Av1EncoderSuperblockWorkspace(configuration);
this.TileWorkspace = new Av1EncoderTileWorkspace(this.FrameHeader, this.SuperblockWorkspace); this.TileWorkspace = new Av1EncoderTileWorkspace(this.FrameHeader, this.SuperblockWorkspace);
this.BlockWorkspace = new Av1EncoderBlockWorkspace(configuration); this.BlockWorkspace = new Av1EncoderBlockWorkspace(configuration, allocateInterMotionCosts: true);
// Tile probabilities adapt within a sample, while error-resilient frame headers prohibit carrying // Tile probabilities adapt within a sample, while error-resilient frame headers prohibit carrying
// those updates into the next sample. The retained encoder is therefore reset before each frame. // those updates into the next sample. The retained encoder is therefore reset before each frame.
@ -1692,6 +1695,7 @@ internal static class Av1FrameEncoder
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort, int effort,
HeifEncodingSpeed speed,
bool encodeAlpha) bool encodeAlpha)
: base( : base(
configuration, configuration,
@ -1700,12 +1704,17 @@ internal static class Av1FrameEncoder
colorConfig, colorConfig,
qIndex, qIndex,
effort, effort,
speed,
encodeAlpha, encodeAlpha,
usesHighBitDepth: false) usesHighBitDepth: false)
{ {
try try
{ {
Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); Av1ColorFormat colorFormat = colorConfig.GetColorFormat();
// Inter prediction needs a complete superblock beyond the image plus interpolation and alignment margins.
int lumaBorder = (this.SequenceHeader.Use128x128Superblock ? 128 : 64) + 32;
this.source = new( this.source = new(
configuration, configuration,
width, width,
@ -1713,7 +1722,8 @@ internal static class Av1FrameEncoder
ByteSampleBitDepth, ByteSampleBitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
this.reference = new( this.reference = new(
configuration, configuration,
@ -1722,7 +1732,8 @@ internal static class Av1FrameEncoder
ByteSampleBitDepth, ByteSampleBitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
this.reconstruction = new( this.reconstruction = new(
configuration, configuration,
@ -1731,7 +1742,8 @@ internal static class Av1FrameEncoder
ByteSampleBitDepth, ByteSampleBitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
} }
catch catch
{ {
@ -1764,7 +1776,7 @@ internal static class Av1FrameEncoder
Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); Rectangle sourceRectangle = new(0, 0, image.Width, image.Height);
this.SymbolEncoder.Reset(); this.SymbolEncoder.Reset();
PrepareFrame( bool isScreenContent = PrepareFrame(
this.Configuration, this.Configuration,
image, image,
sourceRectangle, sourceRectangle,
@ -1776,6 +1788,7 @@ internal static class Av1FrameEncoder
this.ConversionWorkspace); this.ConversionWorkspace);
this.PictureBuffer.Reset(frameHeader); this.PictureBuffer.Reset(frameHeader);
this.PictureBuffer.Picture.Parent.IsScreenContent = isScreenContent;
Encode( Encode(
this.ObuWriter, this.ObuWriter,
stream, stream,
@ -1812,6 +1825,7 @@ internal static class Av1FrameEncoder
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
int qIndex, int qIndex,
int effort, int effort,
HeifEncodingSpeed speed,
bool encodeAlpha) bool encodeAlpha)
: base( : base(
configuration, configuration,
@ -1820,6 +1834,7 @@ internal static class Av1FrameEncoder
colorConfig, colorConfig,
qIndex, qIndex,
effort, effort,
speed,
encodeAlpha, encodeAlpha,
usesHighBitDepth: true) usesHighBitDepth: true)
{ {
@ -1827,6 +1842,10 @@ internal static class Av1FrameEncoder
{ {
int bitDepth = colorConfig.BitDepth.GetBitCount(); int bitDepth = colorConfig.BitDepth.GetBitCount();
Av1ColorFormat colorFormat = colorConfig.GetColorFormat(); Av1ColorFormat colorFormat = colorConfig.GetColorFormat();
// Inter prediction needs a complete superblock beyond the image plus interpolation and alignment margins.
int lumaBorder = (this.SequenceHeader.Use128x128Superblock ? 128 : 64) + 32;
this.source = new( this.source = new(
configuration, configuration,
width, width,
@ -1834,7 +1853,8 @@ internal static class Av1FrameEncoder
bitDepth, bitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
this.reference = new( this.reference = new(
configuration, configuration,
@ -1843,7 +1863,8 @@ internal static class Av1FrameEncoder
bitDepth, bitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
this.reconstruction = new( this.reconstruction = new(
configuration, configuration,
@ -1852,7 +1873,8 @@ internal static class Av1FrameEncoder
bitDepth, bitDepth,
colorFormat, colorFormat,
CenteredChromaSamplePosition, CenteredChromaSamplePosition,
CenteredChromaSamplePosition); CenteredChromaSamplePosition,
lumaBorder);
} }
catch catch
{ {
@ -1885,7 +1907,7 @@ internal static class Av1FrameEncoder
Rectangle sourceRectangle = new(0, 0, image.Width, image.Height); Rectangle sourceRectangle = new(0, 0, image.Width, image.Height);
this.SymbolEncoder.Reset(); this.SymbolEncoder.Reset();
PrepareFrame( bool isScreenContent = PrepareFrame(
this.Configuration, this.Configuration,
image, image,
sourceRectangle, sourceRectangle,
@ -1897,6 +1919,7 @@ internal static class Av1FrameEncoder
this.ConversionWorkspace); this.ConversionWorkspace);
this.PictureBuffer.Reset(frameHeader); this.PictureBuffer.Reset(frameHeader);
this.PictureBuffer.Picture.Parent.IsScreenContent = isScreenContent;
Encode( Encode(
this.ObuWriter, this.ObuWriter,
stream, stream,

3
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs

@ -180,6 +180,9 @@ internal static partial class Av1IntraSuperblockEncoder
this.SelectedBlockStatistics = default; this.SelectedBlockStatistics = default;
} }
/// <inheritdoc/>
public static bool UsesRetainedDecisions => false;
/// <summary> /// <summary>
/// Gets the statistics of the most recently encoded block. /// Gets the statistics of the most recently encoded block.
/// </summary> /// </summary>

235
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs

@ -19,16 +19,13 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// </content> /// </content>
internal static partial class Av1IntraSuperblockEncoder internal static partial class Av1IntraSuperblockEncoder
{ {
/// <summary>
/// The width and height of the fixed block currently used by inter motion search.
/// </summary>
private const int InterSearchBlockDimension = 8;
/// <summary> /// <summary>
/// Defines type-specific block encoding without coupling traversal to sample storage width. /// Defines type-specific block encoding without coupling traversal to sample storage width.
/// </summary> /// </summary>
/// <typeparam name="TSample">The native unsigned sample storage type.</typeparam> /// <typeparam name="TSample">The native unsigned sample storage type.</typeparam>
internal interface IBlockEncodingOperator<TSample> : Av1IntraBlockCopySearchIndex.ISearchOperation<TSample> internal interface IBlockEncodingOperator<TSample> :
Av1IntraBlockCopySearchIndex.ISearchOperation<TSample>,
Av1MotionSearchBase.IMotionSearchOperator<TSample>
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary> /// <summary>
@ -330,22 +327,6 @@ internal static partial class Av1IntraSuperblockEncoder
Av1TransformSize transformSize, Av1TransformSize transformSize,
Av1BitDepth bitDepth); Av1BitDepth bitDepth);
/// <summary>
/// Measures an 8x8 full-pixel reference candidate through the bordered plane storage.
/// </summary>
/// <param name="source">The coded source plane.</param>
/// <param name="sourceOrigin">The source block origin in visible-plane coordinates.</param>
/// <param name="reference">The padded retained reference plane.</param>
/// <param name="predictionOrigin">The candidate origin, which may lie inside the physical border.</param>
/// <param name="bitDepth">The coded sample precision.</param>
/// <returns>The squared error normalized to the eight-bit distortion domain.</returns>
public static abstract long GetInterPredictionError(
Buffer2DRegion<TSample> source,
Point sourceOrigin,
Buffer2DRegion<TSample> reference,
Point predictionOrigin,
Av1BitDepth bitDepth);
/// <summary> /// <summary>
/// Encodes one prepared prediction with the selected transform into decision scratch. /// Encodes one prepared prediction with the selected transform into decision scratch.
/// </summary> /// </summary>
@ -442,6 +423,80 @@ internal static partial class Av1IntraSuperblockEncoder
/// </summary> /// </summary>
internal readonly struct ByteOperator : IBlockEncodingOperator<byte> internal readonly struct ByteOperator : IBlockEncodingOperator<byte>
{ {
/// <inheritdoc/>
public static void PreparePrediction(
ReadOnlySpan<byte> source,
int sourceStride,
ReadOnlySpan<byte> reference,
int referenceStride,
int referenceOrigin,
Span<byte> prediction,
Span<short> residual,
Span<short> scratch,
int width,
int height,
Av1InterpolationFilter horizontalFilter,
Av1InterpolationFilter verticalFilter,
int horizontalPhase,
int verticalPhase,
int bitDepth)
=> Av1MotionSearchBase.ByteOperator.PreparePrediction(
source,
sourceStride,
reference,
referenceStride,
referenceOrigin,
prediction,
residual,
scratch,
width,
height,
horizontalFilter,
verticalFilter,
horizontalPhase,
verticalPhase,
bitDepth);
/// <inheritdoc/>
public static void Predict(
ReadOnlySpan<byte> reference,
int referenceStride,
int referenceOrigin,
Span<byte> buffer,
int width,
int height,
int horizontalPhase,
int verticalPhase,
int taps,
int bitDepth)
=> Av1MotionSearchBase.ByteOperator.Predict(
reference, referenceStride, referenceOrigin, buffer, width, height, horizontalPhase, verticalPhase, taps, bitDepth);
/// <inheritdoc/>
public static int SumAbsoluteDifferences(
ReadOnlySpan<byte> source,
int sourceStride,
ReadOnlySpan<byte> prediction,
int predictionStride,
int width,
int height,
int rowStep)
=> Av1MotionSearchBase.ByteOperator.SumAbsoluteDifferences(
source, sourceStride, prediction, predictionStride, width, height, rowStep);
/// <inheritdoc/>
public static void GetMoments(
ReadOnlySpan<byte> source,
int sourceStride,
ReadOnlySpan<byte> prediction,
int predictionStride,
int width,
int height,
out int sum,
out long squares)
=> Av1MotionSearchBase.ByteOperator.GetMoments(
source, sourceStride, prediction, predictionStride, width, height, out sum, out squares);
/// <inheritdoc/> /// <inheritdoc/>
public static Span<byte> GetLeftReference(Span<short> residual, int length) public static Span<byte> GetLeftReference(Span<short> residual, int length)
=> MemoryMarshal.AsBytes(residual)[..length]; => MemoryMarshal.AsBytes(residual)[..length];
@ -470,36 +525,6 @@ internal static partial class Av1IntraSuperblockEncoder
return true; return true;
} }
/// <inheritdoc/>
public static long GetInterPredictionError(
Buffer2DRegion<byte> source,
Point sourceOrigin,
Buffer2DRegion<byte> reference,
Point predictionOrigin,
Av1BitDepth bitDepth)
{
Rectangle sourceBounds = source.Bounds;
Rectangle referenceBounds = reference.Bounds;
int sourceIndex =
((sourceBounds.Y + sourceOrigin.Y) * source.Stride) +
sourceBounds.X +
sourceOrigin.X;
int referenceIndex =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
referenceBounds.X +
predictionOrigin.X;
// The shared residual kernel selects the widest available vector width and handles the scalar tail.
return Av1ResidualBuilder.SumSquaredError(
source.Buffer.DangerousGetSingleSpan()[sourceIndex..],
source.Stride,
reference.Buffer.DangerousGetSingleSpan()[referenceIndex..],
reference.Stride,
InterSearchBlockDimension,
InterSearchBlockDimension);
}
/// <inheritdoc/> /// <inheritdoc/>
public static int GetSumOfAbsoluteDifferences( public static int GetSumOfAbsoluteDifferences(
Buffer2DRegion<byte> source, Buffer2DRegion<byte> source,
@ -962,6 +987,80 @@ internal static partial class Av1IntraSuperblockEncoder
/// </summary> /// </summary>
internal readonly struct UInt16Operator : IBlockEncodingOperator<ushort> internal readonly struct UInt16Operator : IBlockEncodingOperator<ushort>
{ {
/// <inheritdoc/>
public static void PreparePrediction(
ReadOnlySpan<ushort> source,
int sourceStride,
ReadOnlySpan<ushort> reference,
int referenceStride,
int referenceOrigin,
Span<ushort> prediction,
Span<short> residual,
Span<short> scratch,
int width,
int height,
Av1InterpolationFilter horizontalFilter,
Av1InterpolationFilter verticalFilter,
int horizontalPhase,
int verticalPhase,
int bitDepth)
=> Av1MotionSearchBase.UInt16Operator.PreparePrediction(
source,
sourceStride,
reference,
referenceStride,
referenceOrigin,
prediction,
residual,
scratch,
width,
height,
horizontalFilter,
verticalFilter,
horizontalPhase,
verticalPhase,
bitDepth);
/// <inheritdoc/>
public static void Predict(
ReadOnlySpan<ushort> reference,
int referenceStride,
int referenceOrigin,
Span<ushort> buffer,
int width,
int height,
int horizontalPhase,
int verticalPhase,
int taps,
int bitDepth)
=> Av1MotionSearchBase.UInt16Operator.Predict(
reference, referenceStride, referenceOrigin, buffer, width, height, horizontalPhase, verticalPhase, taps, bitDepth);
/// <inheritdoc/>
public static int SumAbsoluteDifferences(
ReadOnlySpan<ushort> source,
int sourceStride,
ReadOnlySpan<ushort> prediction,
int predictionStride,
int width,
int height,
int rowStep)
=> Av1MotionSearchBase.UInt16Operator.SumAbsoluteDifferences(
source, sourceStride, prediction, predictionStride, width, height, rowStep);
/// <inheritdoc/>
public static void GetMoments(
ReadOnlySpan<ushort> source,
int sourceStride,
ReadOnlySpan<ushort> prediction,
int predictionStride,
int width,
int height,
out int sum,
out long squares)
=> Av1MotionSearchBase.UInt16Operator.GetMoments(
source, sourceStride, prediction, predictionStride, width, height, out sum, out squares);
/// <inheritdoc/> /// <inheritdoc/>
public static Span<ushort> GetLeftReference(Span<short> residual, int length) public static Span<ushort> GetLeftReference(Span<short> residual, int length)
=> MemoryMarshal.Cast<short, ushort>(residual)[..length]; => MemoryMarshal.Cast<short, ushort>(residual)[..length];
@ -997,38 +1096,6 @@ internal static partial class Av1IntraSuperblockEncoder
return true; return true;
} }
/// <inheritdoc/>
public static long GetInterPredictionError(
Buffer2DRegion<ushort> source,
Point sourceOrigin,
Buffer2DRegion<ushort> reference,
Point predictionOrigin,
Av1BitDepth bitDepth)
{
Rectangle sourceBounds = source.Bounds;
Rectangle referenceBounds = reference.Bounds;
int sourceIndex =
((sourceBounds.Y + sourceOrigin.Y) * source.Stride) +
sourceBounds.X +
sourceOrigin.X;
int referenceIndex =
((referenceBounds.Y + predictionOrigin.Y) * reference.Stride) +
referenceBounds.X +
predictionOrigin.X;
long error = Av1ResidualBuilder.SumSquaredError(
source.Buffer.DangerousGetSingleSpan()[sourceIndex..],
source.Stride,
reference.Buffer.DangerousGetSingleSpan()[referenceIndex..],
reference.Stride,
InterSearchBlockDimension,
InterSearchBlockDimension);
int shift = (bitDepth.GetBitCount() - 8) * 2;
return shift == 0 ? error : (error + (1L << (shift - 1))) >> shift;
}
/// <inheritdoc/> /// <inheritdoc/>
public static int GetSumOfAbsoluteDifferences( public static int GetSumOfAbsoluteDifferences(
Buffer2DRegion<ushort> source, Buffer2DRegion<ushort> source,

621
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ReferenceModeDecision.cs

@ -18,16 +18,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// </content> /// </content>
internal static partial class Av1IntraSuperblockEncoder internal static partial class Av1IntraSuperblockEncoder
{ {
/// <summary>
/// The first effort tier that searches a block-local motion vector.
/// </summary>
private const int MinimumInterMotionSearchEffort = 6;
/// <summary>
/// The smallest full-pixel radius used by block-local inter search.
/// </summary>
private const int MinimumInterMotionSearchRadius = 4;
/// <summary> /// <summary>
/// The first effort tier that refines full-pixel motion to quarter-pixel precision. /// The first effort tier that refines full-pixel motion to quarter-pixel precision.
/// </summary> /// </summary>
@ -38,16 +28,6 @@ internal static partial class Av1IntraSuperblockEncoder
/// </summary> /// </summary>
private const int MinimumHighPrecisionMotionSearchEffort = 8; private const int MinimumHighPrecisionMotionSearchEffort = 8;
/// <summary>
/// The physical border reserved on each side for fractional eight-tap filtering.
/// </summary>
private const int FractionalInterpolationBorder = 4;
/// <summary>
/// The number of cardinal and diagonal candidates examined at each search step.
/// </summary>
private const int InterMotionSearchDirectionCount = 8;
/// <summary> /// <summary>
/// One nearest, three near, one global, and three new-motion candidates. /// One nearest, three near, one global, and three new-motion candidates.
/// </summary> /// </summary>
@ -236,22 +216,16 @@ internal static partial class Av1IntraSuperblockEncoder
out Av1EncoderTransformBlockState lumaCandidateState, out Av1EncoderTransformBlockState lumaCandidateState,
out int lumaRate, out int lumaRate,
out long lumaDistortion, out long lumaDistortion,
out bool hasEmptyLuma, out long lumaPredictionDistortion);
out Av1EncoderTransformBlockState emptyLumaState,
out long emptyLumaDistortion);
int blueRate = 0; int blueRate = 0;
int redRate = 0; int redRate = 0;
long blueDistortion = 0; long blueDistortion = 0;
long redDistortion = 0; long redDistortion = 0;
long emptyBlueDistortion = 0; long bluePredictionDistortion = 0;
long emptyRedDistortion = 0; long redPredictionDistortion = 0;
bool hasEmptyBlue = true;
bool hasEmptyRed = true;
Av1EncoderTransformBlockState blueCandidateState = default; Av1EncoderTransformBlockState blueCandidateState = default;
Av1EncoderTransformBlockState redCandidateState = default; Av1EncoderTransformBlockState redCandidateState = default;
Av1EncoderTransformBlockState emptyBlueState = default;
Av1EncoderTransformBlockState emptyRedState = default;
if (!this.source.IsMonochrome) if (!this.source.IsMonochrome)
{ {
Av1TransformType chromaTransformType = lumaCandidateState.TransformType; Av1TransformType chromaTransformType = lumaCandidateState.TransformType;
@ -292,9 +266,7 @@ internal static partial class Av1IntraSuperblockEncoder
out blueCandidateState, out blueCandidateState,
out blueRate, out blueRate,
out blueDistortion, out blueDistortion,
out hasEmptyBlue, out bluePredictionDistortion);
out emptyBlueState,
out emptyBlueDistortion);
this.EvaluateInterPlane( this.EvaluateInterPlane(
writer, writer,
@ -321,40 +293,32 @@ internal static partial class Av1IntraSuperblockEncoder
out redCandidateState, out redCandidateState,
out redRate, out redRate,
out redDistortion, out redDistortion,
out hasEmptyRed, out redPredictionDistortion);
out emptyRedState,
out emptyRedDistortion);
} }
int displacementRate = writer.GetDisplacementVectorCost(candidate, reference); int predictionRate = writer.GetUseIntraBlockCopyCost(true) +
int candidateRate = writer.GetUseIntraBlockCopyCost(true) + writer.GetDisplacementVectorCost(candidate, reference);
displacementRate +
writer.GetSkipCost(false, skipContext) + int residualRate = writer.GetSkipCost(false, skipContext) +
transformPartitionRate + transformPartitionRate +
lumaRate + lumaRate +
blueRate + blueRate +
redRate; redRate;
long candidateDistortion = lumaDistortion + blueDistortion + redDistortion; long candidateDistortion = lumaDistortion + blueDistortion + redDistortion;
Av1RateDistortionStatistics candidateStatistics = new(this.rateMultiplier, candidateRate, candidateDistortion); int skipRate = writer.GetSkipCost(true, skipContext);
bool candidateSkip = false; long skipDistortion = lumaPredictionDistortion + bluePredictionDistortion + redPredictionDistortion;
// The skip alternative is available only when every coded plane has an empty transform. Its // Empty residuals omit the transform tree. Nonempty residuals may also be discarded when
// distortion comes from prediction alone and its rate excludes the transform tree and coefficients. // prediction alone costs no more; exclude shared prediction syntax before rounding either rate.
if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed) bool candidateSkip = (lumaCandidateState.EndOfBlock == 0 &&
{ blueCandidateState.EndOfBlock == 0 && redCandidateState.EndOfBlock == 0) ||
int skipRate = writer.GetUseIntraBlockCopyCost(true) + Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion) <=
displacementRate + Av1RateDistortion.GetCost(this.rateMultiplier, residualRate, candidateDistortion);
writer.GetSkipCost(true, skipContext);
long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion; Av1RateDistortionStatistics candidateStatistics = candidateSkip
Av1RateDistortionStatistics skipStatistics = new(this.rateMultiplier, skipRate, skipDistortion); ? new(this.rateMultiplier, predictionRate + skipRate, skipDistortion)
if (skipStatistics.Cost < candidateStatistics.Cost) : new(this.rateMultiplier, predictionRate + residualRate, candidateDistortion);
{
candidateStatistics = skipStatistics;
candidateSkip = true;
}
}
// Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD. // Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD.
if (candidateStatistics.Cost >= bestStatistics.Cost) if (candidateStatistics.Cost >= bestStatistics.Cost)
@ -370,7 +334,7 @@ internal static partial class Av1IntraSuperblockEncoder
{ {
workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction); workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction);
workspace.SelectedLumaCoefficients.Clear(); workspace.SelectedLumaCoefficients.Clear();
selectedLumaState = emptyLumaState; selectedLumaState = default;
if (!this.source.IsMonochrome) if (!this.source.IsMonochrome)
{ {
int chromaSampleCount = chromaTransformSize.GetSize2d(); int chromaSampleCount = chromaTransformSize.GetSize2d();
@ -378,8 +342,8 @@ internal static partial class Av1IntraSuperblockEncoder
workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction); workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction);
workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear(); workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear();
workspace.SelectedRedCoefficients[..chromaSampleCount].Clear(); workspace.SelectedRedCoefficients[..chromaSampleCount].Clear();
selectedBlueState = emptyBlueState; selectedBlueState = default;
selectedRedState = emptyRedState; selectedRedState = default;
} }
} }
else else
@ -577,45 +541,31 @@ internal static partial class Av1IntraSuperblockEncoder
Span<Av1PredictionMode> candidateModes = stackalloc Av1PredictionMode[MaximumInterModeCandidateCount]; Span<Av1PredictionMode> candidateModes = stackalloc Av1PredictionMode[MaximumInterModeCandidateCount];
Span<byte> candidateReferenceIndices = stackalloc byte[MaximumInterModeCandidateCount]; Span<byte> candidateReferenceIndices = stackalloc byte[MaximumInterModeCandidateCount];
int candidateCount = 0; int candidateCount = 0;
if (this.effort >= MinimumInterMotionSearchEffort)
// Keep distinct syntax choices even when their prediction vectors are equal.
candidateVectors[candidateCount] = referenceMotionVectors.Nearest;
candidateModes[candidateCount] = Av1PredictionMode.NearestMotionVector;
candidateReferenceIndices[candidateCount++] = 0;
int maximumNewIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 1));
for (int referenceIndex = 0; referenceIndex <= maximumNewIndex; referenceIndex++)
{ {
// Predictor-stack modes precede global and new motion so strict ties retain the reference order. candidateVectors[candidateCount] = referenceMotionVectors.GetNewReference(referenceIndex);
candidateVectors[candidateCount] = referenceMotionVectors.Nearest; candidateModes[candidateCount] = Av1PredictionMode.NewMotionVector;
candidateModes[candidateCount] = Av1PredictionMode.NearestMotionVector; candidateReferenceIndices[candidateCount++] = (byte)referenceIndex;
candidateReferenceIndices[candidateCount++] = 0; }
int maximumNearIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 2)); int maximumNearIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 2));
for (int referenceIndex = 0; referenceIndex <= maximumNearIndex; referenceIndex++) for (int referenceIndex = 0; referenceIndex <= maximumNearIndex; referenceIndex++)
{ {
candidateVectors[candidateCount] = referenceMotionVectors.GetNearReference(referenceIndex); candidateVectors[candidateCount] = referenceMotionVectors.GetNearReference(referenceIndex);
candidateModes[candidateCount] = Av1PredictionMode.NearMotionVector; candidateModes[candidateCount] = Av1PredictionMode.NearMotionVector;
candidateReferenceIndices[candidateCount++] = (byte)referenceIndex; candidateReferenceIndices[candidateCount++] = (byte)referenceIndex;
}
} }
candidateVectors[candidateCount] = globalMotion; candidateVectors[candidateCount] = globalMotion;
candidateModes[candidateCount] = Av1PredictionMode.GlobalMotionVector; candidateModes[candidateCount] = Av1PredictionMode.GlobalMotionVector;
candidateReferenceIndices[candidateCount++] = 0; candidateReferenceIndices[candidateCount++] = 0;
if (this.effort >= MinimumInterMotionSearchEffort)
{
int maximumNewIndex = Math.Min(2, Math.Max(0, referenceMotionVectors.Count - 1));
for (int referenceIndex = 0; referenceIndex <= maximumNewIndex; referenceIndex++)
{
Av1MotionVector newReference = referenceMotionVectors.GetNewReference(referenceIndex);
Av1MotionVector searched = this.FindInterMotionVector(
writer,
blockOrigin,
newReference,
referenceIndex);
// Equal prediction vectors can carry different DRL and mode costs. Preserve each syntax choice
// as an independent candidate instead of deduplicating solely by reconstructed pixels.
candidateVectors[candidateCount] = searched;
candidateModes[candidateCount] = Av1PredictionMode.NewMotionVector;
candidateReferenceIndices[candidateCount++] = (byte)referenceIndex;
}
}
Span<TSample> selectedLumaReconstruction = workspace.SelectedLumaReconstruction; Span<TSample> selectedLumaReconstruction = workspace.SelectedLumaReconstruction;
Span<TSample> candidateLumaReconstruction = workspace.LumaCandidateReconstruction; Span<TSample> candidateLumaReconstruction = workspace.LumaCandidateReconstruction;
Span<TSample> selectedBlueReconstruction = workspace.SelectedBlueReconstruction; Span<TSample> selectedBlueReconstruction = workspace.SelectedBlueReconstruction;
@ -698,10 +648,148 @@ internal static partial class Av1IntraSuperblockEncoder
int horizontalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingX ? 1 : 0)) - 1; int horizontalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingX ? 1 : 0)) - 1;
int verticalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingY ? 1 : 0)) - 1; int verticalFractionMask = (Av1MotionVector.SubpixelScale << (block.HasChroma && sequenceHeader.ColorConfig.SubSamplingY ? 1 : 0)) - 1;
Buffer2DRegion<TSample> sourcePlane = this.source.GetPlane(Av1Plane.Y);
Buffer2DRegion<TSample> referencePlane = this.reference.GetPlane(Av1Plane.Y);
int sourceOrigin = ((sourcePlane.Bounds.Y + blockOrigin.Y) * sourcePlane.Stride) + sourcePlane.Bounds.X + blockOrigin.X;
int referenceOrigin = ((referencePlane.Bounds.Y + blockOrigin.Y) * referencePlane.Stride) + referencePlane.Bounds.X + blockOrigin.X;
Size frameSize = new(
this.picture.Parent.Common.ModeInfoColumnCount << Av1Constants.ModeInfoSizeLog2,
this.picture.Parent.Common.ModeInfoRowCount << Av1Constants.ModeInfoSizeLog2);
Rectangle frameBounds = Av1MotionVector.GetFrameSearchBounds(
new Rectangle(blockOrigin, new Size(8)),
frameSize,
Math.Min(referencePlane.Bounds.X, referencePlane.Bounds.Y));
Av1NeighborArrayUnit<byte> coefficientContexts = this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex];
ReadOnlySpan<byte> aboveContexts = coefficientContexts.Top[coefficientContexts.GetTopIndex(blockOrigin)..];
ReadOnlySpan<byte> leftContexts = coefficientContexts.Left[coefficientContexts.GetLeftIndex(blockOrigin)..];
// Frame owners provide contiguous padded planes. Borrow those spans without copying source blocks
// or reconstructing border samples, and keep the search scratch disjoint from retained inter winners.
Av1MotionSearchBase.SingleReferenceSearch<TSample, TOperator> motionSearch = new(
sourcePlane.Buffer.DangerousGetSingleSpan()[sourceOrigin..],
sourcePlane.Stride,
referencePlane.Buffer.DangerousGetSingleSpan(),
referencePlane.Stride,
referenceOrigin,
BlockSize,
frameBounds,
this.blockWorkspace,
this.blockWorkspace.GetMotionSearchPrediction<TSample>(),
this.blockWorkspace.Residual,
workspace.PredictionScratch,
workspace.TransformCoefficients,
writer,
aboveContexts,
leftContexts,
this.bitDepth,
this.quantization.QIndex[0],
this.quantization.DeltaQDc[0],
0,
frameHeader.CodedLossless,
this.rateMultiplier,
transformPartitionRate,
writer.GetSkipCost(false, skipContext),
writer.GetSkipCost(true, skipContext),
defaultFilter,
defaultFilter,
this.blockWorkspace.GetMotionVectorCosts(frameHeader.MotionVectorPrecision));
// The first two reference predictors set the block's spatial range. Clamping at the last
// potentially visible interpolation tap bounds padded reads without changing their prediction.
int spatialMagnitude = 0;
for (int index = 0; index < 2; index++)
{
Av1MotionVector spatial = referenceMotionVectors.GetNewReference(index);
int column = Math.Clamp(spatial.Column, -(blockOrigin.X + 8 + 4) * 8, (frameSize.Width - blockOrigin.X + 4) * 8);
int row = Math.Clamp(spatial.Row, -(blockOrigin.Y + 8 + 4) * 8, (frameSize.Height - blockOrigin.Y + 4) * 8);
spatialMagnitude = Math.Max(spatialMagnitude, Math.Max(Math.Abs(row), Math.Abs(column)) >> 3);
}
Av1MotionSearchSettings motionSettings = this.picture.Parent.MotionSearchSettings;
Av1MotionSearchBase.SingleReferenceState motionState = default;
Span<Av1MotionSearchBase.StartingCandidate> motionStarts = stackalloc Av1MotionSearchBase.StartingCandidate[1];
// Rank interpolation families with prediction-error modeling before running a full transform search. // Rank interpolation families with prediction-error modeling before running a full transform search.
// The selected inter reconstruction remains untouched while two existing prediction views alternate. // The selected inter reconstruction remains untouched while two existing prediction views alternate.
for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++) for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++)
{ {
if (candidateModes[candidateIndex] == Av1PredictionMode.NewMotionVector)
{
int referenceIndex = candidateReferenceIndices[candidateIndex];
Av1MotionVector referenceVector = candidateVectors[candidateIndex];
int drlRate = 0;
for (int index = 0; index < 2 && referenceMotionVectors.Count > index + 1; index++)
{
bool advance = referenceIndex > index;
int context = Av1SymbolContextHelper.GetDrlContext(referenceMotionVectors.Weights, index);
drlRate += writer.GetDynamicReferenceListCost(advance, context);
if (!advance)
{
break;
}
}
int searchRange = int.MaxValue;
if (motionSettings.ReduceSearchRange && referenceIndex > 0)
{
int minimumDifference = int.MaxValue;
int bestMatch = 0;
for (int index = 0; index < referenceIndex; index++)
{
Av1MotionVector previousReference = motionState.References[index].ReferenceVector;
int difference = Math.Max(
Math.Abs(referenceVector.Row - previousReference.Row),
Math.Abs(referenceVector.Column - previousReference.Column));
if (difference < minimumDifference)
{
minimumDifference = difference;
bestMatch = index;
}
}
ref Av1MotionSearchBase.ReferenceSearchResult previous = ref motionState.References[bestMatch];
if (minimumDifference < 16 * 8 && previous.IsValid)
{
int displacement = Math.Max(
Math.Abs(previous.Vector.Row - previous.ReferenceVector.Row),
Math.Abs(previous.Vector.Column - previous.ReferenceVector.Column));
searchRange = (minimumDifference + displacement + 4) >> 3;
}
}
Point startVector = new(
(referenceVector.Column + 3 + (referenceVector.Column >= 0 ? 1 : 0)) >> 3,
(referenceVector.Row + 3 + (referenceVector.Row >= 0 ? 1 : 0)) >> 3);
motionStarts[0] = new Av1MotionSearchBase.StartingCandidate(startVector, 0);
if (!motionSearch.Search(
motionSettings,
this.picture.Parent.MotionSearchStepParameter,
spatialMagnitude,
frameHeader.ShowFrame,
searchRange,
frameHeader.ForceIntegerMotionVector,
frameHeader.AllowHighPrecisionMotionVector,
fineMeshInterval: false,
referenceIndex,
referenceVector,
drlRate,
motionStarts,
totalWeight: 0,
ref motionState,
out Av1MotionSearchBase.FractionalResult searchResult) ||
motionState.References[referenceIndex].Skip)
{
continue;
}
candidateVectors[candidateIndex] = searchResult.Vector;
}
modeInfo.Block.Mode = candidateModes[candidateIndex]; modeInfo.Block.Mode = candidateModes[candidateIndex];
bool writesFilters = Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo.Block); bool writesFilters = Av1TileWriter.UsesSwitchableInterpolation(frameHeader, modeInfo.Block);
Av1InterpolationFilter verticalFilter = defaultFilter; Av1InterpolationFilter verticalFilter = defaultFilter;
@ -811,7 +899,7 @@ internal static partial class Av1IntraSuperblockEncoder
out Av1EncoderTransformBlockState candidateBlueState, out Av1EncoderTransformBlockState candidateBlueState,
out Av1EncoderTransformBlockState candidateRedState); out Av1EncoderTransformBlockState candidateRedState);
// Strict replacement preserves predictor-stack, global, then new-motion order on equal RD cost. // Strict replacement preserves nearest, new, near, then global mode order on equal RD cost.
if (candidateStatistics.Cost >= selectedStatistics.Cost) if (candidateStatistics.Cost >= selectedStatistics.Cost)
{ {
continue; continue;
@ -988,9 +1076,8 @@ internal static partial class Av1IntraSuperblockEncoder
int visibleHeight = Math.Min(height, ((this.source.Height + subsamplingY) >> subsamplingY) - planeOrigin.Y); int visibleHeight = Math.Min(height, ((this.source.Height + subsamplingY) >> subsamplingY) - planeOrigin.Y);
long squaredError = 0; long squaredError = 0;
// This view includes coded alignment samples, matching libaom when do_border_pad is false. // The source view includes samples extended to the coded dimensions. Reduce complete rows together;
// Its conditional border-padding policy is not implemented here; these are not visible-frame bounds. // a partial right edge needs separate row reductions to exclude samples beyond the source view.
// Full blocks use one SIMD reduction; only a partial right edge needs row-sized reductions.
if (visibleWidth == width) if (visibleWidth == width)
{ {
squaredError = Av1ResidualBuilder.SumSquares(residual[..(width * visibleHeight)]); squaredError = Av1ResidualBuilder.SumSquares(residual[..(width * visibleHeight)]);
@ -1097,9 +1184,7 @@ internal static partial class Av1IntraSuperblockEncoder
out lumaState, out lumaState,
out int lumaRate, out int lumaRate,
out long lumaDistortion, out long lumaDistortion,
out bool hasEmptyLuma, out long lumaPredictionDistortion);
out Av1EncoderTransformBlockState emptyLumaState,
out long emptyLumaDistortion);
// Empty luma transforms signal no transform type. Chroma inherits the decoder's inferred DCT // Empty luma transforms signal no transform type. Chroma inherits the decoder's inferred DCT
// type, not the last searched luma type, so normalize before evaluating either chroma plane. // type, not the last searched luma type, so normalize before evaluating either chroma plane.
@ -1124,14 +1209,10 @@ internal static partial class Av1IntraSuperblockEncoder
int redRate = 0; int redRate = 0;
long blueDistortion = 0; long blueDistortion = 0;
long redDistortion = 0; long redDistortion = 0;
long emptyBlueDistortion = 0; long bluePredictionDistortion = 0;
long emptyRedDistortion = 0; long redPredictionDistortion = 0;
bool hasEmptyBlue = true;
bool hasEmptyRed = true;
blueState = default; blueState = default;
redState = default; redState = default;
Av1EncoderTransformBlockState emptyBlueState = default;
Av1EncoderTransformBlockState emptyRedState = default;
if (hasChroma) if (hasChroma)
{ {
Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled( Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled(
@ -1188,9 +1269,7 @@ internal static partial class Av1IntraSuperblockEncoder
out blueState, out blueState,
out blueRate, out blueRate,
out blueDistortion, out blueDistortion,
out hasEmptyBlue, out bluePredictionDistortion);
out emptyBlueState,
out emptyBlueDistortion);
this.EvaluateInterPlane( this.EvaluateInterPlane(
writer, writer,
@ -1217,9 +1296,7 @@ internal static partial class Av1IntraSuperblockEncoder
out redState, out redState,
out redRate, out redRate,
out redDistortion, out redDistortion,
out hasEmptyRed, out redPredictionDistortion);
out emptyRedState,
out emptyRedDistortion);
} }
int predictionRate = commonPredictionRate + int predictionRate = commonPredictionRate +
@ -1239,258 +1316,34 @@ internal static partial class Av1IntraSuperblockEncoder
long codedDistortion = lumaDistortion + blueDistortion + redDistortion; long codedDistortion = lumaDistortion + blueDistortion + redDistortion;
Av1RateDistortionStatistics selectedStatistics = new(this.rateMultiplier, codedRate, codedDistortion); Av1RateDistortionStatistics selectedStatistics = new(this.rateMultiplier, codedRate, codedDistortion);
skip = false; int skipRate = writer.GetSkipCost(true, skipContext);
if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed) long skipDistortion = lumaPredictionDistortion + bluePredictionDistortion + redPredictionDistortion;
{
int skipRate = predictionRate + writer.GetSkipCost(true, skipContext);
long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion;
Av1RateDistortionStatistics skipStatistics = new(this.rateMultiplier, skipRate, skipDistortion);
if (skipStatistics.Cost < selectedStatistics.Cost)
{
selectedStatistics = skipStatistics;
skip = true;
workspace.LumaPrediction[..LumaTransformSize.GetSize2d()].CopyTo(lumaReconstruction);
lumaCoefficients[..LumaTransformSize.GetSize2d()].Clear();
lumaState = emptyLumaState;
if (hasChroma)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
workspace.BluePrediction[..chromaSampleCount].CopyTo(blueReconstruction);
workspace.RedPrediction[..chromaSampleCount].CopyTo(redReconstruction);
blueCoefficients[..chromaSampleCount].Clear();
redCoefficients[..chromaSampleCount].Clear();
blueState = emptyBlueState;
redState = emptyRedState;
}
}
}
return selectedStatistics;
}
/// <summary>
/// Searches a bounded full-pixel neighborhood around the spatial reference vector.
/// </summary>
/// <param name="writer">The live tile entropy model used to measure vector syntax.</param>
/// <param name="blockOrigin">The current 8x8 luma origin.</param>
/// <param name="referenceVector">The differential reference from the spatial candidate stack.</param>
/// <param name="referenceMotionVectorIndex">The selected dynamic-reference-list entry.</param>
/// <returns>The lowest-cost full-pixel vector found by the effort-scaled search.</returns>
private Av1MotionVector FindInterMotionVector(
Av1SymbolEncoder writer,
Point blockOrigin,
Av1MotionVector referenceVector,
int referenceMotionVectorIndex)
{
int effortShift = this.effort - MinimumInterMotionSearchEffort;
int searchRadius = Math.Min(
MinimumInterMotionSearchRadius << effortShift,
Av1EncoderFrame<TSample>.LumaBorder);
int referenceColumn = referenceVector.Column >> Av1MotionVector.SubpixelBits;
int referenceRow = referenceVector.Row >> Av1MotionVector.SubpixelBits;
int minimumColumn = Math.Max(-Av1EncoderFrame<TSample>.LumaBorder, referenceColumn - searchRadius);
int maximumColumn = Math.Min(Av1EncoderFrame<TSample>.LumaBorder, referenceColumn + searchRadius);
int minimumRow = Math.Max(-Av1EncoderFrame<TSample>.LumaBorder, referenceRow - searchRadius);
int maximumRow = Math.Min(Av1EncoderFrame<TSample>.LumaBorder, referenceRow + searchRadius);
Point best = new(
Av1Math.Clamp(referenceColumn, minimumColumn, maximumColumn),
Av1Math.Clamp(referenceRow, minimumRow, maximumRow));
ref Av1ReferenceMotionVectors referenceMotionVectors = ref this.blockWorkspace.ReferenceMotionVectors; // All-empty residuals omit the transform tree. Nonempty residuals can also be discarded when
Av1MotionVector bestVector = new( // prediction alone costs no more; shared prediction syntax must not affect the rounded comparison.
best.Y * Av1MotionVector.SubpixelScale, skip = (lumaState.EndOfBlock == 0 && blueState.EndOfBlock == 0 && redState.EndOfBlock == 0) ||
best.X * Av1MotionVector.SubpixelScale); Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion) <=
Av1RateDistortion.GetCost(this.rateMultiplier, codedRate - predictionRate, codedDistortion);
long bestCost = this.GetInterMotionCandidateCost(
writer,
blockOrigin,
bestVector,
Av1PredictionMode.NewMotionVector,
referenceMotionVectorIndex,
in referenceMotionVectors);
for (int step = searchRadius; step > 0; step >>= 1) if (skip)
{ {
Point stageBest = best; selectedStatistics = new(this.rateMultiplier, predictionRate + skipRate, skipDistortion);
long stageBestCost = bestCost; workspace.LumaPrediction[..LumaTransformSize.GetSize2d()].CopyTo(lumaReconstruction);
for (int directionIndex = 0; directionIndex < InterMotionSearchDirectionCount; directionIndex++) lumaCoefficients[..LumaTransformSize.GetSize2d()].Clear();
lumaState = default;
if (hasChroma)
{ {
Point direction = GetInterMotionSearchDirection(directionIndex); int chromaSampleCount = chromaTransformSize.GetSize2d();
Point candidate = new( workspace.BluePrediction[..chromaSampleCount].CopyTo(blueReconstruction);
best.X + (direction.X * step), workspace.RedPrediction[..chromaSampleCount].CopyTo(redReconstruction);
best.Y + (direction.Y * step)); blueCoefficients[..chromaSampleCount].Clear();
redCoefficients[..chromaSampleCount].Clear();
if (candidate.X < minimumColumn || candidate.X > maximumColumn || blueState = default;
candidate.Y < minimumRow || candidate.Y > maximumRow) redState = default;
{
continue;
}
Av1MotionVector candidateVector = new(
candidate.Y * Av1MotionVector.SubpixelScale,
candidate.X * Av1MotionVector.SubpixelScale);
long candidateCost = this.GetInterMotionCandidateCost(
writer,
blockOrigin,
candidateVector,
Av1PredictionMode.NewMotionVector,
referenceMotionVectorIndex,
in referenceMotionVectors);
// Strict replacement preserves the earlier reference-centered search position on ties.
if (candidateCost < stageBestCost)
{
stageBestCost = candidateCost;
stageBest = candidate;
}
} }
best = stageBest;
bestCost = stageBestCost;
}
bestVector = new(
best.Y * Av1MotionVector.SubpixelScale,
best.X * Av1MotionVector.SubpixelScale);
if (this.effort < MinimumSubpixelMotionSearchEffort)
{
return bestVector;
} }
int minimumSubpixel = (-Av1EncoderFrame<TSample>.LumaBorder + FractionalInterpolationBorder) * return selectedStatistics;
Av1MotionVector.SubpixelScale;
int maximumSubpixel = (Av1EncoderFrame<TSample>.LumaBorder - FractionalInterpolationBorder) *
Av1MotionVector.SubpixelScale;
if (bestVector.Column < minimumSubpixel || bestVector.Column > maximumSubpixel ||
bestVector.Row < minimumSubpixel || bestVector.Row > maximumSubpixel)
{
return bestVector;
}
int finalStep = this.effort >= MinimumHighPrecisionMotionSearchEffort ? 1 : 2;
for (int step = Av1MotionVector.SubpixelScale >> 1; step >= finalStep; step >>= 1)
{
Av1MotionVector stageBest = bestVector;
long stageBestCost = bestCost;
for (int directionIndex = 0; directionIndex < InterMotionSearchDirectionCount; directionIndex++)
{
Point direction = GetInterMotionSearchDirection(directionIndex);
Av1MotionVector candidate = new(
bestVector.Row + (direction.Y * step),
bestVector.Column + (direction.X * step));
if (candidate.Column < minimumSubpixel || candidate.Column > maximumSubpixel ||
candidate.Row < minimumSubpixel || candidate.Row > maximumSubpixel)
{
continue;
}
long candidateCost = this.GetInterMotionCandidateCost(
writer,
blockOrigin,
candidate,
Av1PredictionMode.NewMotionVector,
referenceMotionVectorIndex,
in referenceMotionVectors);
// Each precision stage remains centered on its incoming winner; strict replacement keeps
// the integer or coarser fractional vector when an interpolated candidate only ties it.
if (candidateCost < stageBestCost)
{
stageBestCost = candidateCost;
stageBest = candidate;
}
}
bestVector = stageBest;
bestCost = stageBestCost;
}
return bestVector;
}
/// <summary>
/// Combines normalized prediction error with the exact mode and vector syntax rate.
/// </summary>
/// <param name="writer">The live tile entropy model.</param>
/// <param name="blockOrigin">The current 8x8 luma origin.</param>
/// <param name="vector">The candidate motion vector.</param>
/// <param name="mode">The candidate single-reference inter mode.</param>
/// <param name="referenceMotionVectorIndex">The selected dynamic-reference-list entry.</param>
/// <param name="referenceMotionVectors">The current spatial candidate stack.</param>
/// <returns>The rate-distortion search cost.</returns>
private long GetInterMotionCandidateCost(
Av1SymbolEncoder writer,
Point blockOrigin,
Av1MotionVector vector,
Av1PredictionMode mode,
int referenceMotionVectorIndex,
in Av1ReferenceMotionVectors referenceMotionVectors)
{
long predictionError;
if (((vector.Row | vector.Column) & (Av1MotionVector.SubpixelScale - 1)) == 0)
{
Point predictionOrigin = new(
blockOrigin.X + (vector.Column >> Av1MotionVector.SubpixelBits),
blockOrigin.Y + (vector.Row >> Av1MotionVector.SubpixelBits));
predictionError = TOperator.GetInterPredictionError(
this.source.GetPlane(Av1Plane.Y),
blockOrigin,
this.reference.GetPlane(Av1Plane.Y),
predictionOrigin,
this.bitDepth);
}
else
{
const Av1TransformSize SearchTransformSize = Av1TransformSize.Size8x8;
Av1EncoderInterPredictionWorkspace<TSample> workspace =
this.blockWorkspace.GetInterPredictionWorkspace<TSample>();
int sourceColumnQ4 = (blockOrigin.X << 4) + (vector.Column << 1);
int sourceRowQ4 = (blockOrigin.Y << 4) + (vector.Row << 1);
Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4);
ObuFrameHeader frameHeader = this.picture.Parent.FrameHeader;
// Fractional candidates must pass through the same interpolation and residual kernels used by
// final reconstruction; comparing only their integer origins would choose the wrong phase.
TOperator.PrepareTranslationalInterPrediction(
this.source.GetPlane(Av1Plane.Y),
blockOrigin,
this.reference.GetPlane(Av1Plane.Y),
predictionOrigin,
frameHeader.InterpolationFilter == Av1InterpolationFilter.Switchable ? Av1InterpolationFilter.Regular : frameHeader.InterpolationFilter,
frameHeader.InterpolationFilter == Av1InterpolationFilter.Switchable ? Av1InterpolationFilter.Regular : frameHeader.InterpolationFilter,
sourceColumnQ4 & 15,
sourceRowQ4 & 15,
workspace.LumaPrediction,
workspace.Residual,
workspace.PredictionScratch,
SearchTransformSize,
this.bitDepth);
predictionError = Av1ResidualBuilder.SumSquares(workspace.Residual);
int normalizationShift = (this.bitDepth.GetBitCount() - 8) * 2;
if (normalizationShift != 0)
{
predictionError = (predictionError + (1L << (normalizationShift - 1))) >>
normalizationShift;
}
}
int rate = this.GetInterModeRate(
writer,
mode,
vector,
referenceMotionVectorIndex,
in referenceMotionVectors);
return Av1RateDistortion.GetCost(this.rateMultiplier, rate, predictionError);
} }
/// <summary> /// <summary>
@ -1543,29 +1396,12 @@ internal static partial class Av1IntraSuperblockEncoder
} }
Av1MotionVector reference = referenceMotionVectors.GetNewReference(referenceMotionVectorIndex); Av1MotionVector reference = referenceMotionVectors.GetNewReference(referenceMotionVectorIndex);
return rate + writer.GetMotionVectorCost( Av1MotionVectorCosts costs = this.blockWorkspace.GetMotionVectorCosts(this.picture.Parent.FrameHeader.MotionVectorPrecision);
vector,
reference,
this.picture.Parent.FrameHeader.MotionVectorPrecision);
}
/// <summary> // Mode selection discounts motion syntax to 108/128 of its estimated rate. Apply the rounded
/// Gets one cardinal or diagonal search direction in stable reference order. // weight to the vector alone; mode and dynamic-reference-list symbols retain their full rate.
/// </summary> return rate + (((costs.GetCost(vector, reference) * 108) + 64) >> 7);
/// <param name="index">The zero-based direction index.</param> }
/// <returns>The unit full-pixel direction.</returns>
private static Point GetInterMotionSearchDirection(int index)
=> index switch
{
0 => new Point(0, -1),
1 => new Point(0, 1),
2 => new Point(-1, 0),
3 => new Point(1, 0),
4 => new Point(-1, -1),
5 => new Point(1, 1),
6 => new Point(1, -1),
_ => new Point(-1, 1)
};
/// <summary> /// <summary>
/// Builds one plane prediction and selects its transform without repeating interpolation for each transform type. /// Builds one plane prediction and selects its transform without repeating interpolation for each transform type.
@ -1595,9 +1431,7 @@ internal static partial class Av1IntraSuperblockEncoder
out Av1EncoderTransformBlockState selectedState, out Av1EncoderTransformBlockState selectedState,
out int selectedRate, out int selectedRate,
out long selectedDistortion, out long selectedDistortion,
out bool hasEmptyTransform, out long predictionDistortion)
out Av1EncoderTransformBlockState emptyState,
out long emptyDistortion)
{ {
Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY); Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY);
int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX)); int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX));
@ -1647,8 +1481,15 @@ internal static partial class Av1IntraSuperblockEncoder
transformSize); transformSize);
} }
// Motion compensation and subtraction do not depend on transform type. Keep them outside the // Prediction-only error remains available even when every transform quantizes to nonzero coefficients.
// transform loop so exhaustive luma search traverses the source and reference blocks only once. // Normalize squared sample precision with rounding before adding four fractional distortion bits.
long predictionSquaredError = Av1ResidualBuilder.SumSquares(residual[..sampleCount]);
int normalizationShift = (this.bitDepth.GetBitCount() - 8) * 2;
predictionDistortion = normalizationShift == 0
? predictionSquaredError << 4
: ((predictionSquaredError + (1L << (normalizationShift - 1))) >> normalizationShift) << 4;
// Motion compensation and subtraction are shared by all transform types for this prediction.
Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType( Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType(
transformSize, transformSize,
isInter: true, isInter: true,
@ -1665,13 +1506,9 @@ internal static partial class Av1IntraSuperblockEncoder
selectedState = default; selectedState = default;
selectedRate = 0; selectedRate = 0;
selectedDistortion = 0; selectedDistortion = 0;
hasEmptyTransform = false;
emptyState = default;
emptyDistortion = 0;
// The candidate and best spans alternate ownership whenever a transform improves the result. // Alternate candidate and best spans on improvement. The winning storage stays intact during
// This mirrors the reference's buffer-pointer swap and replaces a copy on every improvement // later trials, with at most one normalization copy into the caller's destination after the search.
// with at most one normalization copy after the transform search.
Span<TSample> candidateReconstruction = transformReconstruction[..sampleCount]; Span<TSample> candidateReconstruction = transformReconstruction[..sampleCount];
Span<int> candidateCoefficients = transformCoefficients[..sampleCount]; Span<int> candidateCoefficients = transformCoefficients[..sampleCount];
Span<TSample> bestReconstruction = selectedReconstruction[..sampleCount]; Span<TSample> bestReconstruction = selectedReconstruction[..sampleCount];
@ -1737,14 +1574,6 @@ internal static partial class Av1IntraSuperblockEncoder
selectedRate = candidateRate; selectedRate = candidateRate;
selectedDistortion = candidateDistortion; selectedDistortion = candidateDistortion;
} }
if (candidateState.EndOfBlock == 0 &&
(!hasEmptyTransform || candidateDistortion < emptyDistortion))
{
hasEmptyTransform = true;
emptyState = candidateState;
emptyDistortion = candidateDistortion;
}
} }
// Callers retain the designated selected spans after this scratch workspace is reused by the // Callers retain the designated selected spans after this scratch workspace is reused by the

22
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs

@ -55,7 +55,8 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param> /// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param> /// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param> /// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect( /// <returns>Whether the frame is classified as screen content for encoder decisions.</returns>
public static bool Detect(
Av1EncoderFrame<byte> source, Av1EncoderFrame<byte> source,
out bool allowScreenContentTools, out bool allowScreenContentTools,
out bool allowIntraBlockCopy) out bool allowIntraBlockCopy)
@ -67,22 +68,23 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param> /// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param> /// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param> /// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect( /// <returns>Whether the frame is classified as screen content for encoder decisions.</returns>
public static bool Detect(
Av1EncoderFrame<ushort> source, Av1EncoderFrame<ushort> source,
out bool allowScreenContentTools, out bool allowScreenContentTools,
out bool allowIntraBlockCopy) out bool allowIntraBlockCopy)
=> Detect<ushort, UShortSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy); => Detect<ushort, UShortSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy);
private static void Detect<TSample, TOperator>( private static bool Detect<TSample, TOperator>(
Av1EncoderFrame<TSample> source, Av1EncoderFrame<TSample> source,
out bool allowScreenContentTools, out bool allowScreenContentTools,
out bool allowIntraBlockCopy) out bool allowIntraBlockCopy)
where TSample : unmanaged where TSample : unmanaged
where TOperator : struct, ISampleOperator<TSample> where TOperator : struct, ISampleOperator<TSample>
{ {
Av1EncoderFrame<TSample>.PlanarView view = source.View; Av1EncoderFrame<TSample>.PlanarView view = source.CodedView;
int width = source.Width; int width = (source.Width + 7) & ~7;
int height = source.Height; int height = (source.Height + 7) & ~7;
long frameArea = (long)width * height; long frameArea = (long)width * height;
int bitDepthShift = source.LumaBitDepth - 8; int bitDepthShift = source.LumaBitDepth - 8;
int paletteBlockCount = 0; int paletteBlockCount = 0;
@ -91,7 +93,8 @@ internal static class Av1ScreenContentDetector
allowScreenContentTools = false; allowScreenContentTools = false;
allowIntraBlockCopy = false; allowIntraBlockCopy = false;
// Complete 16x16 blocks and the strict frame-area threshold preserve the reference detector's decision. // Analyze complete 16x16 blocks in the source's eight-sample-aligned extent. Padding participates in
// both color counting and the area thresholds, while any final partial block is omitted.
for (int blockRow = 0; blockRow + DetectionBlockLength <= height; blockRow += DetectionBlockLength) for (int blockRow = 0; blockRow + DetectionBlockLength <= height; blockRow += DetectionBlockLength)
{ {
for (int blockColumn = 0; blockColumn + DetectionBlockLength <= width; blockColumn += DetectionBlockLength) for (int blockColumn = 0; blockColumn + DetectionBlockLength <= width; blockColumn += DetectionBlockLength)
@ -152,11 +155,14 @@ internal static class Av1ScreenContentDetector
if (allowIntraBlockCopy) if (allowIntraBlockCopy)
{ {
return; return true;
} }
} }
} }
} }
return (long)paletteBlockCount * DetectionBlockArea * 10 > frameArea * 4 &&
(long)intraBlockCopyBlockCount * DetectionBlockArea * 30 > frameArea;
} }
private static long RoundPowerOfTwo(long value, int shift) private static long RoundPowerOfTwo(long value, int shift)

175
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TileEncoder.cs

@ -4,6 +4,7 @@
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
@ -38,7 +39,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>( this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator,
Av1DeblockingFilter.VerticalByteEdgeOperator, Av1DeblockingFilter.HorizontalByteEdgeOperator>(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -74,7 +76,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>( this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator,
Av1DeblockingFilter.VerticalByteEdgeOperator, Av1DeblockingFilter.HorizontalByteEdgeOperator>(
writer, writer,
source, source,
reference, reference,
@ -110,7 +113,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>( this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator,
Av1DeblockingFilter.VerticalByteEdgeOperator, Av1DeblockingFilter.HorizontalByteEdgeOperator>(
writer, writer,
source, source,
reference, reference,
@ -144,7 +148,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>( this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator,
Av1DeblockingFilter.VerticalUInt16EdgeOperator, Av1DeblockingFilter.HorizontalUInt16EdgeOperator>(
writer, writer,
source, source,
reconstruction, reconstruction,
@ -180,7 +185,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>( this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator,
Av1DeblockingFilter.VerticalUInt16EdgeOperator, Av1DeblockingFilter.HorizontalUInt16EdgeOperator>(
writer, writer,
source, source,
reference, reference,
@ -216,7 +222,8 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
{ {
this.picture = picture; this.picture = picture;
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>( this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator,
Av1DeblockingFilter.VerticalUInt16EdgeOperator, Av1DeblockingFilter.HorizontalUInt16EdgeOperator>(
writer, writer,
source, source,
reference, reference,
@ -236,7 +243,7 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
return this.tileData.Span.Slice(offset, length); return this.tileData.Span.Slice(offset, length);
} }
private static ReadOnlyMemory<byte> Encode<TSample, TOperator>( private static ReadOnlyMemory<byte> Encode<TSample, TOperator, TVerticalOperator, THorizontalOperator>(
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
Av1EncoderFrame<TSample> source, Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reference, Av1EncoderFrame<TSample> reference,
@ -248,6 +255,66 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
int effort) int effort)
where TSample : unmanaged where TSample : unmanaged
where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample> where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample>
where TVerticalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
where THorizontalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
{
Av1PictureParentControlSet parent = picture.Parent;
ObuFrameHeader frameHeader = parent.FrameHeader;
Av1MotionSearchSettings motionSettings = new(
parent.EncodingSpeed,
picture.Sequence.SequenceHeader.IsStillPicture,
new Size(source.Width, source.Height),
frameHeader.QuantizationParameters.BaseQIndex,
frameHeader.IsIntra,
parent.IsScreenContent);
parent.MotionSearchSettings = motionSettings;
int maximumDimension = Math.Max(source.Width, source.Height);
int stepParameter = Av1MotionSearchBase.GetInitialStepParameter(maximumDimension);
if (frameHeader.IsIntra)
{
// A key frame seeds the following inter frame with the complete frame range.
parent.MaximumMotionVectorMagnitude = maximumDimension;
}
else if (motionSettings.AutomaticStepSizeLevel != 0)
{
if (frameHeader.ShowFrame && motionSettings.AutomaticStepSizeLevel >= 2 && parent.MaximumMotionVectorMagnitude != -1)
{
int range = Math.Min(maximumDimension, 2 * parent.MaximumMotionVectorMagnitude);
stepParameter = Av1MotionSearchBase.GetInitialStepParameter(range);
}
// The packing pass accumulates actual NEWMV magnitudes. Trial candidates and inherited vectors
// do not contribute; a frame with no written NEWMV leaves a zero maximum for the next frame.
parent.MaximumMotionVectorMagnitude = 0;
}
parent.MotionSearchStepParameter = stepParameter;
_ = ProcessTiles<TSample, TOperator, Av1SymbolEncoder.SymbolUpdateOperation>(
writer, source, reference, reconstruction, picture, coefficientBuffer, tileWorkspace, blockWorkspace, effort);
Av1LoopFilterEncoder.ApplyFrame<TSample, TVerticalOperator, THorizontalOperator>(picture, reconstruction);
// Analysis retains the selected modes, coefficients, palette tokens, and motion contexts. Packing starts
// from the same entropy edges and probabilities while the completed frame decisions remain available.
picture.ResetEntropyContexts();
return ProcessTiles<TSample, TOperator, Av1SymbolEncoder.SymbolWriteOperation>(
writer, source, reference, reconstruction, picture, coefficientBuffer, tileWorkspace, blockWorkspace, effort);
}
private static ReadOnlyMemory<byte> ProcessTiles<TSample, TOperator, TSymbolOperation>(
Av1SymbolEncoder writer,
Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reference,
Av1EncoderFrame<TSample> reconstruction,
Av1PictureControlSet picture,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderTileWorkspace tileWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort)
where TSample : unmanaged
where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample>
where TSymbolOperation : struct, Av1SymbolEncoder.ISymbolOperation
{ {
ObuFrameHeader frameHeader = picture.Parent.FrameHeader; ObuFrameHeader frameHeader = picture.Parent.FrameHeader;
ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader; ObuSequenceHeader sequenceHeader = picture.Sequence.SequenceHeader;
@ -260,7 +327,7 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
ObuTileGroupHeader tileLayout = frameHeader.TilesInfo; ObuTileGroupHeader tileLayout = frameHeader.TilesInfo;
Span<int> tileDataOffsets = picture.TileDataOffsets.Span; Span<int> tileDataOffsets = picture.TileDataOffsets.Span;
Span<int> tileDataLengths = picture.TileDataLengths.Span; Span<int> tileDataLengths = picture.TileDataLengths.Span;
if (frameHeader.AllowIntraBlockCopy) if (!TSymbolOperation.WritesOutput && frameHeader.AllowIntraBlockCopy)
{ {
// Hash the visible source once before reconstruction begins so candidate discovery never depends // Hash the visible source once before reconstruction begins so candidate discovery never depends
// on coding order and the workspace can be reused as compact bucket links afterward. // on coding order and the workspace can be reused as compact bucket links afterward.
@ -276,12 +343,10 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
for (int tileColumn = 0; tileColumn < tileLayout.TileColumnCount; tileColumn++) for (int tileColumn = 0; tileColumn < tileLayout.TileColumnCount; tileColumn++)
{ {
tile.SetTileColumn(tileLayout, frameHeader.ModeInfoColumnCount, tileColumn); tile.SetTileColumn(tileLayout, frameHeader.ModeInfoColumnCount, tileColumn);
if (tileIndex > 0)
{ // Each pass begins every tile from the same frame probabilities. Only the packing pass
// Every tile begins from the same frame probabilities, while its bytes follow the preceding // advances the output offset; the analysis operation does not touch range-coder state.
// tile in the retained output allocation. writer.Reset(tileDataEnd);
writer.Reset(tileDataEnd);
}
Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart); Point firstModeInfoPosition = new(tile.ModeInfoColumnStart, tile.ModeInfoRowStart);
entropyContext.MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition); entropyContext.MacroBlockModeInfo = picture.GetMacroBlockModeInfo(firstModeInfoPosition);
@ -300,36 +365,64 @@ internal readonly struct Av1TileEncoder : IAv1TileWriter
modeInfoColumn << Av1Constants.ModeInfoSizeLog2, modeInfoColumn << Av1Constants.ModeInfoSizeLog2,
modeInfoRow << Av1Constants.ModeInfoSizeLog2); modeInfoRow << Av1Constants.ModeInfoSizeLog2);
Av1IntraSuperblockEncoder.Prepare( if (TSymbolOperation.WritesOutput)
picture, {
superblock, Av1TileWriter.RetainedBlockEncodingHandler blockEncoder = new(picture);
entropyContext.SuperblockOrigin); Av1TileWriter.WriteSuperblock<TSymbolOperation, Av1TileWriter.RetainedBlockEncodingHandler>(
picture,
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator> blockEncoder = new( entropyContext,
source, writer,
reference, superblock,
reconstruction, coefficientBuffer,
picture, (ushort)tileIndex,
superblock, ref blockEncoder);
coefficientBuffer, }
blockWorkspace, else
effort); {
if (!frameHeader.IsIntra)
Av1TileWriter.WriteSuperblock( {
picture, // Candidates within a superblock share one entropy snapshot. Updating while
entropyContext, // trying partitions would make the search depend on discarded alternatives.
writer, writer.FillMotionVectorCosts(blockWorkspace.GetMotionVectorCosts(frameHeader.MotionVectorPrecision));
superblock, }
coefficientBuffer,
(ushort)tileIndex, Av1IntraSuperblockEncoder.Prepare(
ref blockEncoder); picture,
superblock,
entropyContext.SuperblockOrigin);
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator> blockEncoder = new(
source,
reference,
reconstruction,
picture,
superblock,
coefficientBuffer,
blockWorkspace,
effort);
Av1TileWriter.WriteSuperblock<
TSymbolOperation,
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator>>(
picture,
entropyContext,
writer,
superblock,
coefficientBuffer,
(ushort)tileIndex,
ref blockEncoder);
}
} }
} }
_ = writer.Exit(out int tileDataLength); if (TSymbolOperation.WritesOutput)
tileDataOffsets[tileIndex] = tileDataEnd; {
tileDataLengths[tileIndex] = tileDataLength; _ = writer.Exit(out int tileDataLength);
tileDataEnd += tileDataLength; tileDataOffsets[tileIndex] = tileDataEnd;
tileDataLengths[tileIndex] = tileDataLength;
tileDataEnd += tileDataLength;
}
tileIndex++; tileIndex++;
} }
} }

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalByteEdgeOperator.cs

@ -11,7 +11,7 @@ internal static partial class Av1DeblockingFilter
/// <summary> /// <summary>
/// Accesses four columns across a horizontal edge in eight-bit storage. /// Accesses four columns across a horizontal edge in eight-bit storage.
/// </summary> /// </summary>
private readonly struct HorizontalByteEdgeOperator : IEdgeOperator<byte> public readonly struct HorizontalByteEdgeOperator : IEdgeOperator<byte>
{ {
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.HorizontalUInt16EdgeOperator.cs

@ -11,7 +11,7 @@ internal static partial class Av1DeblockingFilter
/// <summary> /// <summary>
/// Accesses four columns across a horizontal edge in 16-bit storage. /// Accesses four columns across a horizontal edge in 16-bit storage.
/// </summary> /// </summary>
private readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator<ushort> public readonly struct HorizontalUInt16EdgeOperator : IEdgeOperator<ushort>
{ {
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.Operator.cs

@ -11,7 +11,7 @@ internal static partial class Av1DeblockingFilter
/// Defines orientation- and storage-specific access to the four samples running along one edge segment. /// Defines orientation- and storage-specific access to the four samples running along one edge segment.
/// </summary> /// </summary>
/// <typeparam name="TSample">The reconstructed sample storage type.</typeparam> /// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
private interface IEdgeOperator<TSample> public interface IEdgeOperator<TSample>
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary> /// <summary>

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalByteEdgeOperator.cs

@ -11,7 +11,7 @@ internal static partial class Av1DeblockingFilter
/// <summary> /// <summary>
/// Accesses four rows across a vertical edge in eight-bit storage. /// Accesses four rows across a vertical edge in eight-bit storage.
/// </summary> /// </summary>
private readonly struct VerticalByteEdgeOperator : IEdgeOperator<byte> public readonly struct VerticalByteEdgeOperator : IEdgeOperator<byte>
{ {
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.VerticalUInt16EdgeOperator.cs

@ -11,7 +11,7 @@ internal static partial class Av1DeblockingFilter
/// <summary> /// <summary>
/// Accesses four rows across a vertical edge in 16-bit storage. /// Accesses four rows across a vertical edge in 16-bit storage.
/// </summary> /// </summary>
private readonly struct VerticalUInt16EdgeOperator : IEdgeOperator<ushort> public readonly struct VerticalUInt16EdgeOperator : IEdgeOperator<ushort>
{ {
/// <inheritdoc/> /// <inheritdoc/>
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]

2
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs

@ -115,7 +115,7 @@ internal static partial class Av1DeblockingFilter
/// <param name="boundaryLimit">The eight-bit-domain edge-discontinuity threshold.</param> /// <param name="boundaryLimit">The eight-bit-domain edge-discontinuity threshold.</param>
/// <param name="highEdgeVarianceThreshold">The eight-bit-domain high-edge-variance threshold.</param> /// <param name="highEdgeVarianceThreshold">The eight-bit-domain high-edge-variance threshold.</param>
/// <param name="bitDepth">The sample bit depth.</param> /// <param name="bitDepth">The sample bit depth.</param>
private static void Filter<TSample, TEdgeOperator>( public static void Filter<TSample, TEdgeOperator>(
Span<TSample> samples, Span<TSample> samples,
int q0Offset, int q0Offset,
int stride, int stride,

276
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterBase.cs

@ -0,0 +1,276 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
/// <summary>
/// Traverses reconstructed transform boundaries for in-place deblocking.
/// </summary>
internal static class Av1LoopFilterBase
{
/// <summary>
/// Supplies the selected block and transform state at a frame position.
/// </summary>
/// <typeparam name="TState">The owning encoder or decoder state.</typeparam>
/// <typeparam name="TMode">The retained block-mode type.</typeparam>
public interface IFrameOperator<TState, TMode>
where TMode : struct
{
/// <summary>
/// Resolves the block and filter parameters covering one plane position.
/// </summary>
/// <param name="state">The owning frame state.</param>
/// <param name="position">The position in luma 4x4 units, adjusted for chroma ownership.</param>
/// <param name="plane">The component plane.</param>
/// <param name="pass">Zero for vertical boundaries; one for horizontal boundaries.</param>
/// <param name="subX">The horizontal subsampling shift.</param>
/// <param name="subY">The vertical subsampling shift.</param>
/// <param name="blockIndex">The storage index identifying the owning prediction block.</param>
/// <param name="skippedTransform">Whether an inter block omits its residual.</param>
/// <param name="transformSize">The transform covering the plane position.</param>
/// <param name="mode">The mode state retained for subsequent level derivation.</param>
public static abstract void GetParameters(
TState state,
Point position,
Av1Plane plane,
int pass,
int subX,
int subY,
out int blockIndex,
out bool skippedTransform,
out Av1TransformSize transformSize,
out TMode mode);
/// <summary>
/// Derives the level for a block whose boundary requires filtering.
/// </summary>
/// <param name="state">The owning frame state.</param>
/// <param name="mode">The resolved block mode, read without another grid lookup or structure copy.</param>
/// <param name="position">The block position in luma 4x4 units.</param>
/// <param name="plane">The component plane.</param>
/// <param name="pass">Zero for vertical boundaries; one for horizontal boundaries.</param>
/// <returns>The adjusted level in the zero-to-63 domain.</returns>
public static abstract int GetFilterLevel(TState state, ref TMode mode, Point position, Av1Plane plane, int pass);
}
/// <summary>
/// Filters one plane band in vertical-then-horizontal boundary order.
/// </summary>
/// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
/// <typeparam name="TState">The encoder or decoder state type.</typeparam>
/// <typeparam name="TMode">The retained block-mode type.</typeparam>
/// <typeparam name="TFrameOperator">The selected block-state accessor.</typeparam>
/// <typeparam name="TVerticalOperator">The vertical sample accessor.</typeparam>
/// <typeparam name="THorizontalOperator">The horizontal sample accessor.</typeparam>
/// <param name="state">The owning frame state.</param>
/// <param name="header">The decoded or selected frame header.</param>
/// <param name="plane">The component plane.</param>
/// <param name="rowStart">The inclusive band origin in luma 4x4 units.</param>
/// <param name="rowEnd">The exclusive band limit in luma 4x4 units.</param>
/// <param name="subX">The horizontal subsampling shift.</param>
/// <param name="subY">The vertical subsampling shift.</param>
/// <param name="samples">The plane storage including the required edge neighborhoods.</param>
/// <param name="origin">The offset of the top-left coded sample within the storage.</param>
/// <param name="stride">The plane stride in samples.</param>
/// <param name="bitDepth">The component precision.</param>
public static void FilterBand<TSample, TState, TMode, TFrameOperator, TVerticalOperator, THorizontalOperator>(
TState state,
ObuFrameHeader header,
Av1Plane plane,
int rowStart,
int rowEnd,
int subX,
int subY,
Span<TSample> samples,
int origin,
int stride,
int bitDepth)
where TSample : unmanaged
where TMode : struct
where TFrameOperator : struct, IFrameOperator<TState, TMode>
where TVerticalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
where THorizontalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
{
int rowStep = 1 << subY;
int columnStep = 1 << subX;
// Chroma remains in luma-grid coordinates. Each step still covers four samples in its own plane.
// Finish vertical edges across each row before starting the horizontal traversal of this band.
for (int row = rowStart; row < rowEnd; row += rowStep)
{
for (int column = 0; column < header.ModeInfoColumnCount; column += columnStep)
{
FilterEdge<TSample, TState, TMode, TFrameOperator, TVerticalOperator>(
state, header, plane, 0, row, column, subX, subY, samples, origin, stride, bitDepth);
}
}
// Visit horizontal edges down each column: adjacent filters can read samples modified by earlier edges.
for (int column = 0; column < header.ModeInfoColumnCount; column += columnStep)
{
for (int row = rowStart; row < rowEnd; row += rowStep)
{
FilterEdge<TSample, TState, TMode, TFrameOperator, THorizontalOperator>(
state, header, plane, 1, row, column, subX, subY, samples, origin, stride, bitDepth);
}
}
}
/// <summary>
/// Combines the frame, superblock, segment, reference, and mode adjustments for one boundary.
/// </summary>
/// <param name="header">The frame's filter and segmentation state.</param>
/// <param name="filterIndex">The luma-direction or chroma-component index.</param>
/// <param name="baseLevel">The selected frame level for that component.</param>
/// <param name="delta">The superblock adjustment for that component.</param>
/// <param name="segmentId">The owning block's segment.</param>
/// <param name="referenceFrame">The primary prediction reference.</param>
/// <param name="mode">The luma prediction mode.</param>
/// <returns>The adjusted level in the zero-to-63 domain.</returns>
public static int GetFilterLevel(
ObuFrameHeader header,
int filterIndex,
int baseLevel,
int delta,
int segmentId,
Av1ReferenceFrameType referenceFrame,
Av1PredictionMode mode)
{
ObuLoopFilterParameters parameters = header.LoopFilterParameters;
int level = Av1Math.Clip3(0, Av1Constants.MaxLoopFilter, baseLevel + delta);
ObuSegmentationLevelFeature feature = (ObuSegmentationLevelFeature)((int)ObuSegmentationLevelFeature.AlternativeLoopFilterYVertical + filterIndex);
ObuSegmentationParameters segmentation = header.SegmentationParameters;
if (segmentation.IsFeatureActive(segmentId, feature))
{
level = Av1Math.Clip3(
0,
Av1Constants.MaxLoopFilter,
level + segmentation.GetFeatureData(segmentId, (int)feature));
}
if (parameters.ReferenceDeltaModeEnabled)
{
int referenceScale = 1 << (level >> 5);
level += parameters.ReferenceDeltas[(int)referenceFrame] * referenceScale;
if (referenceFrame > Av1ReferenceFrameType.Intra)
{
// Every inter mode except the two global-motion modes belongs to the second mode-delta class.
int modeDeltaIndex = mode is Av1PredictionMode.GlobalMotionVector or
Av1PredictionMode.GlobalGlobalMotionVector ? 0 : 1;
level += parameters.ModeDeltas[modeDeltaIndex] * referenceScale;
}
// Reference and mode adjustments use the same scale and may cancel beyond either limit.
// Clipping the intermediate reference sum would discard part of that cancellation.
level = Av1Math.Clip3(0, Av1Constants.MaxLoopFilter, level);
}
return level;
}
/// <summary>
/// Derives and applies the kernel at one transform boundary.
/// </summary>
/// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
/// <typeparam name="TState">The encoder or decoder state type.</typeparam>
/// <typeparam name="TMode">The retained block-mode type.</typeparam>
/// <typeparam name="TFrameOperator">The selected block-state accessor.</typeparam>
/// <typeparam name="TEdgeOperator">The oriented sample accessor.</typeparam>
/// <param name="state">The owning frame state.</param>
/// <param name="header">The decoded or selected frame header.</param>
/// <param name="plane">The component plane.</param>
/// <param name="pass">Zero for vertical boundaries; one for horizontal boundaries.</param>
/// <param name="row">The boundary row in luma 4x4 units.</param>
/// <param name="column">The boundary column in luma 4x4 units.</param>
/// <param name="subX">The horizontal subsampling shift.</param>
/// <param name="subY">The vertical subsampling shift.</param>
/// <param name="samples">The plane storage including the required edge neighborhoods.</param>
/// <param name="origin">The offset of the top-left coded sample.</param>
/// <param name="stride">The plane stride in samples.</param>
/// <param name="bitDepth">The component precision.</param>
private static void FilterEdge<TSample, TState, TMode, TFrameOperator, TEdgeOperator>(
TState state,
ObuFrameHeader header,
Av1Plane plane,
int pass,
int row,
int column,
int subX,
int subY,
Span<TSample> samples,
int origin,
int stride,
int bitDepth)
where TSample : unmanaged
where TMode : struct
where TFrameOperator : struct, IFrameOperator<TState, TMode>
where TEdgeOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
{
int x = column << Av1Constants.ModeInfoSizeLog2;
int y = row << Av1Constants.ModeInfoSizeLog2;
bool verticalBoundary = pass == 0;
if (x >= header.FrameSize.FrameWidth || y >= header.FrameSize.FrameHeight || (verticalBoundary ? x == 0 : y == 0))
{
return;
}
// Subsampled chroma belongs to the bottom/right luma unit within its 8-sample footprint.
Point position = new(column | subX, row | subY);
Point previous = new(position.X - (verticalBoundary ? 1 << subX : 0), position.Y - (verticalBoundary ? 0 : 1 << subY));
TFrameOperator.GetParameters(
state, position, plane, pass, subX, subY, out int blockIndex, out bool skipped, out Av1TransformSize transform, out TMode mode);
TFrameOperator.GetParameters(
state,
previous,
plane,
pass,
subX,
subY,
out int previousIndex,
out bool previousSkipped,
out Av1TransformSize previousTransform,
out TMode previousMode);
int planeX = x >> subX;
int planeY = y >> subY;
bool isTransformEdge = verticalBoundary ? planeX % transform.GetWidth() == 0 : planeY % transform.GetHeight() == 0;
// Residual-free intra predictions still need deblocking. Only skipped inter transforms suppress
// internal boundaries; two different prediction blocks retain their common boundary.
if (!isTransformEdge || (blockIndex == previousIndex && skipped && previousSkipped))
{
return;
}
int level = TFrameOperator.GetFilterLevel(state, ref mode, position, plane, pass);
int filterLevel = level != 0 ? level : TFrameOperator.GetFilterLevel(state, ref previousMode, previous, plane, pass);
if (filterLevel == 0)
{
return;
}
int size = verticalBoundary
? Math.Min(transform.GetWidth(), previousTransform.GetWidth())
: Math.Min(transform.GetHeight(), previousTransform.GetHeight());
int kernelLength = plane == Av1Plane.Y ? size == 4 ? 4 : size == 8 ? 8 : 14 : size == 4 ? 4 : 6;
int sharpness = header.LoopFilterParameters.SharpnessLevel;
int shift = sharpness > 4 ? 2 : sharpness > 0 ? 1 : 0;
int limit = sharpness > 0 ? Av1Math.Clip3(1, 9 - sharpness, filterLevel >> shift) : Math.Max(1, filterLevel);
int boundaryLimit = (2 * (filterLevel + 2)) + limit;
// Thresholds stay in the eight-bit domain; the sample operator and kernel handle storage and precision.
// The explicit origin accommodates bordered encoder planes and the decoder's preceding-row view.
Av1DeblockingFilter.Filter<TSample, TEdgeOperator>(
samples, origin + (planeY * stride) + planeX, stride, kernelLength, limit, boundaryLimit, filterLevel >> 4, bitDepth);
}
}

141
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1LoopFilterEncoder.cs

@ -0,0 +1,141 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
/// <summary>
/// Applies selected deblocking parameters to the encoder reconstruction.
/// </summary>
internal static class Av1LoopFilterEncoder
{
/// <summary>
/// Filters the completed reconstruction before it becomes a prediction reference.
/// </summary>
/// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
/// <typeparam name="TVerticalOperator">The vertical sample accessor.</typeparam>
/// <typeparam name="THorizontalOperator">The horizontal sample accessor.</typeparam>
/// <param name="picture">The completed frame decisions.</param>
/// <param name="reconstruction">The writable reconstructed component planes.</param>
public static void ApplyFrame<TSample, TVerticalOperator, THorizontalOperator>(
Av1PictureControlSet picture,
Av1EncoderFrame<TSample> reconstruction)
where TSample : unmanaged
where TVerticalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
where THorizontalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
{
ObuFrameHeader header = picture.Parent.FrameHeader;
ObuLoopFilterParameters parameters = header.LoopFilterParameters;
if (header.CodedLossless || header.AllowIntraBlockCopy || (parameters.FilterLevel[0] == 0 && parameters.FilterLevel[1] == 0))
{
return;
}
int planeCount = picture.Sequence.SequenceHeader.ColorConfig.PlaneCount;
for (int planeIndex = 0; planeIndex < planeCount; planeIndex++)
{
ApplyPlane<TSample, TVerticalOperator, THorizontalOperator>(picture, reconstruction, (Av1Plane)planeIndex);
}
}
/// <summary>
/// Filters one component using the current frame levels.
/// </summary>
/// <typeparam name="TSample">The reconstructed sample storage type.</typeparam>
/// <typeparam name="TVerticalOperator">The vertical sample accessor.</typeparam>
/// <typeparam name="THorizontalOperator">The horizontal sample accessor.</typeparam>
/// <param name="picture">The completed frame decisions.</param>
/// <param name="reconstruction">The writable reconstructed component planes.</param>
/// <param name="plane">The component to filter.</param>
public static void ApplyPlane<TSample, TVerticalOperator, THorizontalOperator>(
Av1PictureControlSet picture,
Av1EncoderFrame<TSample> reconstruction,
Av1Plane plane)
where TSample : unmanaged
where TVerticalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
where THorizontalOperator : struct, Av1DeblockingFilter.IEdgeOperator<TSample>
{
ObuFrameHeader header = picture.Parent.FrameHeader;
ObuLoopFilterParameters parameters = header.LoopFilterParameters;
int level = plane switch
{
Av1Plane.U => parameters.FilterLevelU,
Av1Plane.V => parameters.FilterLevelV,
_ => Math.Max(parameters.FilterLevel[0], parameters.FilterLevel[1])
};
if (level == 0)
{
return;
}
Buffer2DRegion<TSample> samples = reconstruction.CodedView.GetPlane(plane);
int origin = (samples.Bounds.Y * samples.Stride) + samples.Bounds.X;
int subX = plane == Av1Plane.Y ? 0 : reconstruction.ChromaSubsamplingX;
int subY = plane == Av1Plane.Y ? 0 : reconstruction.ChromaSubsamplingY;
int rowsPerBand = 1 << (Av1Constants.MaxSuperBlockSizeLog2 - Av1Constants.ModeInfoSizeLog2);
// The frame owner supplies one contiguous allocation. Retain its full bordered view so kernels may
// access their edge neighborhoods without copying the plane or materializing decoder frame state.
Span<TSample> storage = samples.Buffer.DangerousGetSingleSpan();
for (int rowStart = 0; rowStart < header.ModeInfoRowCount; rowStart += rowsPerBand)
{
int rowEnd = Math.Min(rowStart + rowsPerBand, header.ModeInfoRowCount);
Av1LoopFilterBase.FilterBand<TSample, Av1PictureControlSet, Av1EncoderBlockModeInfo, FrameOperator, TVerticalOperator, THorizontalOperator>(
picture, header, plane, rowStart, rowEnd, subX, subY, storage, origin, samples.Stride, reconstruction.LumaBitDepth);
}
}
/// <summary>
/// Reads deblocking parameters from the retained encoder mode grid.
/// </summary>
private readonly struct FrameOperator : Av1LoopFilterBase.IFrameOperator<Av1PictureControlSet, Av1EncoderBlockModeInfo>
{
/// <inheritdoc/>
public static void GetParameters(
Av1PictureControlSet state,
Point position,
Av1Plane plane,
int pass,
int subX,
int subY,
out int blockIndex,
out bool skippedTransform,
out Av1TransformSize transformSize,
out Av1EncoderBlockModeInfo mode)
{
blockIndex = state.ModeInfoGrid.Span[(position.Y * state.ModeInfoStride) + position.X];
mode = state.ModeInfoAllocation.Span[blockIndex].Block;
skippedTransform = mode.Skip && mode.ReferenceFrame > Av1ReferenceFrameType.Intra;
// The selected encoder transform size is uniform within each luma block. Chroma owns its
// maximum plane transform independently of luma splits, while lossless segments always use 4x4.
transformSize = state.Parent.FrameHeader.LosslessArray[mode.SegmentId]
? Av1TransformSize.Size4x4
: plane == Av1Plane.Y ? mode.TransformSize : mode.BlockSize.GetMaxUvTransformSize(subX != 0, subY != 0);
}
/// <inheritdoc/>
public static int GetFilterLevel(Av1PictureControlSet state, ref Av1EncoderBlockModeInfo mode, Point position, Av1Plane plane, int pass)
{
ObuFrameHeader header = state.Parent.FrameHeader;
ObuLoopFilterParameters parameters = header.LoopFilterParameters;
int index = plane == Av1Plane.Y ? pass : (int)plane + 1;
int level = index switch
{
0 => parameters.FilterLevel[0],
1 => parameters.FilterLevel[1],
2 => parameters.FilterLevelU,
_ => parameters.FilterLevelV
};
// The encoder does not emit superblock filter deltas. Frame, segment, reference, and mode
// adjustments still follow the same clipping and scaling as the decoder.
return Av1LoopFilterBase.GetFilterLevel(header, index, level, 0, mode.SegmentId, mode.ReferenceFrame, mode.Mode);
}
}
}

26
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderDisplacementVector.cs

@ -18,3 +18,29 @@ internal struct Av1EncoderDisplacementVector
/// </summary> /// </summary>
public short Column; public short Column;
} }
/// <summary>
/// Stores the reference-vector contexts selected before later blocks populate the frame grid.
/// </summary>
internal struct Av1EncoderReferenceContext
{
/// <summary>
/// Stores the differential reference vectors for the four usable stack entries.
/// </summary>
public InlineArray4<Av1EncoderDisplacementVector> References;
/// <summary>
/// Stores the candidate weights used by dynamic-reference-list syntax.
/// </summary>
public InlineArray4<ushort> Weights;
/// <summary>
/// Stores the packed inter-mode context.
/// </summary>
public ushort ModeContext;
/// <summary>
/// Stores the number of discovered candidates.
/// </summary>
public byte Count;
}

50
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs

@ -132,18 +132,40 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
: 0; : 0;
int paletteStorageEnd = checked(paletteStorageOffset + paletteStorageLength); int paletteStorageEnd = checked(paletteStorageOffset + paletteStorageLength);
int blockPaletteStorageLength = allocateScreenContentState
? checked(this.modeInfo.Allocation.Length * Unsafe.SizeOf<Av1EncoderPaletteInfo>())
: 0;
int paletteTokenStorageOffset = checked(paletteStorageEnd + blockPaletteStorageLength);
// Each of the two palette planes needs at most one packed token per sample. Maximum-superblock
// rounding keeps this picture-owned capacity valid for either supported superblock geometry.
int paletteTokenStorageLength = allocateScreenContentState
? checked(
Av1Math.AlignPowerOf2(width, Av1Constants.MaxSuperBlockSizeLog2) *
Av1Math.AlignPowerOf2(height, Av1Constants.MaxSuperBlockSizeLog2) *
Math.Min(2, colorConfig.PlaneCount))
: 0;
int paletteTokenStorageEnd = checked(paletteTokenStorageOffset + paletteTokenStorageLength);
int blockEncodingStorageLength = checked(this.modeInfo.Allocation.Length * Av1EncoderBlockStruct.StorageSize);
int blockEncodingStorageEnd = checked(paletteTokenStorageEnd + blockEncodingStorageLength);
int displacementVectorLength = allocateMotionVectorState ? this.modeInfo.Allocation.Length : 0; int displacementVectorLength = allocateMotionVectorState ? this.modeInfo.Allocation.Length : 0;
int displacementVectorStorageOffset = allocateMotionVectorState int displacementVectorStorageOffset = allocateMotionVectorState
? Av1Math.AlignPowerOf2(paletteStorageEnd, 1) ? Av1Math.AlignPowerOf2(blockEncodingStorageEnd, 1)
: paletteStorageEnd; : blockEncodingStorageEnd;
int displacementVectorStorageLength = checked( int displacementVectorStorageLength = checked(
displacementVectorLength * Unsafe.SizeOf<Av1EncoderDisplacementVector>()); displacementVectorLength * Unsafe.SizeOf<Av1EncoderDisplacementVector>());
int displacementVectorStorageEnd = checked(displacementVectorStorageOffset + displacementVectorStorageLength); int displacementVectorStorageEnd = checked(displacementVectorStorageOffset + displacementVectorStorageLength);
int referenceContextStorageLength = checked(
displacementVectorLength * Unsafe.SizeOf<Av1EncoderReferenceContext>());
int referenceContextStorageEnd = checked(displacementVectorStorageEnd + referenceContextStorageLength);
int intraBlockCopySearchStorageOffset = allocateIntraBlockCopySearch int intraBlockCopySearchStorageOffset = allocateIntraBlockCopySearch
? Av1Math.AlignPowerOf2(displacementVectorStorageEnd, 2) ? Av1Math.AlignPowerOf2(referenceContextStorageEnd, 2)
: displacementVectorStorageEnd; : referenceContextStorageEnd;
int intraBlockCopySearchStorageLength = allocateIntraBlockCopySearch int intraBlockCopySearchStorageLength = allocateIntraBlockCopySearch
? Av1IntraBlockCopySearchIndex.GetStorageLength(width, height) ? Av1IntraBlockCopySearchIndex.GetStorageLength(width, height)
@ -178,6 +200,7 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
this.redCoefficientContexts = new Av1NeighborArrayUnit<byte>[tileCount]; this.redCoefficientContexts = new Av1NeighborArrayUnit<byte>[tileCount];
this.transformContexts = new Av1NeighborArrayUnit<byte>[tileCount]; this.transformContexts = new Av1NeighborArrayUnit<byte>[tileCount];
Memory<Av1EncoderPaletteInfo> paletteStorage = Memory<Av1EncoderPaletteInfo>.Empty; Memory<Av1EncoderPaletteInfo> paletteStorage = Memory<Av1EncoderPaletteInfo>.Empty;
Memory<Av1EncoderPaletteInfo> blockPalettes = Memory<Av1EncoderPaletteInfo>.Empty;
if (allocateScreenContentState) if (allocateScreenContentState)
{ {
// Palette entries contain 16-bit colors, so their packed typed region begins at an even byte offset. // Palette entries contain 16-bit colors, so their packed typed region begins at an even byte offset.
@ -185,6 +208,10 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
stateStorage.Slice(paletteStorageOffset, paletteStorageLength)); stateStorage.Slice(paletteStorageOffset, paletteStorageLength));
paletteStorage = paletteMemory.Memory; paletteStorage = paletteMemory.Memory;
ByteMemoryManager<Av1EncoderPaletteInfo> blockPaletteMemory = new(
stateStorage.Slice(paletteStorageEnd, blockPaletteStorageLength));
blockPalettes = blockPaletteMemory.Memory;
this.paletteContexts = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>[tileCount]; this.paletteContexts = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>[tileCount];
} }
else else
@ -193,6 +220,13 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
} }
Memory<Av1EncoderDisplacementVector> displacementVectors = Memory<Av1EncoderDisplacementVector>.Empty; Memory<Av1EncoderDisplacementVector> displacementVectors = Memory<Av1EncoderDisplacementVector>.Empty;
// Final syntax parameters use their own eight-byte entries so frequent neighbor lookups retain
// the compact mode-info layout. This typed view borrows the same picture-state owner.
ByteMemoryManager<Av1EncoderBlockStruct> blockEncodingMemory = new(
stateStorage.Slice(paletteTokenStorageEnd, blockEncodingStorageLength));
Memory<Av1EncoderReferenceContext> referenceContexts = Memory<Av1EncoderReferenceContext>.Empty;
if (allocateMotionVectorState) if (allocateMotionVectorState)
{ {
// Each component lies strictly inside plus or minus 16384. Two signed 16-bit fields preserve both // Each component lies strictly inside plus or minus 16384. Two signed 16-bit fields preserve both
@ -201,6 +235,10 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
stateStorage.Slice(displacementVectorStorageOffset, displacementVectorStorageLength)); stateStorage.Slice(displacementVectorStorageOffset, displacementVectorStorageLength));
displacementVectors = displacementVectorMemory.Memory; displacementVectors = displacementVectorMemory.Memory;
ByteMemoryManager<Av1EncoderReferenceContext> referenceContextMemory = new(
stateStorage.Slice(displacementVectorStorageEnd, referenceContextStorageLength));
referenceContexts = referenceContextMemory.Memory;
} }
Av1IntraBlockCopySearchIndex intraBlockCopySearch = default; Av1IntraBlockCopySearchIndex intraBlockCopySearch = default;
@ -314,6 +352,10 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
ModeInfoGrid = this.modeInfo.Grid, ModeInfoGrid = this.modeInfo.Grid,
ModeInfoAllocation = this.modeInfo.Allocation, ModeInfoAllocation = this.modeInfo.Allocation,
DisplacementVectors = displacementVectors, DisplacementVectors = displacementVectors,
ReferenceContexts = referenceContexts,
BlockEncodings = blockEncodingMemory.Memory,
BlockPalettes = blockPalettes,
PaletteTokens = stateStorage.Slice(paletteTokenStorageOffset, paletteTokenStorageLength),
IntraBlockCopySearch = intraBlockCopySearch, IntraBlockCopySearch = intraBlockCopySearch,
ModeInfoStride = this.modeInfo.ModeInfoStride, ModeInfoStride = this.modeInfo.ModeInfoStride,
Disallow4x4AllFrames = this.modeInfo.Disallow4x4AllFrames, Disallow4x4AllFrames = this.modeInfo.Disallow4x4AllFrames,

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderTransformBlockState.cs

@ -22,6 +22,11 @@ internal struct Av1EncoderTransformBlockState
/// </summary> /// </summary>
private Av1TransformType transformType; private Av1TransformType transformType;
/// <summary>
/// Stores the skip context in bits 0 through 3 and DC-sign context in bits 4 and 5.
/// </summary>
public byte EntropyContext;
/// <summary> /// <summary>
/// Gets or sets the position after the final nonzero coefficient. /// Gets or sets the position after the final nonzero coefficient.
/// </summary> /// </summary>

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EntropyCodingContext.cs

@ -37,5 +37,10 @@ internal partial class Av1TileWriter
/// Gets or sets the number of chroma coefficient positions consumed in the current superblock. /// Gets or sets the number of chroma coefficient positions consumed in the current superblock.
/// </summary> /// </summary>
public int CodedAreaSuperblockUv { get; set; } public int CodedAreaSuperblockUv { get; set; }
/// <summary>
/// Gets or sets the next palette token position in the picture token region.
/// </summary>
public int PaletteTokenOffset { get; set; }
} }
} }

52
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureControlSet.cs

@ -70,6 +70,26 @@ internal class Av1PictureControlSet
/// </summary> /// </summary>
public Memory<Av1EncoderDisplacementVector> DisplacementVectors { get; set; } public Memory<Av1EncoderDisplacementVector> DisplacementVectors { get; set; }
/// <summary>
/// Gets or sets the reference contexts retained at each allocated block origin.
/// </summary>
public Memory<Av1EncoderReferenceContext> ReferenceContexts { get; set; }
/// <summary>
/// Gets or sets the prediction parameters retained at each allocated block origin.
/// </summary>
public Memory<Av1EncoderBlockStruct> BlockEncodings { get; set; }
/// <summary>
/// Gets or sets the palette colors retained at each allocated block origin.
/// </summary>
public Memory<Av1EncoderPaletteInfo> BlockPalettes { get; set; }
/// <summary>
/// Gets or sets the packed color-map tokens retained for final syntax packing.
/// </summary>
public Memory<byte> PaletteTokens { get; set; }
/// <summary> /// <summary>
/// Gets or sets the non-owning visible-frame hash index used by intra-block-copy motion search. /// Gets or sets the non-owning visible-frame hash index used by intra-block-copy motion search.
/// </summary> /// </summary>
@ -101,6 +121,38 @@ internal class Av1PictureControlSet
/// </summary> /// </summary>
public required Memory<int> TileDataLengths { get; set; } public required Memory<int> TileDataLengths { get; set; }
/// <summary>
/// Restores initial tile entropy edges while preserving selected block decisions and reconstruction.
/// </summary>
public void ResetEntropyContexts()
{
this.SegmentationNeighborMap.Span.Clear();
for (int tileIndex = 0; tileIndex < this.PartitionContexts.Length; tileIndex++)
{
this.PartitionContexts[tileIndex].Left.Clear();
this.PartitionContexts[tileIndex].Top.Clear();
this.LuminanceDcSignLevelCoefficientNeighbors[tileIndex].Left.Clear();
this.LuminanceDcSignLevelCoefficientNeighbors[tileIndex].Top.Clear();
this.CbDcSignLevelCoefficientNeighbors[tileIndex].Left.Clear();
this.CbDcSignLevelCoefficientNeighbors[tileIndex].Top.Clear();
this.CrDcSignLevelCoefficientNeighbors[tileIndex].Left.Clear();
this.CrDcSignLevelCoefficientNeighbors[tileIndex].Top.Clear();
// Transform contexts use the maximum-size sentinel until a preceding block supplies a size.
this.TransformFunctionContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize);
this.TransformFunctionContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize);
}
foreach (Av1NeighborArrayUnit<Av1EncoderPaletteInfo> context in this.PaletteContexts)
{
context.Left.Clear();
context.Top.Clear();
}
this.CdefPreset.Span.Fill(-1);
this.Parent.PreviousQIndex.Span.Fill(this.Parent.FrameHeader.QuantizationParameters.BaseQIndex);
}
/// <summary> /// <summary>
/// Gets the mode-information entry mapped to a frame position. /// Gets the mode-information entry mapped to a frame position.
/// </summary> /// </summary>

26
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1PictureParentControlSet.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; namespace SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
@ -29,4 +30,29 @@ internal class Av1PictureParentControlSet
/// Gets or sets the encoder palette-search level. /// Gets or sets the encoder palette-search level.
/// </summary> /// </summary>
public int PaletteLevel { get; set; } public int PaletteLevel { get; set; }
/// <summary>
/// Gets or sets the native-valued encoding speed.
/// </summary>
public HeifEncodingSpeed EncodingSpeed { get; set; }
/// <summary>
/// Gets or sets a value indicating whether source analysis classifies this frame as screen content.
/// </summary>
public bool IsScreenContent { get; set; }
/// <summary>
/// Gets or sets the resolved motion-search policy for the current frame.
/// </summary>
public Av1MotionSearchSettings MotionSearchSettings { get; set; }
/// <summary>
/// Gets or sets the initial full-pixel search step derived before the frame's first block.
/// </summary>
public int MotionSearchStepParameter { get; set; }
/// <summary>
/// Gets or sets the largest whole-sample magnitude written by a new-motion mode in the preceding frame.
/// </summary>
public int MaximumMotionVectorMagnitude { get; set; } = -1;
} }

101
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.BlockEncoding.cs

@ -15,6 +15,11 @@ internal partial class Av1TileWriter
/// </summary> /// </summary>
internal interface IBlockEncodingHandler internal interface IBlockEncodingHandler
{ {
/// <summary>
/// Gets a value indicating whether decisions come from completed frame analysis.
/// </summary>
static abstract bool UsesRetainedDecisions { get; }
/// <summary> /// <summary>
/// Selects the partition used for the current tree node. /// Selects the partition used for the current tree node.
/// </summary> /// </summary>
@ -53,8 +58,104 @@ internal partial class Av1TileWriter
ref Av1EncoderPaletteInfo paletteInfo); ref Av1EncoderPaletteInfo paletteInfo);
} }
internal readonly struct RetainedBlockEncodingHandler : IBlockEncodingHandler
{
private readonly Av1PictureControlSet picture;
public RetainedBlockEncodingHandler(Av1PictureControlSet picture)
=> this.picture = picture;
public static bool UsesRetainedDecisions => true;
public Av1PartitionType SelectPartition(
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,
Point blockOrigin,
ushort tileIndex,
Av1BlockSize blockSize,
Av1PartitionType preparedPartition)
{
Point position = new(blockOrigin.X >> Av1Constants.ModeInfoSizeLog2, blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2);
Av1BlockSize selectedSize = this.picture.GetFromModeInfoGrid(position).Block.BlockSize;
if (selectedSize == blockSize)
{
return Av1PartitionType.None;
}
int width = blockSize.Get4x4WideCount();
int height = blockSize.Get4x4HighCount();
int selectedWidth = selectedSize.Get4x4WideCount();
int selectedHeight = selectedSize.Get4x4HighCount();
if (blockSize > Av1BlockSize.Block8x8 &&
position.Y + (height / 2) < this.picture.Parent.Common.ModeInfoRowCount &&
position.X + (width / 2) < this.picture.Parent.Common.ModeInfoColumnCount)
{
// A half-sized top-left block alone cannot distinguish an asymmetric partition from a split.
// The mapped blocks at the two half boundaries identify which half remains unsplit.
Av1BlockSize below = this.picture.GetFromModeInfoGrid(position + new Size(0, height / 2)).Block.BlockSize;
Av1BlockSize right = this.picture.GetFromModeInfoGrid(position + new Size(width / 2, 0)).Block.BlockSize;
if (selectedWidth == width)
{
return selectedHeight * 4 == height
? Av1PartitionType.Horizontal4
: below == selectedSize ? Av1PartitionType.Horizontal : Av1PartitionType.HorizontalB;
}
if (selectedHeight == height)
{
return selectedWidth * 4 == width
? Av1PartitionType.Vertical4
: right == selectedSize ? Av1PartitionType.Vertical : Av1PartitionType.VerticalB;
}
if (selectedWidth * 2 == width && selectedHeight * 2 == height)
{
if (below.Get4x4WideCount() == width)
{
return Av1PartitionType.HorizontalA;
}
if (right.Get4x4HighCount() == height)
{
return Av1PartitionType.VerticalA;
}
}
return Av1PartitionType.Split;
}
// At a frame edge only the basic partitions are available. Each smaller dimension contributes
// one split axis; recursive descent then reaches the retained leaf geometry.
return selectedWidth == width
? Av1PartitionType.Horizontal
: selectedHeight == height ? Av1PartitionType.Vertical : Av1PartitionType.Split;
}
public void EncodeBlock(
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,
Point blockOrigin,
ushort tileIndex,
ref Av1MacroBlockModeInfo modeInfo,
ref Av1EncoderBlockStruct block,
ref Av1EncoderPaletteInfo paletteInfo)
{
int row = blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2;
int column = blockOrigin.X >> Av1Constants.ModeInfoSizeLog2;
int allocationOffset = this.picture.ModeInfoGrid.Span[(row * this.picture.ModeInfoStride) + column];
block = this.picture.BlockEncodings.Span[allocationOffset];
if (this.picture.Parent.FrameHeader.AllowScreenContentTools)
{
paletteInfo = this.picture.BlockPalettes.Span[allocationOffset];
}
}
}
private readonly struct PrecomputedBlockEncodingHandler : IBlockEncodingHandler private readonly struct PrecomputedBlockEncodingHandler : IBlockEncodingHandler
{ {
/// <inheritdoc/>
public static bool UsesRetainedDecisions => false;
/// <inheritdoc/> /// <inheritdoc/>
public Av1PartitionType SelectPartition( public Av1PartitionType SelectPartition(
Av1SymbolEncoder writer, Av1SymbolEncoder writer,

612
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs

File diff suppressed because it is too large

24
src/ImageSharp/Formats/Heif/HeifEncoder.cs

@ -23,6 +23,11 @@ public sealed class HeifEncoder : AnimatedImageEncoder
/// </summary> /// </summary>
private int effort = 5; private int effort = 5;
/// <summary>
/// The AV1 encoding speed.
/// </summary>
private HeifEncodingSpeed speed;
/// <summary> /// <summary>
/// Gets the compression method used for the primary image item. /// Gets the compression method used for the primary image item.
/// The default is <see cref="HeifCompressionMethod.Av1"/>. /// The default is <see cref="HeifCompressionMethod.Av1"/>.
@ -96,6 +101,25 @@ public sealed class HeifEncoder : AnimatedImageEncoder
/// </summary> /// </summary>
public bool Lossless { get; init; } public bool Lossless { get; init; }
/// <summary>
/// Gets the AV1 encoding speed. Higher levels prioritize speed over compression efficiency.
/// The default is <see cref="HeifEncodingSpeed.Level0"/>.
/// </summary>
/// <exception cref="ArgumentException">The speed is outside the range 0 to 9.</exception>
public HeifEncodingSpeed Speed
{
get => this.speed;
init
{
if (value is < HeifEncodingSpeed.Level0 or > HeifEncodingSpeed.Level9)
{
throw new ArgumentException("Speed must be in the range [0..9].");
}
this.speed = value;
}
}
/// <summary> /// <summary>
/// Gets the encoded precision of each image component, or <see langword="null"/> to use the HEIF metadata bit /// Gets the encoded precision of each image component, or <see langword="null"/> to use the HEIF metadata bit
/// depth. Metadata that does not specify a bit depth defaults to <see cref="HeifBitDepth.Bit8"/>. Legacy JPEG /// depth. Metadata that does not specify a bit depth defaults to <see cref="HeifBitDepth.Bit8"/>. Legacy JPEG

6
src/ImageSharp/Formats/Heif/HeifEncoderCore.Sequence.cs

@ -235,7 +235,8 @@ internal sealed partial class HeifEncoderCore
image.Height, image.Height,
settings.ColorConfig, settings.ColorConfig,
settings.ColorQIndex, settings.ColorQIndex,
this.encoder.Effort)) this.encoder.Effort,
speed: this.encoder.Speed))
{ {
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
long colorOffset = stream.Length; long colorOffset = stream.Length;
@ -292,7 +293,8 @@ internal sealed partial class HeifEncoderCore
image.Height, image.Height,
settings.AlphaConfig, settings.AlphaConfig,
settings.AlphaQIndex, settings.AlphaQIndex,
this.encoder.Effort)) this.encoder.Effort,
speed: this.encoder.Speed))
{ {
cancellationToken.ThrowIfCancellationRequested(); cancellationToken.ThrowIfCancellationRequested();
long alphaOffset = stream.Length; long alphaOffset = stream.Length;

13
tests/ImageSharp.Benchmarks/Codecs/Heif/Av1SequenceEncoderBenchmarks.cs

@ -3,6 +3,7 @@
using System.Numerics; using System.Numerics;
using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Attributes;
using SixLabors.ImageSharp.Formats.Heif;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
@ -106,7 +107,7 @@ public class Av1SequenceEncoderBenchmarks
// Export the production-converted source planes only for checking reconstructed output quality. // Export the production-converted source planes only for checking reconstructed output quality.
// Neither timed encoder reads this file: both convert the original RGB frames during each operation. // Neither timed encoder reads this file: both convert the original RGB frames during each operation.
using Av1EncoderFrameBuffer<byte> planar = new(this.configuration, this.Dimension, this.Dimension, 8, Av1ColorFormat.Yuv420, 0, 0); using Av1EncoderFrameBuffer<byte> planar = new(this.configuration, this.Dimension, this.Dimension, 8, Av1ColorFormat.Yuv420, 0, 0, lumaBorder: 64);
using FileStream raw = File.Create(Path.Combine(this.outputDirectory, $"bike-{this.Dimension}-3frames.source.yuv")); using FileStream raw = File.Create(Path.Combine(this.outputDirectory, $"bike-{this.Dimension}-3frames.source.yuv"));
foreach (ImageFrame<Rgb24> frame in this.sequence.Frames) foreach (ImageFrame<Rgb24> frame in this.sequence.Frames)
{ {
@ -131,7 +132,13 @@ public class Av1SequenceEncoderBenchmarks
{ {
this.output.SetLength(0); this.output.SetLength(0);
using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder(
this.configuration, this.Dimension, this.Dimension, this.colorConfig, QIndex, this.Effort); this.configuration,
this.Dimension,
this.Dimension,
this.colorConfig,
QIndex,
this.Effort,
speed: HeifEncodingSpeed.Level0);
// One operation owns the real sequence lifetime: allocation, conversion, key/inter coding, and disposal. // One operation owns the real sequence lifetime: allocation, conversion, key/inter coding, and disposal.
// The caller's destination is reused, excluding filesystem and MemoryStream growth from steady-state timing. // The caller's destination is reused, excluding filesystem and MemoryStream growth from steady-state timing.
@ -155,7 +162,7 @@ public class Av1SequenceEncoderBenchmarks
// good-quality speed six while comparing the three managed interpolation-search boundaries. // good-quality speed six while comparing the three managed interpolation-search boundaries.
this.output.SetLength(0); this.output.SetLength(0);
using LibaomBenchmarkEncoder encoder = LibaomBenchmarkEncoder.Open(this.Dimension, this.Dimension, NativeQuality, NativeCpuUsed); using LibaomBenchmarkEncoder encoder = LibaomBenchmarkEncoder.Open(this.Dimension, this.Dimension, NativeQuality, NativeCpuUsed);
using Av1EncoderFrameBuffer<byte> planar = new(this.configuration, this.Dimension, this.Dimension, 8, Av1ColorFormat.Yuv420, 0, 0); using Av1EncoderFrameBuffer<byte> planar = new(this.configuration, this.Dimension, this.Dimension, 8, Av1ColorFormat.Yuv420, 0, 0, lumaBorder: 64);
using Av1FrameEncoder.Av1EncoderConversionWorkspace conversion = new(this.configuration, this.Dimension, this.colorConfig, false, false); using Av1FrameEncoder.Av1EncoderConversionWorkspace conversion = new(this.configuration, this.Dimension, this.colorConfig, false, false);
Rectangle bounds = new(0, 0, this.Dimension, this.Dimension); Rectangle bounds = new(0, 0, this.Dimension, this.Dimension);
for (int frameIndex = 0; frameIndex < FrameCount; frameIndex++) for (int frameIndex = 0; frameIndex < FrameCount; frameIndex++)

169
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

@ -3,6 +3,7 @@
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Heif;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
@ -316,14 +317,16 @@ public class Av1EncoderFrameTests
Height, Height,
colorConfig, colorConfig,
qIndex: 37, qIndex: 37,
effort: 5) effort: 5,
speed: HeifEncodingSpeed.Level0)
: Av1FrameEncoder.CreateColorSequenceEncoder( : Av1FrameEncoder.CreateColorSequenceEncoder(
Configuration.Default, Configuration.Default,
Width, Width,
Height, Height,
colorConfig, colorConfig,
qIndex: 37, qIndex: 37,
effort: 5); effort: 5,
speed: HeifEncodingSpeed.Level0);
encoder.EncodeKeyFrame(source.Frames.RootFrame, stream); encoder.EncodeKeyFrame(source.Frames.RootFrame, stream);
ObuSequenceHeader encodedHeader = encoder.SequenceHeader; ObuSequenceHeader encodedHeader = encoder.SequenceHeader;
@ -345,19 +348,50 @@ public class Av1EncoderFrameTests
/// Verifies dependent color samples with odd visible dimensions and motion across subsampled chroma phases. /// Verifies dependent color samples with odd visible dimensions and motion across subsampled chroma phases.
/// </summary> /// </summary>
[Theory] [Theory]
[InlineData(EightBit, Yuv420, 8)] [InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level0)]
[InlineData(TenBit, Yuv420, 8)] [InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level0)]
[InlineData(TwelveBit, Yuv420, 8)] [InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level0)]
[InlineData(EightBit, Yuv420, 9)] [InlineData(EightBit, Yuv420, 9, HeifEncodingSpeed.Level0)]
[InlineData(TenBit, Yuv420, 9)] [InlineData(TenBit, Yuv420, 9, HeifEncodingSpeed.Level0)]
[InlineData(TwelveBit, Yuv420, 9)] [InlineData(TwelveBit, Yuv420, 9, HeifEncodingSpeed.Level0)]
[InlineData(EightBit, Yuv422, 9)] [InlineData(EightBit, Yuv422, 9, HeifEncodingSpeed.Level0)]
[InlineData(TenBit, Yuv422, 9)] [InlineData(TenBit, Yuv422, 9, HeifEncodingSpeed.Level0)]
[InlineData(TwelveBit, Yuv422, 9)] [InlineData(TwelveBit, Yuv422, 9, HeifEncodingSpeed.Level0)]
[InlineData(EightBit, Yuv444, 9)] [InlineData(EightBit, Yuv444, 9, HeifEncodingSpeed.Level0)]
[InlineData(TenBit, Yuv444, 9)] [InlineData(TenBit, Yuv444, 9, HeifEncodingSpeed.Level0)]
[InlineData(TwelveBit, Yuv444, 9)] [InlineData(TwelveBit, Yuv444, 9, HeifEncodingSpeed.Level0)]
public void SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion(int bitDepthValue, int colorFormatValue, int effort) [InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level1)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level2)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level3)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level4)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level5)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level6)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level7)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level8)]
[InlineData(EightBit, Yuv420, 8, HeifEncodingSpeed.Level9)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level1)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level2)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level3)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level4)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level5)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level6)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level7)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level8)]
[InlineData(TenBit, Yuv420, 8, HeifEncodingSpeed.Level9)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level1)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level2)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level3)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level4)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level5)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level6)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level7)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level8)]
[InlineData(TwelveBit, Yuv420, 8, HeifEncodingSpeed.Level9)]
public void SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion(
int bitDepthValue,
int colorFormatValue,
int effort,
HeifEncodingSpeed speed)
{ {
const int Width = 23; const int Width = 23;
const int Height = 19; const int Height = 19;
@ -369,15 +403,21 @@ public class Av1EncoderFrameTests
ReadOnlySpan<int> period = [0, 28, 40, 28, 0, -28, -40, -12]; ReadOnlySpan<int> period = [0, 28, 40, 28, 0, -28, -40, -12];
using Image<Rgb48> source = new(Width, Height); using Image<Rgb48> source = new(Width, Height);
using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder(
Configuration.Default, Width, Height, colorConfig, QIndex, effort); Configuration.Default,
Width,
Height,
colorConfig,
QIndex,
effort,
speed);
string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion)); string outputDirectory = TestEnvironment.CreateOutputDirectory("Heif", "Av1", nameof(this.SequenceEncoderPreservesNativeColorPlanesWithSubpixelMotion));
string outputName = $"{bitDepth.GetBitCount()}-{colorFormat}-effort{effort}"; string outputName = $"{bitDepth.GetBitCount()}-{colorFormat}-effort{effort}-speed{(int)speed}";
using FileStream output = File.Create(Path.Combine(outputDirectory, outputName + ".obu")); using FileStream output = File.Create(Path.Combine(outputDirectory, outputName + ".obu"));
using BinaryWriter rawOutput = new(File.Create(Path.Combine(outputDirectory, outputName + ".managed.yuv"))); using BinaryWriter rawOutput = new(File.Create(Path.Combine(outputDirectory, outputName + ".managed.yuv")));
using Av1Decoder decoder = new(Configuration.Default); using Av1Decoder decoder = new(Configuration.Default);
using MemoryStream sample = new(); using MemoryStream sample = new();
for (int frameIndex = 0; frameIndex < 2; frameIndex++) for (int frameIndex = 0; frameIndex < 3; frameIndex++)
{ {
// The second source translates all three channels by one luma sample on each axis. Chroma is // The second source translates all three channels by one luma sample on each axis. Chroma is
// converted independently by the production converter, so 4:2:0 and 4:2:2 cannot hide behind // converted independently by the production converter, so 4:2:0 and 4:2:2 cannot hide behind
@ -504,7 +544,8 @@ public class Av1EncoderFrameTests
Height, Height,
colorConfig, colorConfig,
qIndex: 37, qIndex: 37,
effort); effort,
speed: HeifEncodingSpeed.Level0);
encoder.EncodeKeyFrame(source.Frames.RootFrame, firstSample); encoder.EncodeKeyFrame(source.Frames.RootFrame, firstSample);
encoder.EncodeInterFrame(source.Frames.RootFrame, secondSample); encoder.EncodeInterFrame(source.Frames.RootFrame, secondSample);
@ -601,7 +642,8 @@ public class Av1EncoderFrameTests
Height, Height,
colorConfig, colorConfig,
qIndex: 4, qIndex: 4,
effort: 6); effort: 6,
speed: HeifEncodingSpeed.Level0);
encoder.EncodeKeyFrame(first.Frames.RootFrame, firstSample); encoder.EncodeKeyFrame(first.Frames.RootFrame, firstSample);
encoder.EncodeInterFrame(second.Frames.RootFrame, secondSample); encoder.EncodeInterFrame(second.Frames.RootFrame, secondSample);
@ -649,7 +691,13 @@ public class Av1EncoderFrameTests
Assert.Throws<InvalidImageContentException>(() => Assert.Throws<InvalidImageContentException>(() =>
{ {
using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder( using Av1FrameEncoder.SequenceEncoder encoder = Av1FrameEncoder.CreateColorSequenceEncoder(
configuration, 32, 32, colorConfig, 17, 9); configuration,
32,
32,
colorConfig,
17,
9,
speed: HeifEncodingSpeed.Level0);
}); });
Assert.Empty(allocator.AllocationLog); Assert.Empty(allocator.AllocationLog);
@ -674,8 +722,8 @@ public class Av1EncoderFrameTests
successfulAllocator.EnableNonThreadSafeLogging(); successfulAllocator.EnableNonThreadSafeLogging();
configuration.MemoryAllocator = successfulAllocator; configuration.MemoryAllocator = successfulAllocator;
using (Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha using (Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha
? Av1FrameEncoder.CreateAlphaSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9) ? Av1FrameEncoder.CreateAlphaSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9, speed: HeifEncodingSpeed.Level0)
: Av1FrameEncoder.CreateColorSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9)) : Av1FrameEncoder.CreateColorSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9, speed: HeifEncodingSpeed.Level0))
{ {
Assert.NotEmpty(successfulAllocator.AllocationLog); Assert.NotEmpty(successfulAllocator.AllocationLog);
} }
@ -691,8 +739,8 @@ public class Av1EncoderFrameTests
InvalidMemoryOperationException exception = Assert.Throws<InvalidMemoryOperationException>(() => InvalidMemoryOperationException exception = Assert.Throws<InvalidMemoryOperationException>(() =>
{ {
using Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha using Av1FrameEncoder.SequenceEncoder encoder = encodeAlpha
? Av1FrameEncoder.CreateAlphaSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9) ? Av1FrameEncoder.CreateAlphaSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9, speed: HeifEncodingSpeed.Level0)
: Av1FrameEncoder.CreateColorSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9); : Av1FrameEncoder.CreateColorSequenceEncoder(configuration, 32, 32, colorConfig, 17, 9, speed: HeifEncodingSpeed.Level0);
}); });
Assert.Equal("Sequence allocation failure.", exception.Message); Assert.Equal("Sequence allocation failure.", exception.Message);
@ -739,14 +787,16 @@ public class Av1EncoderFrameTests
Height, Height,
colorConfig, colorConfig,
qIndex: 37, qIndex: 37,
effort: 6) effort: 6,
speed: HeifEncodingSpeed.Level0)
: Av1FrameEncoder.CreateColorSequenceEncoder( : Av1FrameEncoder.CreateColorSequenceEncoder(
configuration, configuration,
Width, Width,
Height, Height,
colorConfig, colorConfig,
qIndex: 37, qIndex: 37,
effort: 6)) effort: 6,
speed: HeifEncodingSpeed.Level0))
{ {
rowStorage = Assert.Single( rowStorage = Assert.Single(
allocator.AllocationLog, allocator.AllocationLog,
@ -808,7 +858,8 @@ public class Av1EncoderFrameTests
bitDepth.GetBitCount(), bitDepth.GetBitCount(),
Av1ColorFormat.Yuv444, Av1ColorFormat.Yuv444,
1, 1,
1); 1,
lumaBorder: 64);
Av1FrameEncoder.PrepareSource( Av1FrameEncoder.PrepareSource(
Configuration.Default, Configuration.Default,
@ -1268,7 +1319,8 @@ public class Av1EncoderFrameTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
@ -1339,7 +1391,8 @@ public class Av1EncoderFrameTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
for (int row = 0; row < height; row++) for (int row = 0; row < height; row++)
{ {
@ -1384,7 +1437,8 @@ public class Av1EncoderFrameTests
10, 10,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
for (int row = 0; row < 16; row++) for (int row = 0; row < 16; row++)
{ {
@ -1448,7 +1502,8 @@ public class Av1EncoderFrameTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> luma = frame.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> luma = frame.Frame.View.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++) for (int row = 0; row < Height; row++)
@ -1762,7 +1817,8 @@ public class Av1EncoderFrameTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit); ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit);
@ -1792,7 +1848,8 @@ public class Av1EncoderFrameTests
10, 10,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.TenBit); ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.TenBit);
@ -1802,13 +1859,15 @@ public class Av1EncoderFrameTests
AssertReplicatedSingleRow(frameBuffer.Luma, border, expected); AssertReplicatedSingleRow(frameBuffer.Luma, border, expected);
} }
[Fact] [Theory]
public void ExtendBordersReplicatesEveryPhysicalPlaneEdge() [InlineData(64)]
[InlineData(96)]
[InlineData(160)]
public void ExtendBordersReplicatesEveryPhysicalPlaneEdge(int lumaBorder)
{ {
const int visibleWidth = 5; const int visibleWidth = 5;
const int visibleHeight = 3; const int visibleHeight = 3;
const int lumaBorder = Av1EncoderFrame<byte>.LumaBorder; int chromaBorder = lumaBorder / 2;
const int chromaBorder = lumaBorder / 2;
using Av1EncoderFrameBuffer<byte> frameBuffer = new( using Av1EncoderFrameBuffer<byte> frameBuffer = new(
Configuration.Default, Configuration.Default,
@ -1817,7 +1876,8 @@ public class Av1EncoderFrameTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder);
Buffer2D<byte> luma = frameBuffer.Luma; Buffer2D<byte> luma = frameBuffer.Luma;
Buffer2D<byte> chromaBlue = Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaBlue); Buffer2D<byte> chromaBlue = Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaBlue);
@ -1835,12 +1895,19 @@ public class Av1EncoderFrameTests
} }
[Theory] [Theory]
[InlineData(5, 3, 0, 0, 160, 136)] [InlineData(64, 5, 3, 0, 0, 160, 136)]
[InlineData(5, 3, 1, 0, 80, 136)] [InlineData(64, 5, 3, 1, 0, 80, 136)]
[InlineData(5, 3, 1, 1, 80, 68)] [InlineData(64, 5, 3, 1, 1, 80, 68)]
[InlineData(1921, 1081, 0, 0, 2080, 1216)] [InlineData(64, 1921, 1081, 0, 0, 2080, 1216)]
[InlineData(1921, 1081, 1, 1, 1040, 608)] [InlineData(64, 1921, 1081, 1, 1, 1040, 608)]
[InlineData(96, 5, 3, 0, 0, 224, 200)]
[InlineData(96, 5, 3, 1, 0, 112, 200)]
[InlineData(96, 5, 3, 1, 1, 112, 100)]
[InlineData(160, 5, 3, 0, 0, 352, 328)]
[InlineData(160, 5, 3, 1, 0, 176, 328)]
[InlineData(160, 5, 3, 1, 1, 176, 164)]
public void GetPlaneBufferSizeMatchesLibaomLayout( public void GetPlaneBufferSizeMatchesLibaomLayout(
int lumaBorder,
int width, int width,
int height, int height,
int subsamplingX, int subsamplingX,
@ -1848,13 +1915,16 @@ public class Av1EncoderFrameTests
int expectedWidth, int expectedWidth,
int expectedHeight) int expectedHeight)
{ {
Size actual = Av1EncoderFrame<byte>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY); Size actual = Av1EncoderFrame<byte>.GetPlaneBufferSize(width, height, subsamplingX, subsamplingY, lumaBorder);
Assert.Equal(new Size(expectedWidth, expectedHeight), actual); Assert.Equal(new Size(expectedWidth, expectedHeight), actual);
} }
[Fact] [Theory]
public void FrameBufferUsesOneExactSizeOwnerForAllPlanes() [InlineData(64, 55_296)]
[InlineData(96, 98_304)]
[InlineData(160, 221_184)]
public void FrameBufferUsesOneExactSizeOwnerForAllPlanes(int lumaBorder, int expectedLength)
{ {
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
@ -1869,12 +1939,13 @@ public class Av1EncoderFrameTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1)) 1,
lumaBorder))
{ {
allocation = Assert.Single(allocator.AllocationLog); allocation = Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
Assert.Equal(typeof(byte), allocation.ElementType); Assert.Equal(typeof(byte), allocation.ElementType);
Assert.Equal(55_296, allocation.Length); Assert.Equal(expectedLength, allocation.Length);
Assert.Single(frameBuffer.Luma.MemoryGroup); Assert.Single(frameBuffer.Luma.MemoryGroup);
Assert.Single(Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaBlue).MemoryGroup); Assert.Single(Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaBlue).MemoryGroup);
Assert.Single(Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaRed).MemoryGroup); Assert.Single(Assert.IsType<Buffer2D<byte>>(frameBuffer.ChromaRed).MemoryGroup);

84
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderModeInfoBufferTests.cs

@ -1,6 +1,7 @@
// Copyright (c) Six Labors. // Copyright (c) Six Labors.
// Licensed under the Six Labors Split License. // Licensed under the Six Labors Split License.
using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion; using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
@ -61,6 +62,21 @@ public class Av1EncoderModeInfoBufferTests
// Four CDEF presets, the preceding quantizer, and two payload bounds follow the context regions. // Four CDEF presets, the preceding quantizer, and two payload bounds follow the context regions.
const int TileStateStorageLength = 7 * sizeof(int); const int TileStateStorageLength = 7 * sizeof(int);
const int AllocatedBlockCount = 256;
const int BlockEncodingStorageLength = AllocatedBlockCount * 8;
const int BlockPaletteStorageLength = AllocatedBlockCount * 50;
const int PaletteTokenStorageLength = 2 * 128 * 128;
// Four vectors (16 bytes), four weights (8), mode context (2), count (1), and one alignment byte.
const int ReferenceContextStorageLength = AllocatedBlockCount * 28;
// Retained syntax uses fixed-width entries at the existing block origins. Palette tokens reserve
// two complete maximum-superblock planes, including coded padding beyond this small visible frame.
int retainedStorageLength = BlockEncodingStorageLength +
(allowScreenContentTools ? BlockPaletteStorageLength + PaletteTokenStorageLength : 0) +
(allowIntraBlockCopy ? ReferenceContextStorageLength : 0);
int expectedTileStateOffset = expectedContextStorageLength + retainedStorageLength;
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
Configuration configuration = Configuration.Default.Clone(); Configuration configuration = Configuration.Default.Clone();
@ -111,11 +127,15 @@ public class Av1EncoderModeInfoBufferTests
Assert.Equal(6_144, allocations[0].Length); Assert.Equal(6_144, allocations[0].Length);
Assert.Equal(AllocationOptions.Clean, allocations[0].AllocationOptions); Assert.Equal(AllocationOptions.Clean, allocations[0].AllocationOptions);
Assert.Equal(typeof(byte), allocations[1].ElementType); Assert.Equal(typeof(byte), allocations[1].ElementType);
Assert.Equal(expectedContextStorageLength + TileStateStorageLength, allocations[1].Length); Assert.Equal(expectedTileStateOffset + TileStateStorageLength, allocations[1].Length);
Assert.Equal(AllocationOptions.Clean, allocations[1].AllocationOptions); Assert.Equal(AllocationOptions.Clean, allocations[1].AllocationOptions);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
Av1PictureControlSet picture = buffer.Picture; Av1PictureControlSet picture = buffer.Picture;
Assert.Equal(AllocatedBlockCount, picture.BlockEncodings.Length);
Assert.Equal(8, sizeof(Av1EncoderBlockStruct));
Assert.Equal(28, sizeof(Av1EncoderReferenceContext));
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.BlockEncodings.Span).IndexOfAnyExcept((byte)0));
Assert.Equal(16, picture.SegmentationNeighborMap.Length); Assert.Equal(16, picture.SegmentationNeighborMap.Length);
Assert.Equal(32, picture.PartitionContexts[0].Left.Length); Assert.Equal(32, picture.PartitionContexts[0].Left.Length);
Assert.Equal(32, picture.PartitionContexts[0].Top.Length); Assert.Equal(32, picture.PartitionContexts[0].Top.Length);
@ -140,7 +160,7 @@ public class Av1EncoderModeInfoBufferTests
lengths = picture.TileDataLengths.Span) lengths = picture.TileDataLengths.Span)
{ {
Assert.Equal((nuint)0, (nuint)cdef % (nuint)sizeof(int)); Assert.Equal((nuint)0, (nuint)cdef % (nuint)sizeof(int));
Assert.Equal(expectedContextStorageLength, (byte*)cdef - state); Assert.Equal(expectedTileStateOffset, (byte*)cdef - state);
Assert.Equal(4, quantizer - cdef); Assert.Equal(4, quantizer - cdef);
Assert.Equal(1, offsets - quantizer); Assert.Equal(1, offsets - quantizer);
Assert.Equal(1, lengths - offsets); Assert.Equal(1, lengths - offsets);
@ -150,6 +170,22 @@ public class Av1EncoderModeInfoBufferTests
if (allowScreenContentTools) if (allowScreenContentTools)
{ {
Assert.Equal(AllocatedBlockCount, picture.BlockPalettes.Length);
Assert.Equal(PaletteTokenStorageLength, picture.PaletteTokens.Length);
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.BlockPalettes.Span).IndexOfAnyExcept((byte)0));
Assert.Equal(0, picture.PaletteTokens.Span[^1]);
fixed (byte* tokens = picture.PaletteTokens.Span)
{
fixed (Av1EncoderPaletteInfo* palettes = picture.BlockPalettes.Span)
{
fixed (Av1EncoderBlockStruct* encodings = picture.BlockEncodings.Span)
{
Assert.Equal(BlockPaletteStorageLength, tokens - (byte*)palettes);
Assert.Equal(PaletteTokenStorageLength, (byte*)encodings - tokens);
}
}
}
Av1NeighborArrayUnit<Av1EncoderPaletteInfo> paletteContext = Assert.Single(picture.PaletteContexts); Av1NeighborArrayUnit<Av1EncoderPaletteInfo> paletteContext = Assert.Single(picture.PaletteContexts);
Assert.Equal(32, paletteContext.Left.Length); Assert.Equal(32, paletteContext.Left.Length);
Assert.Equal(32, paletteContext.Top.Length); Assert.Equal(32, paletteContext.Top.Length);
@ -169,10 +205,26 @@ public class Av1EncoderModeInfoBufferTests
else else
{ {
Assert.Empty(picture.PaletteContexts); Assert.Empty(picture.PaletteContexts);
Assert.True(picture.BlockPalettes.IsEmpty);
Assert.True(picture.PaletteTokens.IsEmpty);
} }
if (allowIntraBlockCopy) if (allowIntraBlockCopy)
{ {
Assert.Equal(AllocatedBlockCount, picture.ReferenceContexts.Length);
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.ReferenceContexts.Span).IndexOfAnyExcept((byte)0));
fixed (Av1EncoderDisplacementVector* vectors = picture.DisplacementVectors.Span)
{
fixed (Av1EncoderReferenceContext* references = picture.ReferenceContexts.Span)
{
fixed (Av1EncoderBlockStruct* encodings = picture.BlockEncodings.Span)
{
Assert.Equal(BlockEncodingStorageLength, (byte*)vectors - (byte*)encodings);
Assert.Equal(AllocatedBlockCount * 4, (byte*)references - (byte*)vectors);
}
}
}
Assert.Equal(256, picture.DisplacementVectors.Length); Assert.Equal(256, picture.DisplacementVectors.Length);
Assert.Equal(4, sizeof(Av1EncoderDisplacementVector)); Assert.Equal(4, sizeof(Av1EncoderDisplacementVector));
Assert.Equal(9, picture.IntraBlockCopySearch.OriginWidth); Assert.Equal(9, picture.IntraBlockCopySearch.OriginWidth);
@ -193,6 +245,7 @@ public class Av1EncoderModeInfoBufferTests
else else
{ {
Assert.Equal(0, picture.DisplacementVectors.Length); Assert.Equal(0, picture.DisplacementVectors.Length);
Assert.True(picture.ReferenceContexts.IsEmpty);
} }
} }
@ -313,6 +366,10 @@ public class Av1EncoderModeInfoBufferTests
picture.TransformFunctionContexts[0].Top[0] = 8; picture.TransformFunctionContexts[0].Top[0] = 8;
picture.PaletteContexts[0].Left[0].PaletteSizes[0] = 2; picture.PaletteContexts[0].Left[0].PaletteSizes[0] = 2;
picture.DisplacementVectors.Span[0] = new Av1EncoderDisplacementVector { Row = -8, Column = 16 }; picture.DisplacementVectors.Span[0] = new Av1EncoderDisplacementVector { Row = -8, Column = 16 };
picture.BlockEncodings.Span[0].QuantizationIndex = 53;
picture.BlockPalettes.Span[0].PaletteSizes[0] = 3;
picture.PaletteTokens.Span[0] = 0x42;
picture.ReferenceContexts.Span[0].ModeContext = 37;
picture.CdefPreset.Span[0] = 2; picture.CdefPreset.Span[0] = 2;
picture.Parent.PreviousQIndex.Span[0] = InitialQIndex + 1; picture.Parent.PreviousQIndex.Span[0] = InitialQIndex + 1;
picture.TileDataOffsets.Span[0] = 11; picture.TileDataOffsets.Span[0] = 11;
@ -345,12 +402,35 @@ public class Av1EncoderModeInfoBufferTests
Assert.Equal(Av1Constants.MaxTransformSize, picture.TransformFunctionContexts[0].Top[0]); Assert.Equal(Av1Constants.MaxTransformSize, picture.TransformFunctionContexts[0].Top[0]);
Assert.Equal(0, picture.PaletteContexts[0].Left[0].PaletteSizes[0]); Assert.Equal(0, picture.PaletteContexts[0].Left[0].PaletteSizes[0]);
Assert.Equal(default, picture.DisplacementVectors.Span[0]); Assert.Equal(default, picture.DisplacementVectors.Span[0]);
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.BlockEncodings.Span).IndexOfAnyExcept((byte)0));
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.BlockPalettes.Span).IndexOfAnyExcept((byte)0));
Assert.Equal(0, picture.PaletteTokens.Span[0]);
Assert.Equal(-1, MemoryMarshal.AsBytes(picture.ReferenceContexts.Span).IndexOfAnyExcept((byte)0));
Assert.Equal(-1, picture.CdefPreset.Span[0]); Assert.Equal(-1, picture.CdefPreset.Span[0]);
Assert.Equal(NextQIndex, picture.Parent.PreviousQIndex.Span[0]); Assert.Equal(NextQIndex, picture.Parent.PreviousQIndex.Span[0]);
Assert.Equal(0, picture.TileDataOffsets.Span[0]); Assert.Equal(0, picture.TileDataOffsets.Span[0]);
Assert.Equal(0, picture.TileDataLengths.Span[0]); Assert.Equal(0, picture.TileDataLengths.Span[0]);
Assert.Same(nextFrameHeader, picture.Parent.FrameHeader); Assert.Same(nextFrameHeader, picture.Parent.FrameHeader);
Assert.Same(nextTiles, picture.Parent.Common.TilesInfo); Assert.Same(nextTiles, picture.Parent.Common.TilesInfo);
picture.ModeInfoAllocation.Span[0].Block.Mode = Av1PredictionMode.Paeth;
picture.BlockEncodings.Span[0].QuantizationIndex = 53;
picture.BlockPalettes.Span[0].PaletteSizes[0] = 3;
picture.PaletteTokens.Span[0] = 0x42;
picture.ReferenceContexts.Span[0].ModeContext = 37;
picture.LuminanceDcSignLevelCoefficientNeighbors[0].Top[0] = 0x41;
picture.TransformFunctionContexts[0].Left[0] = 4;
picture.ResetEntropyContexts();
Assert.Equal(allocationCount, allocator.AllocationLog.Count);
Assert.Empty(allocator.ReturnLog);
Assert.Equal(Av1PredictionMode.Paeth, picture.ModeInfoAllocation.Span[0].Block.Mode);
Assert.Equal(53, picture.BlockEncodings.Span[0].QuantizationIndex);
Assert.Equal(3, picture.BlockPalettes.Span[0].PaletteSizes[0]);
Assert.Equal(0x42, picture.PaletteTokens.Span[0]);
Assert.Equal(37, picture.ReferenceContexts.Span[0].ModeContext);
Assert.Equal(0, picture.LuminanceDcSignLevelCoefficientNeighbors[0].Top[0]);
Assert.Equal(Av1Constants.MaxTransformSize, picture.TransformFunctionContexts[0].Left[0]);
} }
[Theory] [Theory]

108
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraBlockCopyTests.cs

@ -266,7 +266,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -275,7 +276,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y);
Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y);
@ -352,7 +354,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -361,7 +364,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y);
Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y);
@ -448,7 +452,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -457,7 +462,8 @@ public class Av1IntraBlockCopyTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y);
Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y);
@ -501,6 +507,90 @@ public class Av1IntraBlockCopyTests
Assert.Equal(2048, candidates[0].Column); Assert.Equal(2048, candidates[0].Column);
} }
/// <summary>
/// Checks that high-bit-depth pixel search trades prediction error against motion rate in one common scale.
/// </summary>
/// <param name="bits">The coded sample precision.</param>
[Theory]
[InlineData(10)]
[InlineData(12)]
public void PixelSearchNormalizesSadBeforeComparingMotionRate(int bits)
{
const int Width = 640;
const int Height = 256;
const int QIndex = 90;
int scale = 1 << (bits - 8);
Point blockOrigin = new(0, 128);
Point predictionOrigin = new(15, 120);
Av1MotionVector reference = new(-64, 120);
ObuSequenceHeader sequenceHeader = CreateSequenceHeader();
sequenceHeader.ColorConfig.BitDepth = bits == 10 ? Av1BitDepth.TenBit : Av1BitDepth.TwelveBit;
ObuFrameHeader frameHeader = CreateFrameHeader();
frameHeader.AllowScreenContentTools = true;
frameHeader.AllowIntraBlockCopy = true;
using Av1EncoderPictureBuffer pictureBuffer = new(
Configuration.Default,
sequenceHeader,
frameHeader,
Width,
Height,
disallow4x4AllFrames: true);
using Av1EncoderFrameBuffer<ushort> source = new(
Configuration.Default, Width, Height, bits, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<ushort> reconstruction = new(
Configuration.Default, Width, Height, bits, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
Buffer2DRegion<ushort> sourceLuma = source.Frame.CodedView.GetPlane(Av1Plane.Y);
Buffer2DRegion<ushort> reconstructedLuma = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++)
{
sourceLuma.DangerousGetRowSpan(row).Clear();
reconstructedLuma.DangerousGetRowSpan(row).Clear();
}
// The reference candidate differs by one eight-bit unit in its first column. Moving right one pixel
// removes that error but adds motion syntax. Raw high-bit-depth SAD would overvalue that small gain.
for (int row = 0; row < 8; row++)
{
sourceLuma.DangerousGetRowSpan(blockOrigin.Y + row).Slice(blockOrigin.X, 8).Fill((ushort)(100 * scale));
reconstructedLuma.DangerousGetRowSpan(predictionOrigin.Y + row).Slice(predictionOrigin.X, 9).Fill((ushort)(100 * scale));
reconstructedLuma.DangerousGetRowSpan(predictionOrigin.Y + row)[predictionOrigin.X] = (ushort)(101 * scale);
}
using Av1SymbolEncoder writer = new(Configuration.Default, 64, QIndex, updateCdf: true);
Span<Av1MotionVector> candidates = stackalloc Av1MotionVector[2];
for (int i = 0; i < 64; i++)
{
// Repeated use of the spatial reference makes a new differential vector appreciably more costly.
writer.WriteDisplacementVector(reference, reference);
}
int sadPerBit = Av1RateDistortion.GetMotionSearchSadPerBit(QIndex, sequenceHeader.ColorConfig.BitDepth);
int referenceRate = writer.GetDisplacementVectorSearchCost(reference, reference);
int adjacentRate = writer.GetDisplacementVectorSearchCost(new Av1MotionVector(-64, 128), reference);
int referenceMotionCost = ((referenceRate * sadPerBit) + 256) >> 9;
int adjacentMotionCost = ((adjacentRate * sadPerBit) + 256) >> 9;
Assert.True(8 + referenceMotionCost < adjacentMotionCost);
Assert.True((8 * scale) + referenceMotionCost > adjacentMotionCost);
int count = pictureBuffer.Picture.IntraBlockCopySearch.FindPixelCandidates<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
sourceLuma,
reconstructedLuma,
blockOrigin,
new Av1TileInfo(0, 0, frameHeader),
sequenceHeader,
writer,
reference,
QIndex,
Av1RateDistortion.GetKeyFrameRateMultiplier(QIndex, sequenceHeader.ColorConfig.BitDepth),
candidates);
Assert.Equal(1, count);
Assert.Equal(reference, candidates[0]);
}
/// <summary> /// <summary>
/// Verifies high-bit-depth SIMD variance normalization against the eight-bit search domain. /// Verifies high-bit-depth SIMD variance normalization against the eight-bit search domain.
/// </summary> /// </summary>
@ -515,7 +605,8 @@ public class Av1IntraBlockCopyTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<ushort> reconstruction = new( using Av1EncoderFrameBuffer<ushort> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -524,7 +615,8 @@ public class Av1IntraBlockCopyTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<ushort> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<ushort> sourceLuma = source.Frame.View.GetPlane(Av1Plane.Y);
Buffer2DRegion<ushort> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y); Buffer2DRegion<ushort> reconstructionLuma = reconstruction.Frame.View.GetPlane(Av1Plane.Y);

640
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

@ -4,8 +4,10 @@
using System.Buffers; using System.Buffers;
using System.Numerics; using System.Numerics;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SixLabors.ImageSharp.Formats.Heif;
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy; using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers;
@ -69,9 +71,9 @@ public class Av1IntraSuperblockEncoderTests
}; };
using Image<L8> referenceImage = new(Width, Height); using Image<L8> referenceImage = new(Width, Height);
using Av1EncoderFrameBuffer<byte> reference = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<byte> reference = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> source = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<byte> source = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<byte> reconstruction = new(configuration, Width, Height, 8, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
for (int y = 0; y < Height; y++) for (int y = 0; y < Height; y++)
{ {
Span<L8> pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); Span<L8> pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
@ -98,7 +100,8 @@ public class Av1IntraSuperblockEncoderTests
Height, Height,
colorConfig, colorConfig,
qIndex: 0, qIndex: 0,
effort); effort,
speed: HeifEncodingSpeed.Level0);
keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample); keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample);
ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader; ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader;
@ -125,7 +128,7 @@ public class Av1IntraSuperblockEncoderTests
using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true); using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true);
using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height); using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration);
using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); using Av1EncoderBlockWorkspace blockWorkspace = new(configuration, allocateInterMotionCosts: true);
using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true); using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true);
Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace); Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace);
int allocationCount = allocator.AllocationLog.Count; int allocationCount = allocator.AllocationLog.Count;
@ -240,9 +243,9 @@ public class Av1IntraSuperblockEncoderTests
}; };
using Image<L16> referenceImage = new(Width, Height); using Image<L16> referenceImage = new(Width, Height);
using Av1EncoderFrameBuffer<ushort> reference = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<ushort> reference = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<ushort> source = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<ushort> source = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<ushort> reconstruction = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<ushort> reconstruction = new(configuration, Width, Height, bitDepth, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
for (int y = 0; y < Height; y++) for (int y = 0; y < Height; y++)
{ {
Span<L16> pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y); Span<L16> pixels = referenceImage.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(y);
@ -266,7 +269,13 @@ public class Av1IntraSuperblockEncoderTests
ClearPlane(reconstruction.Luma); ClearPlane(reconstruction.Luma);
using MemoryStream firstSample = new(); using MemoryStream firstSample = new();
using Av1FrameEncoder.SequenceEncoder keyEncoder = Av1FrameEncoder.CreateColorSequenceEncoder( using Av1FrameEncoder.SequenceEncoder keyEncoder = Av1FrameEncoder.CreateColorSequenceEncoder(
configuration, Width, Height, colorConfig, qIndex: 0, Effort); configuration,
Width,
Height,
colorConfig,
qIndex: 0,
Effort,
speed: HeifEncodingSpeed.Level0);
keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample); keyEncoder.EncodeKeyFrame(referenceImage.Frames.RootFrame, firstSample);
ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader; ObuSequenceHeader sequenceHeader = keyEncoder.SequenceHeader;
@ -293,7 +302,7 @@ public class Av1IntraSuperblockEncoderTests
using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true); using Av1EncoderPictureBuffer picture = new(configuration, sequenceHeader, frameHeader, Width, Height, disallow4x4AllFrames: true);
using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height); using Av1EncoderCoefficientBuffer coefficients = new(configuration, sequenceHeader, Width, Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration); using Av1EncoderSuperblockWorkspace superblockWorkspace = new(configuration);
using Av1EncoderBlockWorkspace blockWorkspace = new(configuration); using Av1EncoderBlockWorkspace blockWorkspace = new(configuration, allocateInterMotionCosts: true);
using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true); using Av1SymbolEncoder symbolEncoder = new(configuration, TileBufferLength, QIndex, updateCdf: true);
Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace); Av1EncoderTileWorkspace tileWorkspace = new(frameHeader, superblockWorkspace);
int allocationCount = allocator.AllocationLog.Count; int allocationCount = allocator.AllocationLog.Count;
@ -374,7 +383,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -383,7 +393,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder: 64);
FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128); FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128);
FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.U), (byte)128); FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.U), (byte)128);
@ -560,7 +571,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder: 64);
ClearPlane(tileReconstruction.Luma); ClearPlane(tileReconstruction.Luma);
ClearPlane(Assert.IsType<Buffer2D<byte>>(tileReconstruction.ChromaBlue)); ClearPlane(Assert.IsType<Buffer2D<byte>>(tileReconstruction.ChromaBlue));
@ -599,6 +611,369 @@ public class Av1IntraSuperblockEncoderTests
Assert.True(encoded.GetSpan().SequenceEqual(tileWriter.GetTileData(0))); Assert.True(encoded.GetSpan().SequenceEqual(tileWriter.GetTileData(0)));
} }
[Theory]
[InlineData(8)]
[InlineData(10)]
[InlineData(12)]
public void InterBlockCanSkipNonzeroQuantizedResiduals(int bitDepthValue)
{
if (bitDepthValue == 8)
{
VerifyReferenceBlockCanSkipNonzeroQuantizedResiduals<byte, Av1IntraSuperblockEncoder.ByteOperator>(
Av1BitDepth.EightBit,
bitDepthValue,
isIntraBlockCopy: false,
static value => (byte)value);
}
else
{
VerifyReferenceBlockCanSkipNonzeroQuantizedResiduals<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
bitDepthValue == 10 ? Av1BitDepth.TenBit : Av1BitDepth.TwelveBit,
bitDepthValue,
isIntraBlockCopy: false,
static value => (ushort)value);
}
}
[Theory]
[InlineData(8)]
[InlineData(10)]
[InlineData(12)]
public void IntraBlockCopyCanSkipNonzeroQuantizedResiduals(int bitDepthValue)
{
if (bitDepthValue == 8)
{
VerifyReferenceBlockCanSkipNonzeroQuantizedResiduals<byte, Av1IntraSuperblockEncoder.ByteOperator>(
Av1BitDepth.EightBit,
bitDepthValue,
isIntraBlockCopy: true,
static value => (byte)value);
}
else
{
VerifyReferenceBlockCanSkipNonzeroQuantizedResiduals<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
bitDepthValue == 10 ? Av1BitDepth.TenBit : Av1BitDepth.TwelveBit,
bitDepthValue,
isIntraBlockCopy: true,
static value => (ushort)value);
}
}
private static void VerifyReferenceBlockCanSkipNonzeroQuantizedResiduals<TSample, TOperator>(
Av1BitDepth bitDepth,
int bitDepthValue,
bool isIntraBlockCopy,
SampleFactory<TSample> createSample)
where TSample : unmanaged
where TOperator : struct, Av1IntraSuperblockEncoder.IBlockEncodingOperator<TSample>
{
const int Width = 8;
const int Height = 8;
Point blockOrigin = new(isIntraBlockCopy ? 320 : 0, 0);
Point modeInfoPosition = blockOrigin >> Av1Constants.ModeInfoSizeLog2;
int frameWidth = blockOrigin.X + Width;
int superblockIndex = blockOrigin.X / 64;
ObuColorConfig colorConfig = new()
{
IsMonochrome = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = bitDepth
};
using Av1EncoderFrameBuffer<TSample> source = new(
Configuration.Default,
frameWidth,
Height,
bitDepthValue,
Av1ColorFormat.Yuv400,
0,
0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reference = new(
Configuration.Default,
frameWidth,
Height,
bitDepthValue,
Av1ColorFormat.Yuv400,
0,
0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default,
frameWidth,
Height,
bitDepthValue,
Av1ColorFormat.Yuv400,
0,
0,
lumaBorder: 64);
TSample[] prediction = new TSample[Width * Height];
short[] residual = new short[Width * Height];
TSample[] trialReconstruction = new TSample[Width * Height];
int[] trialCoefficients = new int[Width * Height];
int verifiedCases = 0;
for (int qIndex = 64; qIndex <= 192; qIndex += 32)
{
for (int amplitude = 2; amplitude <= 12; amplitude += 2)
{
// Independent texture and small, signed perturbations distinguish temporal prediction from spatial DC.
// Samples remain inside the coded range. Unaligned high-depth perturbations exercise SSE rounding.
long squaredError = 0;
for (int y = 0; y < Height; y++)
{
Span<TSample> sourceRow = source.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
Span<TSample> referenceRow = reference.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
if (isIntraBlockCopy)
{
// Completed blocks provide a repeated reconstructed reference before the current superblock.
// The target differs from it, so pixel search must supply the candidate without an exact hash match.
Span<TSample> reconstructionRow = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y);
for (int x = 0; x < frameWidth; x++)
{
int phase = x % Width;
int sample = 64 + (((phase * 37) + (y * 53) + (phase * y * 19)) % 128);
sourceRow[x] = reconstructionRow[x] = createSample(sample << (bitDepthValue - 8));
}
}
for (int x = 0; x < Width; x++)
{
int index = (y * Width) + x;
int sample = 64 + (((x * 37) + (y * 53) + (x * y * 19)) % 128);
int difference = (((x * 13) + (y * 7) + (x * y * 3)) % ((amplitude * 2) + 1)) - amplitude;
int precisionShift = bitDepthValue - 8;
sample <<= precisionShift;
difference = (difference << precisionShift) + (precisionShift > 0 && index % 3 == 0 ? 1 : 0);
prediction[index] = referenceRow[x] = createSample(sample);
sourceRow[blockOrigin.X + x] = createSample(sample + difference);
residual[index] = (short)difference;
squaredError += difference * difference;
}
}
source.Frame.ExtendBorders();
reference.Frame.ExtendBorders();
using Av1EncoderModeInfoBuffer modeInfoBuffer = new(Configuration.Default, frameWidth, Height, disallow4x4AllFrames: true);
Av1PictureControlSet template = CreatePicture(modeInfoBuffer, colorConfig, use128x128Superblock: false, qIndex);
template.Parent.FrameHeader.FrameType = isIntraBlockCopy ? ObuFrameType.KeyFrame : ObuFrameType.InterFrame;
template.Parent.FrameHeader.AllowIntraBlockCopy = isIntraBlockCopy;
template.Parent.FrameHeader.AllowScreenContentTools = isIntraBlockCopy;
template.Parent.FrameHeader.FrameSize.FrameWidth = frameWidth;
template.Parent.FrameHeader.FrameSize.FrameHeight = Height;
template.Parent.FrameHeader.TransformMode = Av1TransformMode.Largest;
template.Parent.FrameHeader.InterpolationFilter = Av1InterpolationFilter.Regular;
template.Parent.FrameHeader.ReferenceMode = ObuReferenceMode.SingleReference;
using Av1EncoderPictureBuffer pictureBuffer = new(
Configuration.Default,
template.Sequence.SequenceHeader,
template.Parent.FrameHeader,
frameWidth,
Height,
disallow4x4AllFrames: true);
Av1PictureControlSet picture = pictureBuffer.Picture;
if (isIntraBlockCopy)
{
picture.IntraBlockCopySearch.Initialize<TSample, TOperator>(source.Frame.View.GetPlane(Av1Plane.Y));
}
using Av1EncoderCoefficientBuffer coefficients = new(Configuration.Default, template.Sequence.SequenceHeader, frameWidth, Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(Configuration.Default);
using Av1EncoderBlockWorkspace blockWorkspace = new(Configuration.Default, allocateInterMotionCosts: !isIntraBlockCopy);
using Av1SymbolEncoder writer = new(Configuration.Default, 4096, qIndex, updateCdf: true);
if (!isIntraBlockCopy)
{
writer.FillMotionVectorCosts(blockWorkspace.GetMotionVectorCosts(picture.Parent.FrameHeader.MotionVectorPrecision));
}
Av1Superblock superblock = new()
{
Workspace = superblockWorkspace,
TileInfo = new Av1TileInfo(0, 0, picture.Parent.FrameHeader),
Index = superblockIndex
};
Av1IntraSuperblockEncoder.Prepare(picture, superblock, blockOrigin);
picture.MapModeInfoBlock(modeInfoPosition, Av1BlockSize.Block8x8);
Av1MacroBlockD macroBlock = new() { Tile = superblock.TileInfo };
Av1TileWriter.SetModeInfoRowAndColumn(
picture,
macroBlock,
superblock.TileInfo,
modeInfoPosition,
Av1BlockSize.Block8x8,
picture.Parent.Common.ModeInfoStride,
picture.Parent.Common.ModeInfoRowCount,
picture.Parent.Common.ModeInfoColumnCount);
if (!isIntraBlockCopy)
{
picture.Parent.MotionSearchSettings = new Av1MotionSearchSettings(
HeifEncodingSpeed.Level0, false, new Size(frameWidth, Height), qIndex, false, false);
picture.Parent.MotionSearchStepParameter = Av1MotionSearchBase.GetInitialStepParameter(Math.Max(frameWidth, Height));
ref Av1ReferenceMotionVectors references = ref blockWorkspace.ReferenceMotionVectors;
references.Build(
picture,
macroBlock,
modeInfoPosition,
Av1BlockSize.Block8x8,
Av1PartitionType.None,
template.Sequence.SequenceHeader,
picture.Parent.FrameHeader,
Av1ReferenceFrameType.Last);
// Thirty-two preceding global-motion symbols make the zero global predictor the cheapest
// mode. Keep every competing mode enabled so this fixture tests residual skipping independently
// of mode-search restrictions, while checking exact syntax rates, distortion, and reconstruction.
for (int index = 0; index < 32; index++)
{
writer.WriteInterMode<Av1SymbolEncoder.SymbolUpdateOperation>(Av1PredictionMode.GlobalMotionVector, references.ModeContext);
}
int globalRate = writer.GetInterModeCost(Av1PredictionMode.GlobalMotionVector, references.ModeContext);
Assert.True(globalRate < writer.GetInterModeCost(Av1PredictionMode.NearestMotionVector, references.ModeContext));
Assert.True(globalRate < writer.GetInterModeCost(Av1PredictionMode.NearMotionVector, references.ModeContext));
Assert.True(globalRate < writer.GetInterModeCost(Av1PredictionMode.NewMotionVector, references.ModeContext));
}
int multiplier = isIntraBlockCopy
? Av1RateDistortion.GetKeyFrameRateMultiplier(qIndex, bitDepth)
: Av1RateDistortion.GetInterFrameRateMultiplier(qIndex, bitDepth);
int skipContext = Av1TileWriter.GetSkipContext(macroBlock);
int skipRate = writer.GetSkipCost(true, skipContext);
int squaredPrecisionScale = 1 << ((bitDepthValue - 8) * 2);
long expectedDistortion = ((squaredError + (squaredPrecisionScale / 2)) / squaredPrecisionScale) * 16;
long skipCost = ((((long)skipRate * multiplier) + 256) / 512) + (expectedDistortion * 128);
long bestCodedCost = long.MaxValue;
bool allTransformsAreNonzero = true;
Av1TransformSetType transformSet = Av1SymbolContextHelper.GetExtendedTransformSetType(
Av1TransformSize.Size8x8,
isInter: true,
picture.Parent.FrameHeader.UseReducedTransformSet);
// Evaluate residual coding independently of the block decision. Qualifying cases must have
// nonzero coefficients in every legal transform, yet cost more than prediction-only reconstruction.
for (Av1TransformType transformType = Av1TransformType.DctDct;
transformType < Av1TransformType.AllTransformTypes;
transformType++)
{
if (!transformType.IsExtendedSetUsed(transformSet))
{
continue;
}
Av1EncoderTransformBlockState state = default;
long distortion = TOperator.EncodePredictionCandidate(
blockWorkspace,
source.Frame.CodedView.GetPlane(Av1Plane.Y),
blockOrigin,
prediction,
residual,
trialReconstruction,
Width,
trialCoefficients,
Av1TransformSize.Size8x8,
transformType,
Av1Plane.Y,
qIndex,
0,
0,
bitDepth,
ref state);
allTransformsAreNonzero &= state.EndOfBlock != 0;
int rate = writer.GetSkipCost(false, skipContext) + writer.GetCoefficientCost(
Av1TransformSize.Size8x8,
transformType,
isIntraBlockCopy ? Av1PredictionMode.DC : Av1PredictionMode.GlobalMotionVector,
trialCoefficients,
Av1ComponentType.Luminance,
default,
state.EndOfBlock,
picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: true);
long cost = ((((long)rate * multiplier) + 256) / 512) + (distortion * 128);
bestCodedCost = Math.Min(bestCodedCost, cost);
}
if (!allTransformsAreNonzero || skipCost > bestCodedCost)
{
continue;
}
Av1IntraSuperblockEncoder.ModeDecision<TSample, TOperator> decision = new(
source.Frame,
reference.Frame,
reconstruction.Frame,
picture,
superblock,
coefficients,
blockWorkspace,
effort: 0);
ref Av1MacroBlockModeInfo modeInfo = ref picture.GetMacroBlockModeInfo(modeInfoPosition);
Av1EncoderBlockStruct block = default;
Av1EncoderPaletteInfo palette = default;
Av1MotionVector displacementReference = default;
if (isIntraBlockCopy)
{
displacementReference = Av1IntraBlockCopy.FindReference(
picture,
macroBlock,
modeInfoPosition,
Av1BlockSize.Block8x8,
Av1PartitionType.None,
new Av1MotionVector[8],
new int[8]);
}
decision.EncodeBlock(writer, macroBlock, blockOrigin, 0, ref modeInfo, ref block, ref palette);
Assert.Equal(isIntraBlockCopy, modeInfo.Block.UseIntraBlockCopy);
if (!isIntraBlockCopy)
{
Assert.Equal(Av1ReferenceFrameType.Last, modeInfo.Block.ReferenceFrame);
}
Assert.True(modeInfo.Block.Skip, $"qIndex={qIndex}, amplitude={amplitude}, skipCost={skipCost}, codedCost={bestCodedCost}");
Assert.Equal(isIntraBlockCopy ? Av1PredictionMode.DC : Av1PredictionMode.GlobalMotionVector, modeInfo.Block.Mode);
Assert.Equal(expectedDistortion, decision.SelectedBlockStatistics.Distortion);
int expectedRate = skipRate + (isIntraBlockCopy
? writer.GetUseIntraBlockCopyCost(true) +
writer.GetDisplacementVectorCost(picture.GetDisplacementVector(modeInfoPosition), displacementReference)
: writer.GetIsInterCost(true, Av1TileWriter.GetIntraInterContext(macroBlock)) +
writer.GetSingleReferenceCost(Av1ReferenceFrameType.Last, new byte[Av1Constants.ReferenceFrameCount]) +
writer.GetInterModeCost(Av1PredictionMode.GlobalMotionVector, blockWorkspace.ReferenceMotionVectors.ModeContext));
Assert.Equal(expectedRate, decision.SelectedBlockStatistics.Rate);
Assert.Equal(
((((long)expectedRate * multiplier) + 256) / 512) + (expectedDistortion * 128),
decision.SelectedBlockStatistics.Cost);
Assert.Equal((ushort)0, coefficients.GetTransformBlockSpan(superblockIndex, Av1Plane.Y)[0].EndOfBlock);
Assert.Equal(Av1TransformType.DctDct, coefficients.GetTransformBlockSpan(superblockIndex, Av1Plane.Y)[0].TransformType);
Assert.Equal((byte)0, coefficients.GetTransformBlockSpan(superblockIndex, Av1Plane.Y)[0].EntropyContext);
Assert.All(coefficients.GetPlaneSpan(superblockIndex, Av1Plane.Y)[..64].ToArray(), value => Assert.Equal(0, value));
for (int y = 0; y < Height; y++)
{
Assert.Equal(
MemoryMarshal.AsBytes(prediction.AsSpan(y * Width, Width)),
MemoryMarshal.AsBytes(reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y).DangerousGetRowSpan(y).Slice(blockOrigin.X, Width)));
}
verifiedCases++;
}
}
Assert.True(verifiedCases > 0, "The input set must exercise skipping despite nonzero coefficients in every legal transform.");
}
[Theory] [Theory]
[InlineData(true)] [InlineData(true)]
[InlineData(false)] [InlineData(false)]
@ -622,7 +997,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
1, 1,
1); 1,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -631,7 +1007,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
1, 1,
1); 1,
lumaBorder: 64);
FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128); FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128);
ClearPlane(reconstruction.Luma); ClearPlane(reconstruction.Luma);
@ -713,7 +1090,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
1, 1,
1); 1,
lumaBorder: 64);
ClearPlane(liveReconstruction.Luma); ClearPlane(liveReconstruction.Luma);
if (!isMonochrome) if (!isMonochrome)
@ -775,7 +1153,8 @@ public class Av1IntraSuperblockEncoderTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<ushort> reconstruction = new( using Av1EncoderFrameBuffer<ushort> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -784,7 +1163,8 @@ public class Av1IntraSuperblockEncoderTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<ushort> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<ushort> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int y = 0; y < sourcePlane.Height; y++) for (int y = 0; y < sourcePlane.Height; y++)
@ -847,7 +1227,8 @@ public class Av1IntraSuperblockEncoderTests
12, 12,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
ClearPlane(tileReconstruction.Luma); ClearPlane(tileReconstruction.Luma);
using Av1EncoderPictureBuffer tilePicture = new( using Av1EncoderPictureBuffer tilePicture = new(
@ -907,8 +1288,8 @@ public class Av1IntraSuperblockEncoderTests
BitDepth = Av1BitDepth.EightBit BitDepth = Av1BitDepth.EightBit
}; };
using Av1EncoderFrameBuffer<byte> source = new(Configuration.Default, Width, Height, 8, colorFormat, 1, 1); using Av1EncoderFrameBuffer<byte> source = new(Configuration.Default, Width, Height, 8, colorFormat, 1, 1, lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new(Configuration.Default, Width, Height, 8, colorFormat, 1, 1); using Av1EncoderFrameBuffer<byte> reconstruction = new(Configuration.Default, Width, Height, 8, colorFormat, 1, 1, lumaBorder: 64);
int planeCount = isMonochrome ? 1 : 3; int planeCount = isMonochrome ? 1 : 3;
for (int planeIndex = 0; planeIndex < planeCount; planeIndex++) for (int planeIndex = 0; planeIndex < planeCount; planeIndex++)
{ {
@ -1422,7 +1803,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -1431,7 +1813,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y);
Buffer2DRegion<byte> blueSource = source.Frame.CodedView.GetPlane(Av1Plane.U); Buffer2DRegion<byte> blueSource = source.Frame.CodedView.GetPlane(Av1Plane.U);
@ -1657,7 +2040,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -1666,7 +2050,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
@ -1823,7 +2208,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -1832,7 +2218,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128); FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), (byte)128);
FillChromaModeSelectionPlane( FillChromaModeSelectionPlane(
@ -1986,7 +2373,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> pilotReconstruction = new( using Av1EncoderFrameBuffer<TSample> pilotReconstruction = new(
Configuration.Default, Configuration.Default,
@ -1995,7 +2383,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
Buffer2DRegion<TSample> pilotLuma = pilotSource.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> pilotLuma = pilotSource.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int y = 0; y < pilotLuma.Height; y++) for (int y = 0; y < pilotLuma.Height; y++)
@ -2081,7 +2470,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new( using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -2090,7 +2480,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
colorFormat, colorFormat,
chromaSubsamplingX, chromaSubsamplingX,
chromaSubsamplingY); chromaSubsamplingY,
lumaBorder: 64);
for (int y = 0; y < pilotLuma.Height; y++) for (int y = 0; y < pilotLuma.Height; y++)
{ {
@ -2335,7 +2726,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
1, 1,
1); 1,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> pilotReconstruction = new( using Av1EncoderFrameBuffer<TSample> pilotReconstruction = new(
Configuration.Default, Configuration.Default,
@ -2344,7 +2736,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
1, 1,
1); 1,
lumaBorder: 64);
Buffer2DRegion<TSample> pilotLuma = pilotSource.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> pilotLuma = pilotSource.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int y = 0; y < pilotLuma.Height; y++) for (int y = 0; y < pilotLuma.Height; y++)
@ -2497,7 +2890,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
1, 1,
1); 1,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new( using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -2506,7 +2900,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepth, bitDepth,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
1, 1,
1); 1,
lumaBorder: 64);
Buffer2DRegion<TSample> sourceLuma = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> sourceLuma = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int y = 0; y < pilotLuma.Height; y++) for (int y = 0; y < pilotLuma.Height; y++)
@ -2659,7 +3054,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -2668,7 +3064,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
FillPlane(sourcePlane, (byte)128); FillPlane(sourcePlane, (byte)128);
@ -2830,7 +3227,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepthValue, bitDepthValue,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new( using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -2839,7 +3237,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepthValue, bitDepthValue,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<TSample> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++) for (int row = 0; row < Height; row++)
@ -2941,7 +3340,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -2950,7 +3350,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv420, Av1ColorFormat.Yuv420,
1, 1,
1); 1,
lumaBorder: 64);
Buffer2DRegion<byte> lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++) for (int row = 0; row < Height; row++)
@ -3110,7 +3511,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new( using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -3119,7 +3521,8 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), 251, 29); FillPlane(source.Frame.CodedView.GetPlane(Av1Plane.Y), 251, 29);
ClearPlane(reconstruction.Luma); ClearPlane(reconstruction.Luma);
@ -3201,8 +3604,8 @@ public class Av1IntraSuperblockEncoderTests
BitDepth = Av1BitDepth.EightBit BitDepth = Av1BitDepth.EightBit
}; };
using Av1EncoderFrameBuffer<byte> source = new(Configuration.Default, size, size, 8, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<byte> source = new(Configuration.Default, size, size, 8, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstruction = new(Configuration.Default, size, size, 8, Av1ColorFormat.Yuv400, 0, 0); using Av1EncoderFrameBuffer<byte> reconstruction = new(Configuration.Default, size, size, 8, Av1ColorFormat.Yuv400, 0, 0, lumaBorder: 64);
Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int y = 0; y < size; y++) for (int y = 0; y < size; y++)
{ {
@ -3274,6 +3677,143 @@ public class Av1IntraSuperblockEncoderTests
} }
} }
[Theory]
[InlineData((int)Av1ColorFormat.Yuv400)]
[InlineData((int)Av1ColorFormat.Yuv420)]
[InlineData((int)Av1ColorFormat.Yuv422)]
[InlineData((int)Av1ColorFormat.Yuv444)]
public void ProductionDeblockingPreservesEightBitReconstruction(int colorFormatValue)
=> VerifyProductionDeblocking<byte>(
colorFormatValue,
8,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new(writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace, effort: 5));
[Theory]
[InlineData((int)Av1ColorFormat.Yuv400, 10)]
[InlineData((int)Av1ColorFormat.Yuv400, 12)]
[InlineData((int)Av1ColorFormat.Yuv420, 10)]
[InlineData((int)Av1ColorFormat.Yuv420, 12)]
[InlineData((int)Av1ColorFormat.Yuv422, 10)]
[InlineData((int)Av1ColorFormat.Yuv422, 12)]
[InlineData((int)Av1ColorFormat.Yuv444, 10)]
[InlineData((int)Av1ColorFormat.Yuv444, 12)]
public void ProductionDeblockingPreservesHighBitDepthReconstruction(int colorFormatValue, int bitDepth)
=> VerifyProductionDeblocking<ushort>(
colorFormatValue,
bitDepth,
static (writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new(writer, source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace, effort: 5));
/// <summary>
/// Verifies retained reconstruction and exports the same encoded stream for an independent decoder comparison.
/// </summary>
/// <typeparam name="TSample">The component sample type.</typeparam>
/// <param name="colorFormatValue">The component layout.</param>
/// <param name="bitDepth">The component precision.</param>
/// <param name="createWriter">The typed production tile constructor.</param>
private static void VerifyProductionDeblocking<TSample>(int colorFormatValue, int bitDepth, TileWriterFactory<TSample> createWriter)
where TSample : unmanaged, IBinaryInteger<TSample>
{
const int Width = 33;
const int Height = 137;
const int QIndex = 37;
Av1ColorFormat colorFormat = (Av1ColorFormat)colorFormatValue;
ObuColorConfig colorConfig = new()
{
IsMonochrome = colorFormat == Av1ColorFormat.Yuv400,
SubSamplingX = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420 or Av1ColorFormat.Yuv422,
SubSamplingY = colorFormat is Av1ColorFormat.Yuv400 or Av1ColorFormat.Yuv420,
BitDepth = bitDepth == 8 ? Av1BitDepth.EightBit : bitDepth == 10 ? Av1BitDepth.TenBit : Av1BitDepth.TwelveBit
};
using Av1EncoderFrameBuffer<TSample> source = new(Configuration.Default, Width, Height, bitDepth, colorFormat, 1, 1, lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new(Configuration.Default, Width, Height, bitDepth, colorFormat, 1, 1, lumaBorder: 64);
int planeCount = colorConfig.PlaneCount;
TSample[][] unfiltered = new TSample[planeCount][];
for (int planeIndex = 0; planeIndex < planeCount; planeIndex++)
{
Buffer2DRegion<TSample> plane = source.Frame.View.GetPlane((Av1Plane)planeIndex);
unfiltered[planeIndex] = new TSample[plane.Width * plane.Height];
for (int y = 0; y < plane.Height; y++)
{
Span<TSample> row = plane.DangerousGetRowSpan(y);
for (int x = 0; x < row.Length; x++)
{
// Small discontinuities at coding boundaries activate deblocking. Odd dimensions and a
// height above 128 exercise chroma ownership, coded padding, and intersecting row bands.
int value = 96 + (4 * ((x / 8) + (y / 8))) + (planeIndex * 8);
row[x] = TSample.CreateChecked(value << (bitDepth - 8));
}
}
}
source.Frame.ExtendBorders();
using Av1EncoderModeInfoBuffer modeInfo = new(Configuration.Default, Width, Height, disallow4x4AllFrames: true);
Av1PictureControlSet template = CreatePicture(modeInfo, colorConfig, use128x128Superblock: false, QIndex);
ObuFrameHeader header = template.Parent.FrameHeader;
header.FrameSize.FrameWidth = Width;
header.FrameSize.FrameHeight = Height;
using Av1EncoderPictureBuffer picture = new(
Configuration.Default, template.Sequence.SequenceHeader, header, Width, Height, disallow4x4AllFrames: true);
using Av1EncoderCoefficientBuffer coefficients = new(Configuration.Default, template.Sequence.SequenceHeader, Width, Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(Configuration.Default);
using Av1EncoderBlockWorkspace blockWorkspace = new(Configuration.Default);
using Av1SymbolEncoder symbolEncoder = CreateTileSymbolEncoder(picture.Picture, 8192);
_ = createWriter(symbolEncoder, source.Frame, reconstruction.Frame, picture.Picture, coefficients, superblockWorkspace, blockWorkspace);
for (int planeIndex = 0; planeIndex < planeCount; planeIndex++)
{
Buffer2DRegion<TSample> plane = reconstruction.Frame.View.GetPlane((Av1Plane)planeIndex);
for (int y = 0; y < plane.Height; y++)
{
plane.DangerousGetRowSpan(y).CopyTo(unfiltered[planeIndex].AsSpan(y * plane.Width, plane.Width));
}
}
header.LoopFilterParameters.FilterLevel[0] = 63;
header.LoopFilterParameters.FilterLevel[1] = 37;
header.LoopFilterParameters.FilterLevelU = 31;
header.LoopFilterParameters.FilterLevelV = 47;
header.LoopFilterParameters.SharpnessLevel = 3;
header.LoopFilterParameters.ReferenceDeltaModeEnabled = true;
picture.Reset(header);
Av1TileEncoder tileWriter = createWriter(
symbolEncoder, source.Frame, reconstruction.Frame, picture.Picture, coefficients, superblockWorkspace, blockWorkspace);
byte[] payload = WriteCompleteTileObu(picture.Picture, tileWriter, Width, Height);
using Av1Decoder decoder = new(Configuration.Default);
using Av1FrameBuffer<byte> decodedFrame = decoder.DecodeFrameBuffer(payload, null, null, out _);
int changedSamples = 0;
string directory = Path.Combine(TestEnvironment.ActualOutputDirectoryFullPath, "Heif", "Av1", "ProductionDeblocking");
Directory.CreateDirectory(directory);
string name = $"{bitDepth}-{colorFormat}";
File.WriteAllBytes(Path.Combine(directory, name + ".obu"), payload);
using FileStream raw = File.Create(Path.Combine(directory, name + ".retained.yuv"));
for (int planeIndex = 0; planeIndex < planeCount; planeIndex++)
{
Av1Plane plane = (Av1Plane)planeIndex;
Buffer2DRegion<TSample> retained = reconstruction.Frame.View.GetPlane(plane);
int subX = plane == Av1Plane.Y ? 0 : reconstruction.Frame.ChromaSubsamplingX;
int subY = plane == Av1Plane.Y ? 0 : reconstruction.Frame.ChromaSubsamplingY;
Buffer2DRegion<byte> decoded = decodedFrame.DeriveBlockPointer(plane, subX, subY);
for (int y = 0; y < retained.Height; y++)
{
ReadOnlySpan<TSample> row = retained.DangerousGetRowSpan(y);
ReadOnlySpan<TSample> decodedRow = MemoryMarshal.Cast<byte, TSample>(decoded.DangerousGetRowSpan(y));
Assert.Equal(row, decodedRow);
for (int x = 0; x < row.Length; x++)
{
changedSamples += row[x] != unfiltered[planeIndex][(y * retained.Width) + x] ? 1 : 0;
}
raw.Write(MemoryMarshal.AsBytes(row));
}
}
Assert.True(changedSamples > 0);
}
private static byte[] WriteCompleteTileObu( private static byte[] WriteCompleteTileObu(
Av1PictureControlSet pictureTemplate, Av1PictureControlSet pictureTemplate,
IAv1TileWriter tileWriter, IAv1TileWriter tileWriter,
@ -3421,7 +3961,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepthValue, bitDepthValue,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<TSample> reconstruction = new( using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default, Configuration.Default,
@ -3430,7 +3971,8 @@ public class Av1IntraSuperblockEncoderTests
bitDepthValue, bitDepthValue,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
Buffer2DRegion<TSample> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < sourcePlane.Height; row++) for (int row = 0; row < sourcePlane.Height; row++)
@ -3815,6 +4357,8 @@ public class Av1IntraSuperblockEncoderTests
this.Count = 0; this.Count = 0;
} }
public static bool UsesRetainedDecisions => false;
/// <summary> /// <summary>
/// Gets the number of final blocks visited by the writer. /// Gets the number of final blocks visited by the writer.
/// </summary> /// </summary>
@ -3891,6 +4435,8 @@ public class Av1IntraSuperblockEncoderTests
this.Count = 0; this.Count = 0;
} }
public static bool UsesRetainedDecisions => false;
/// <summary> /// <summary>
/// Gets the number of final blocks visited by the writer. /// Gets the number of final blocks visited by the writer.
/// </summary> /// </summary>

16
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs

@ -59,7 +59,8 @@ public class Av1TransformBlockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
using Av1EncoderFrameBuffer<byte> reconstructionFrame = new( using Av1EncoderFrameBuffer<byte> reconstructionFrame = new(
Configuration.Default, Configuration.Default,
@ -68,7 +69,8 @@ public class Av1TransformBlockEncoderTests
8, 8,
Av1ColorFormat.Yuv400, Av1ColorFormat.Yuv400,
0, 0,
0); 0,
lumaBorder: 64);
reconstructionFrame.Luma.DangerousGetSingleSpan().Fill(PaddingSentinel); reconstructionFrame.Luma.DangerousGetSingleSpan().Fill(PaddingSentinel);
Buffer2DRegion<byte> sourcePlane = sourceFrame.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<byte> sourcePlane = sourceFrame.Frame.CodedView.GetPlane(Av1Plane.Y);
@ -596,8 +598,10 @@ public class Av1TransformBlockEncoderTests
/// <summary> /// <summary>
/// Verifies that the block workspace uses one exact-size allocator owner and returns it exactly once. /// Verifies that the block workspace uses one exact-size allocator owner and returns it exactly once.
/// </summary> /// </summary>
[Fact] [Theory]
public void BlockWorkspaceUsesOneExactSizeOwner() [InlineData(false, 0)]
[InlineData(true, 135836)]
public void BlockWorkspaceUsesOneExactSizeOwner(bool allocateInterMotionCosts, int additionalLength)
{ {
TestMemoryAllocator allocator = new(); TestMemoryAllocator allocator = new();
allocator.EnableNonThreadSafeLogging(); allocator.EnableNonThreadSafeLogging();
@ -605,12 +609,12 @@ public class Av1TransformBlockEncoderTests
configuration.MemoryAllocator = allocator; configuration.MemoryAllocator = allocator;
TestMemoryAllocator.AllocationRequest allocation; TestMemoryAllocator.AllocationRequest allocation;
using (Av1EncoderBlockWorkspace workspace = new(configuration)) using (Av1EncoderBlockWorkspace workspace = new(configuration, allocateInterMotionCosts))
{ {
allocation = Assert.Single(allocator.AllocationLog); allocation = Assert.Single(allocator.AllocationLog);
Assert.Empty(allocator.ReturnLog); Assert.Empty(allocator.ReturnLog);
Assert.Equal(typeof(int), allocation.ElementType); Assert.Equal(typeof(int), allocation.ElementType);
Assert.Equal(Av1EncoderBlockWorkspace.StorageLength, allocation.Length); Assert.Equal(Av1EncoderBlockWorkspace.StorageLength + additionalLength, allocation.Length);
Assert.Equal(Av1EncoderBlockWorkspace.MaximumResidualCount, workspace.Residual.Length); Assert.Equal(Av1EncoderBlockWorkspace.MaximumResidualCount, workspace.Residual.Length);
Assert.Equal(Av1EncoderBlockWorkspace.MaximumCoefficientCount, workspace.TransformCoefficients.Length); Assert.Equal(Av1EncoderBlockWorkspace.MaximumCoefficientCount, workspace.TransformCoefficients.Length);
Assert.Equal(Av1EncoderBlockWorkspace.MaximumCoefficientCount, workspace.DequantizedCoefficients.Length); Assert.Equal(Av1EncoderBlockWorkspace.MaximumCoefficientCount, workspace.DequantizedCoefficients.Length);

Loading…
Cancel
Save