tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Honor AV1 encoding effort

pull/2633/head
James Jackson-South 4 weeks ago
parent
fe44469b1c
commit
e168a04390
6 changed files with 174 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      HEIF_IMPLEMENTATION_PLAN.md
  2. 61
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  3. 15
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs
  4. 29
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  5. 14
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs
  6. 86
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

3
HEIF_IMPLEMENTATION_PLAN.md
View File

File diff suppressed because one or more lines are too long

61
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
View File

@ -26,15 +26,17 @@ internal static class Av1FrameEncoder
/// <param name="stream">The destination receiving the complete AV1 item payload.</param>
/// <param name="colorConfig">The resolved native color and precision configuration.</param>
/// <param name="qIndex">The frame quantizer index.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
/// <returns>The sequence header describing the encoded payload.</returns>
public static ObuSequenceHeader Encode<TPixel>(
Configuration configuration,
ImageFrame<TPixel> image,
Stream stream,
ObuColorConfig colorConfig,
int qIndex)
int qIndex,
int effort = 5)
where TPixel : unmanaged, IPixel<TPixel>
=> Encode(configuration, image, stream, colorConfig, qIndex, false);
=> Encode(configuration, image, stream, colorConfig, qIndex, effort, false);
/// <summary>
/// Encodes one packed alpha channel as a reduced-still-picture monochrome AV1 frame.
@ -45,15 +47,17 @@ internal static class Av1FrameEncoder
/// <param name="stream">The destination receiving the complete AV1 item payload.</param>
/// <param name="colorConfig">The resolved monochrome precision configuration.</param>
/// <param name="qIndex">The frame quantizer index.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
/// <returns>The sequence header describing the encoded payload.</returns>
public static ObuSequenceHeader EncodeAlpha<TPixel>(
Configuration configuration,
ImageFrame<TPixel> image,
Stream stream,
ObuColorConfig colorConfig,
int qIndex)
int qIndex,
int effort = 5)
where TPixel : unmanaged, IPixel<TPixel>
=> Encode(configuration, image, stream, colorConfig, qIndex, true);
=> Encode(configuration, image, stream, colorConfig, qIndex, effort, true);
private static ObuSequenceHeader Encode<TPixel>(
Configuration configuration,
@ -61,6 +65,7 @@ internal static class Av1FrameEncoder
Stream stream,
ObuColorConfig colorConfig,
int qIndex,
int effort,
bool encodeAlpha)
where TPixel : unmanaged, IPixel<TPixel>
{
@ -87,7 +92,7 @@ internal static class Av1FrameEncoder
Use128x128Superblock = false,
ForceScreenContentTools = 2,
ForceIntegerMotionVector = 2,
EnableFilterIntra = true,
EnableFilterIntra = effort >= 4,
EnableIntraEdgeFilter = false,
EnableSuperResolution = false,
EnableCdef = false,
@ -148,11 +153,11 @@ internal static class Av1FrameEncoder
int initialTileSize = checked((int)Math.Max(8192L, (sampleCount * sampleSize * 5) / 2));
if (colorConfig.BitDepth == Av1BitDepth.EightBit)
{
EncodeByte(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, encodeAlpha);
EncodeByte(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, effort, encodeAlpha);
}
else
{
EncodeHighBitDepth(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, encodeAlpha);
EncodeHighBitDepth(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, effort, encodeAlpha);
}
return sequenceHeader;
@ -198,6 +203,7 @@ internal static class Av1FrameEncoder
ObuFrameHeader frameHeader,
Av1ColorFormat colorFormat,
int initialTileSize,
int effort,
bool encodeAlpha)
where TPixel : unmanaged, IPixel<TPixel>
{
@ -219,7 +225,7 @@ internal static class Av1FrameEncoder
chromaPositionX: 1,
chromaPositionY: 1);
Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, encodeAlpha);
Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, effort, encodeAlpha);
}
private static void EncodeHighBitDepth<TPixel>(
@ -230,6 +236,7 @@ internal static class Av1FrameEncoder
ObuFrameHeader frameHeader,
Av1ColorFormat colorFormat,
int initialTileSize,
int effort,
bool encodeAlpha)
where TPixel : unmanaged, IPixel<TPixel>
{
@ -252,7 +259,7 @@ internal static class Av1FrameEncoder
chromaPositionX: 1,
chromaPositionY: 1);
Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, encodeAlpha);
Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, effort, encodeAlpha);
}
private static void Encode<TPixel>(
@ -264,6 +271,7 @@ internal static class Av1FrameEncoder
Av1EncoderFrameBuffer<byte> source,
Av1EncoderFrameBuffer<byte> reconstruction,
int initialTileSize,
int effort,
bool encodeAlpha)
where TPixel : unmanaged, IPixel<TPixel>
{
@ -274,10 +282,16 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig,
encodeAlpha);
Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
bool allowScreenContentTools = false;
bool allowIntraBlockCopy = false;
if (effort >= 5)
{
// Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan.
Av1ScreenContentDetector.Detect(
source.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
}
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
@ -304,7 +318,8 @@ internal static class Av1FrameEncoder
coefficients,
superblockWorkspace,
blockWorkspace,
initialTileSize);
initialTileSize,
effort);
ObuWriter writer = new();
writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter);
@ -319,6 +334,7 @@ internal static class Av1FrameEncoder
Av1EncoderFrameBuffer<ushort> source,
Av1EncoderFrameBuffer<ushort> reconstruction,
int initialTileSize,
int effort,
bool encodeAlpha)
where TPixel : unmanaged, IPixel<TPixel>
{
@ -329,10 +345,16 @@ internal static class Av1FrameEncoder
sequenceHeader.ColorConfig,
encodeAlpha);
Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
bool allowScreenContentTools = false;
bool allowIntraBlockCopy = false;
if (effort >= 5)
{
// Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan.
Av1ScreenContentDetector.Detect(
source.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
}
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
@ -359,7 +381,8 @@ internal static class Av1FrameEncoder
coefficients,
superblockWorkspace,
blockWorkspace,
initialTileSize);
initialTileSize,
effort);
ObuWriter writer = new();
writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter);

15
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs
View File

@ -164,7 +164,15 @@ internal static partial class Av1IntraSuperblockEncoder
int baseModeCount = ChromaModeSearchOrder.Length;
int deltaCount = AngleDeltaSearchOrder.Length;
int directionalModeCount = (int)Av1ChromaPredictionMode.Directional67Degrees - (int)Av1ChromaPredictionMode.Vertical + 1;
int candidateCount = baseModeCount + (directionalModeCount * deltaCount);
// Effort zero evaluates DC only, effort one adds every zero-angle mode, and higher levels add all directional adjustments.
int candidateCount = this.effort switch
{
0 => 1,
1 => baseModeCount,
_ => baseModeCount + (directionalModeCount * deltaCount)
};
bool hasLumaPalette = paletteInfo.PaletteSizes[0] != 0;
int paletteDisabledCost = this.picture.Parent.FrameHeader.AllowScreenContentTools
? writer.GetPaletteUvModeCost(false, hasLumaPalette)
@ -249,7 +257,7 @@ internal static partial class Av1IntraSuperblockEncoder
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
if (chromaFromLumaAllowed)
if (this.effort >= 4 && chromaFromLumaAllowed)
{
Span<short> lumaQ3 = stackalloc short[Av1ChromaFromLumaContext.BufferLine * 8];
TOperator.PrepareChromaFromLuma(
@ -441,7 +449,8 @@ internal static partial class Av1IntraSuperblockEncoder
}
}
if (this.picture.Parent.FrameHeader.AllowScreenContentTools &&
if (this.effort >= 5 &&
this.picture.Parent.FrameHeader.AllowScreenContentTools &&
this.SelectChromaPalette(
writer,
macroBlock,

29
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
View File

@ -117,6 +117,7 @@ internal static partial class Av1IntraSuperblockEncoder
private readonly ObuQuantizationParameters quantization;
private readonly Av1BitDepth bitDepth;
private readonly int rateMultiplier;
private readonly int effort;
private int codedAreaLuma;
private int codedAreaChroma;
@ -129,13 +130,15 @@ internal static partial class Av1IntraSuperblockEncoder
/// <param name="superblock">The current superblock.</param>
/// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public ModeDecision(
Av1EncoderFrame<TSample> source,
Av1EncoderFrame<TSample> reconstruction,
Av1PictureControlSet picture,
Av1Superblock superblock,
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderBlockWorkspace blockWorkspace)
Av1EncoderBlockWorkspace blockWorkspace,
int effort = 5)
{
this.source = source.CodedView;
this.reconstruction = reconstruction.CodedView;
@ -146,6 +149,7 @@ internal static partial class Av1IntraSuperblockEncoder
this.quantization = picture.Parent.FrameHeader.QuantizationParameters;
this.bitDepth = picture.Sequence.SequenceHeader.ColorConfig.BitDepth;
this.rateMultiplier = Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth);
this.effort = effort;
this.codedAreaLuma = 0;
this.codedAreaChroma = 0;
}
@ -528,7 +532,15 @@ internal static partial class Av1IntraSuperblockEncoder
int baseModeCount = LumaModeSearchOrder.Length;
int deltaCount = AngleDeltaSearchOrder.Length;
int directionalModeCount = (int)Av1PredictionMode.Directional67Degrees - (int)Av1PredictionMode.Vertical + 1;
int candidateCount = baseModeCount + (directionalModeCount * deltaCount);
// Effort zero evaluates DC only, effort one adds every zero-angle mode, and higher levels add all directional adjustments.
int candidateCount = this.effort switch
{
0 => 1,
1 => baseModeCount,
_ => baseModeCount + (directionalModeCount * deltaCount)
};
bool useReducedTransformSet = this.picture.Parent.FrameHeader.UseReducedTransformSet;
Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType(
TransformSize,
@ -594,11 +606,11 @@ internal static partial class Av1IntraSuperblockEncoder
}
}
// Mode selection uses the mode-derived default transform, then the winning mode alone pays for
// an exhaustive transform refinement. This preserves reference tie order without a 61-by-7 search.
long bestTransformCost = long.MaxValue;
// Lower effort levels retain the mode-derived default transform. Higher levels refine only the winning
// mode across the legal transform set, preserving search order without evaluating every mode-transform pair.
long bestTransformCost = bestCost;
for (Av1TransformType transformType = Av1TransformType.DctDct;
transformType < Av1TransformType.AllTransformTypes;
this.effort >= 3 && transformType < Av1TransformType.AllTransformTypes;
transformType++)
{
if (!transformType.IsExtendedSetUsed(transformSetType))
@ -641,7 +653,7 @@ internal static partial class Av1IntraSuperblockEncoder
}
}
if (this.picture.Sequence.SequenceHeader.EnableFilterIntra)
if (this.effort >= 4 && this.picture.Sequence.SequenceHeader.EnableFilterIntra)
{
Span<TSample> filterPrediction = stackalloc TSample[SampleCount];
Span<short> filterResidual = stackalloc short[SampleCount];
@ -710,7 +722,8 @@ internal static partial class Av1IntraSuperblockEncoder
}
}
if (this.picture.Parent.FrameHeader.AllowScreenContentTools &&
if (this.effort >= 5 &&
this.picture.Parent.FrameHeader.AllowScreenContentTools &&
this.SelectLumaPalette(
writer,
macroBlock,

14
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs
View File

@ -28,6 +28,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="initialSize">The estimated encoded tile size in bytes.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1IntraTileWriter(
Configuration configuration,
Av1EncoderFrame<byte> source,
@ -36,7 +37,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int initialSize)
int initialSize,
int effort = 5)
{
this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
configuration,
@ -46,6 +48,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
coefficientBuffer,
superblockWorkspace,
blockWorkspace,
effort,
initialSize,
out this.tileDataLength);
}
@ -61,6 +64,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
/// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
/// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
/// <param name="initialSize">The estimated encoded tile size in bytes.</param>
/// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
public Av1IntraTileWriter(
Configuration configuration,
Av1EncoderFrame<ushort> source,
@ -69,7 +73,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int initialSize)
int initialSize,
int effort = 5)
{
this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
configuration,
@ -79,6 +84,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
coefficientBuffer,
superblockWorkspace,
blockWorkspace,
effort,
initialSize,
out this.tileDataLength);
}
@ -107,6 +113,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
Av1EncoderCoefficientBuffer coefficientBuffer,
Av1EncoderSuperblockWorkspace superblockWorkspace,
Av1EncoderBlockWorkspace blockWorkspace,
int effort,
int initialSize,
out int tileDataLength)
where TSample : unmanaged
@ -171,7 +178,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
picture,
superblock,
coefficientBuffer,
blockWorkspace);
blockWorkspace,
effort);
Av1TileWriter.WriteSuperblock(
picture,

86
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs
View File

@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Formats;
using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Components;
using SixLabors.ImageSharp.Formats.Heif.Components.Alpha;
@ -482,6 +483,91 @@ public class Av1EncoderFrameTests
File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-16x16-8b-444-palette.obu"), payload);
}
[Theory]
[InlineData(0, false, false)]
[InlineData(1, false, false)]
[InlineData(2, false, false)]
[InlineData(3, false, false)]
[InlineData(4, true, false)]
[InlineData(5, true, true)]
public void EncodeEffortControlsSearchFeatures(int effort, bool enableFilterIntra, bool enableScreenContentTools)
{
const int width = 16;
const int height = 16;
using Image<Rgba32> source = new(width, height);
for (int row = 0; row < height; row++)
{
Span<Rgba32> pixels = source.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row);
for (int column = 0; column < width; column++)
{
pixels[column] = (((column >> 2) + (row >> 2)) & 1) == 0
? new Rgba32(224, 32, 32)
: new Rgba32(32, 32, 224);
}
}
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv444);
using MemoryStream stream = new();
_ = Av1FrameEncoder.Encode(
Configuration.Default,
source.Frames.RootFrame,
stream,
colorConfig,
qIndex: 37,
effort);
byte[] payload = stream.ToArray();
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
ObuSequenceHeader sequenceHeader = Assert.IsType<ObuSequenceHeader>(decoder.SequenceHeader);
ObuFrameHeader frameHeader = Assert.IsType<ObuFrameHeader>(decoder.FrameHeader);
Av1FrameInfo frameInfo = Assert.IsType<Av1FrameInfo>(decoder.FrameInfo);
Assert.Equal(enableFilterIntra, sequenceHeader.EnableFilterIntra);
Assert.Equal(enableScreenContentTools, frameHeader.AllowScreenContentTools);
Assert.Equal(enableScreenContentTools, frameHeader.AllowIntraBlockCopy);
Assert.Equal(new Size(width, height), decoded.Size);
int modeCount = 0;
foreach (Av1BlockModeInfo modeInfo in frameInfo.GetSuperblock(Point.Empty).GetModeInfos())
{
modeCount++;
if (effort == 0)
{
Assert.Equal(Av1PredictionMode.DC, modeInfo.YMode);
Assert.Equal(Av1ChromaPredictionMode.DC, modeInfo.UvMode);
}
if (effort <= 1)
{
Assert.Equal(0, modeInfo.GetAngleDelta(Av1Plane.Y));
Assert.Equal(0, modeInfo.GetAngleDelta(Av1Plane.U));
}
if (effort < 4)
{
Assert.False(modeInfo.UseFilterIntra);
}
if (effort < 5)
{
Assert.False(modeInfo.UseIntraBlockCopy);
Assert.Equal(0, modeInfo.GetPaletteSize(Av1Plane.Y));
Assert.Equal(0, modeInfo.GetPaletteSize(Av1Plane.U));
}
}
Assert.NotEqual(0, modeCount);
string outputDirectory = Path.Combine(
TestEnvironment.ActualOutputDirectoryFullPath,
"Formats",
"Heif",
"Av1");
Directory.CreateDirectory(outputDirectory);
File.WriteAllBytes(Path.Combine(outputDirectory, $"encoder-frame-16x16-8b-444-effort-{effort}.obu"), payload);
}
[Fact]
public void EncodeSelectsIntraBlockCopyForRepeatedScreenContent()
{

Write Preview
Loading…
Cancel
Save