tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Preserve libaom non-skip policy for ordinary AV1 intra blocks

pull/2633/head
James Jackson-South 4 weeks ago
parent
c778217a9b
commit
f7bd907d68
5 changed files with 92 additions and 196 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 56
      HEIF_IMPLEMENTATION_PLAN.md
  2. 190
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  3. 5
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.cs
  4. 28
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs
  5. 9
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

56
HEIF_IMPLEMENTATION_PLAN.md
View File

File diff suppressed because one or more lines are too long

190
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
View File

@ -619,42 +619,10 @@ internal static partial class Av1IntraSuperblockEncoder
block.FilterIntraMode = filterIntraMode;
modeInfo.Block.TransformSize = lumaTransformSize;
// Mode decision retains one state for every uniform transform tile in the coding block. The block
// can skip coefficient syntax only when every retained transform has an empty end-of-block marker.
int lumaTransformSampleCount = lumaTransformSize.GetSize2d();
int lumaTransformBlockCount =
(blockSize.GetWidth() * blockSize.GetHeight()) / lumaTransformSampleCount;
int lumaStateStride =
lumaTransformSampleCount / Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
bool lumaTransformEmpty = true;
for (int transformIndex = 0; transformIndex < lumaTransformBlockCount; transformIndex++)
{
lumaTransformEmpty &= retainedLumaStates[transformIndex * lumaStateStride].EndOfBlock == 0;
}
// Ordinary intra keeps the block non-skipped, including when all transforms are empty. Its RD cost
// includes those transform symbols and the non-skip flag; only inter or IBC winners can replace this state.
if (this.source.IsMonochrome)
{
int emptyTransformRate = lumaTransformEmpty
? this.GetEmptyTransformRate(
writer,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Luminance,
blockOrigin,
blockSize,
lumaTransformSize,
modeInfo.Block.Mode,
block.FilterIntraMode)
: 0;
modeInfo.Block.Skip =
!this.picture.Parent.FrameHeader.CodedLossless &&
lumaTransformEmpty && Av1TileWriter.ShouldSkipCoefficients(
writer,
Av1TileWriter.GetSkipContext(macroBlock),
emptyTransformRate);
bool allowIntraBlockCopy = blockSize == Av1BlockSize.Block8x8 &&
this.picture.Parent.FrameHeader.AllowIntraBlockCopy;
@ -662,8 +630,6 @@ internal static partial class Av1IntraSuperblockEncoder
writer,
macroBlock,
lumaCost,
emptyTransformRate,
modeInfo.Block.Skip,
allowIntraBlockCopy);
if (!this.picture.Parent.FrameHeader.IsIntra)
@ -755,68 +721,6 @@ internal static partial class Av1IntraSuperblockEncoder
block.PredictionUnit.ChromaFromLumaSigns = chromaFromLumaSigns;
}
// Skip suppresses coefficient syntax for the entire coding block, not one plane independently.
// Preserve normal coefficient coding when any selected luma or chroma transform is nonempty.
int chromaTransformSampleCount = chromaTransformSize.GetSize2d();
int chromaTransformBlockCount =
(chromaBlockSize.GetWidth() * chromaBlockSize.GetHeight()) / chromaTransformSampleCount;
int chromaStateStride =
chromaTransformSampleCount / Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
bool chromaTransformsEmpty = true;
for (int transformIndex = 0; transformIndex < chromaTransformBlockCount; transformIndex++)
{
int stateIndex = transformIndex * chromaStateStride;
chromaTransformsEmpty &= retainedBlueStates[stateIndex].EndOfBlock == 0 &&
retainedRedStates[stateIndex].EndOfBlock == 0;
}
bool allTransformsEmpty = lumaTransformEmpty && (!block.HasChroma || chromaTransformsEmpty);
int regularEmptyTransformRate = 0;
if (allTransformsEmpty)
{
regularEmptyTransformRate = this.GetEmptyTransformRate(
writer,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Luminance,
blockOrigin,
blockSize,
lumaTransformSize,
modeInfo.Block.Mode,
block.FilterIntraMode);
if (block.HasChroma)
{
regularEmptyTransformRate += this.GetEmptyTransformRate(
writer,
this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Chroma,
chromaOrigin,
chromaBlockSize,
chromaTransformSize,
modeInfo.Block.Mode,
Av1FilterIntraMode.AllFilterIntraModes);
regularEmptyTransformRate += this.GetEmptyTransformRate(
writer,
this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Chroma,
chromaOrigin,
chromaBlockSize,
chromaTransformSize,
modeInfo.Block.Mode,
Av1FilterIntraMode.AllFilterIntraModes);
}
modeInfo.Block.Skip = !this.picture.Parent.FrameHeader.CodedLossless &&
Av1TileWriter.ShouldSkipCoefficients(
writer,
Av1TileWriter.GetSkipContext(macroBlock),
regularEmptyTransformRate);
}
bool allowColorIntraBlockCopy = blockSize == Av1BlockSize.Block8x8 &&
this.picture.Parent.FrameHeader.AllowIntraBlockCopy;
@ -824,8 +728,6 @@ internal static partial class Av1IntraSuperblockEncoder
writer,
macroBlock,
lumaCost + chromaCost,
regularEmptyTransformRate,
modeInfo.Block.Skip,
allowColorIntraBlockCopy);
if (!this.picture.Parent.FrameHeader.IsIntra)
@ -1292,11 +1194,9 @@ internal static partial class Av1IntraSuperblockEncoder
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,
long modeCost,
int emptyTransformRate,
bool skip,
bool allowIntraBlockCopy)
{
int rateAdjustment = writer.GetSkipCost(skip, Av1TileWriter.GetSkipContext(macroBlock));
int rateAdjustment = writer.GetSkipCost(false, Av1TileWriter.GetSkipContext(macroBlock));
if (!this.picture.Parent.FrameHeader.IsIntra)
{
int intraInterContext = Av1TileWriter.GetIntraInterContext(macroBlock);
@ -1308,93 +1208,9 @@ internal static partial class Av1IntraSuperblockEncoder
rateAdjustment += writer.GetUseIntraBlockCopyCost(false);
}
if (skip)
{
// Mode search includes empty transform symbols, while block skip suppresses them from the bitstream.
rateAdjustment -= emptyTransformRate;
}
return modeCost + Av1RateDistortion.GetCost(this.rateMultiplier, rateAdjustment, 0);
}
private int GetEmptyTransformRate(
Av1SymbolEncoder writer,
Av1NeighborArrayUnit<byte> coefficientNeighbors,
Av1ComponentType componentType,
Point blockOrigin,
Av1BlockSize blockSize,
Av1TransformSize transformSize,
Av1PredictionMode lumaMode,
Av1FilterIntraMode filterIntraMode)
{
int blockWidth = blockSize.Get4x4WideCount();
int blockHeight = blockSize.Get4x4HighCount();
int transformWidth = transformSize.Get4x4WideCount();
int transformHeight = transformSize.Get4x4HighCount();
if (blockWidth == transformWidth && blockHeight == transformHeight)
{
Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts(
componentType,
coefficientNeighbors,
blockOrigin,
blockSize,
transformSize);
return writer.GetCoefficientCost(
transformSize,
Av1TransformType.DctDct,
lumaMode,
ReadOnlySpan<int>.Empty,
componentType,
blockContext,
0,
this.picture.Parent.FrameHeader.UseReducedTransformSet,
filterIntraMode,
usesInterTransformSet: false);
}
Span<byte> contexts = this.blockWorkspace
.GetModeDecisionWorkspace<TSample>()
.TransformContexts;
Span<byte> topContexts = contexts[..blockWidth];
Span<byte> leftContexts = contexts.Slice(blockWidth, blockHeight);
int topIndex = coefficientNeighbors.GetTopIndex(blockOrigin);
int leftIndex = coefficientNeighbors.GetLeftIndex(blockOrigin);
coefficientNeighbors.Top.Slice(topIndex, blockWidth).CopyTo(topContexts);
coefficientNeighbors.Left.Slice(leftIndex, blockHeight).CopyTo(leftContexts);
int rate = 0;
for (int blockRow = 0; blockRow < blockHeight; blockRow += transformHeight)
{
for (int blockColumn = 0; blockColumn < blockWidth; blockColumn += transformWidth)
{
Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts(
componentType,
topContexts.Slice(blockColumn, transformWidth),
leftContexts.Slice(blockRow, transformHeight),
blockSize,
transformSize);
rate += writer.GetCoefficientCost(
transformSize,
Av1TransformType.DctDct,
lumaMode,
ReadOnlySpan<int>.Empty,
componentType,
blockContext,
0,
this.picture.Parent.FrameHeader.UseReducedTransformSet,
filterIntraMode,
usesInterTransformSet: false);
topContexts.Slice(blockColumn, transformWidth).Clear();
leftContexts.Slice(blockRow, transformHeight).Clear();
}
}
return rate;
}
private Av1PredictionMode SelectLumaMode(
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,

5
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.cs
View File

@ -245,10 +245,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref lumaState);
this.codedAreaLuma += LumaTransformSize.GetSize2d();
bool skipTransform = lumaState.EndOfBlock == 0;
if (this.source.IsMonochrome)
{
modeInfo.Block.Skip = skipTransform;
return;
}
@ -279,8 +277,7 @@ internal static partial class Av1IntraSuperblockEncoder
this.redCoefficients[this.codedAreaChroma..],
ref redState);
// A block-level skip is valid only when every coded plane reconstructs directly from its prediction.
modeInfo.Block.Skip = skipTransform && blueState.EndOfBlock == 0 && redState.EndOfBlock == 0;
// Ordinary intra retains non-skip syntax even for empty transforms, matching live mode selection.
this.codedAreaChroma += chromaTransformSize.GetSize2d();
}

28
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs
View File

@ -401,6 +401,17 @@ public class Av1EncoderFrameTests
Assert.Equal(Width, decoded.Width);
Assert.Equal(Height, decoded.Height);
Assert.Equal(bitDepth, decoded.BitDepth);
Av1FrameInfo decodedFrameInfo = Assert.IsType<Av1FrameInfo>(decoder.FrameInfo);
foreach (Av1BlockModeInfo mode in decodedFrameInfo.GetModeInfos(Point.Empty, decodedFrameInfo.GetModeInfoCount(Point.Empty)))
{
// The same ordinary-intra policy applies in key and inter frames. Read the emitted syntax,
// rather than infer the skip flag from pixel agreement between encoder and decoder.
if (mode.ReferenceFrames[0] == Av1ReferenceFrameType.Intra && !mode.UseIntraBlockCopy)
{
Assert.False(mode.Skip);
}
}
for (int planeIndex = 0; planeIndex < 3; planeIndex++)
{
Av1Plane plane = (Av1Plane)planeIndex;
@ -501,6 +512,14 @@ public class Av1EncoderFrameTests
null,
null);
Av1FrameInfo firstFrameInfo = Assert.IsType<Av1FrameInfo>(decoder.FrameInfo);
foreach (Av1BlockModeInfo mode in firstFrameInfo.GetModeInfos(Point.Empty, firstFrameInfo.GetModeInfoCount(Point.Empty)))
{
Assert.Equal(Av1ReferenceFrameType.Intra, mode.ReferenceFrames[0]);
Assert.False(mode.UseIntraBlockCopy);
Assert.False(mode.Skip);
}
using ImageFrame<Rgba32> decodedSecond = decoder.DecodeSequenceFrame<Rgba32>(
secondSample.ToArray(),
null,
@ -515,6 +534,15 @@ public class Av1EncoderFrameTests
Assert.Equal(37, frameHeader.QuantizationParameters.BaseQIndex);
Assert.Equal(switchableFilters ? Av1InterpolationFilter.Switchable : Av1InterpolationFilter.Regular, frameHeader.InterpolationFilter);
Assert.Equal(dualFilters, decoder.SequenceHeader.EnableDualFilter);
Av1FrameInfo secondFrameInfo = Assert.IsType<Av1FrameInfo>(decoder.FrameInfo);
bool hasSkippedInterBlock = false;
foreach (Av1BlockModeInfo mode in secondFrameInfo.GetModeInfos(Point.Empty, secondFrameInfo.GetModeInfoCount(Point.Empty)))
{
hasSkippedInterBlock |= mode.ReferenceFrames[0] == Av1ReferenceFrameType.Last && mode.Skip;
}
// Repeated frames still use the inter skip alternative when prediction supplies the retained samples.
Assert.True(hasSkippedInterBlock);
for (int y = 0; y < Height; y++)
{
Assert.Equal(

9
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs
View File

@ -463,7 +463,7 @@ public class Av1IntraSuperblockEncoderTests
Assert.Equal(Av1TransformSize.Size8x8, block.Block.TransformSize);
Assert.Equal(Av1PredictionMode.DC, block.Block.Mode);
Assert.Equal(Av1ChromaPredictionMode.DC, block.Block.UvMode);
Assert.True(block.Block.Skip);
Assert.False(block.Block.Skip);
}
Span<Av1EncoderTransformBlockState> lumaStates = coefficients.GetTransformBlockSpan(0, Av1Plane.Y);
@ -602,7 +602,7 @@ public class Av1IntraSuperblockEncoderTests
[Theory]
[InlineData(true)]
[InlineData(false)]
public void MarksAllZeroTransformBlockAsSkipped(bool isMonochrome)
public void PreservesIntraNonSkipForAllZeroTransforms(bool isMonochrome)
{
const int Width = 8;
const int Height = 8;
@ -678,7 +678,10 @@ public class Av1IntraSuperblockEncoderTests
blockWorkspace);
ref Av1MacroBlockModeInfo block = ref picture.GetMacroBlockModeInfo(default);
Assert.True(block.Block.Skip);
// Ordinary intra blocks retain the non-skip flag and empty transform symbols. Libaom applies
// this policy before final coding even when skipping would reconstruct the same samples.
Assert.False(block.Block.Skip);
Assert.Equal((ushort)0, coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[0].EndOfBlock);
if (!isMonochrome)
{

Write Preview
Loading…
Cancel
Save