Browse Source

Add AV1 intra-block-copy mode selection

pull/2633/head
James Jackson-South 1 month ago
parent
commit
87a11a4c28
  1. 6
      HEIF_IMPLEMENTATION_PLAN.md
  2. 4
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs
  3. 528
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs
  4. 74
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  5. 94
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs
  6. 287
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

6
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

4
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs

@ -56,7 +56,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderPaletteInfo paletteInfo,
out int selectedAngleDelta,
out byte selectedChromaFromLumaIndex,
out sbyte selectedChromaFromLumaSigns)
out sbyte selectedChromaFromLumaSigns,
out long selectedCost)
{
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const int MaximumSampleCount = 8 * 8;
@ -469,6 +470,7 @@ internal static partial class Av1IntraSuperblockEncoder
selectedChromaFromLumaSigns = 0;
}
selectedCost = bestCost;
return bestMode;
}

528
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs

@ -0,0 +1,528 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
using SixLabors.ImageSharp.Formats.Heif.Av1.Entropy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <content>
/// Provides full rate-distortion selection for intra-block-copy candidates.
/// </content>
internal static partial class Av1IntraSuperblockEncoder
{
internal partial struct ModeDecision<TSample, TOperator>
where TSample : unmanaged
where TOperator : struct, IBlockEncodingOperator<TSample>
{
private void SelectIntraBlockCopy(
Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock,
Point blockOrigin,
ushort tileIndex,
long regularModeCost,
int regularEmptyTransformRate,
ref Av1MacroBlockModeInfo modeInfo,
ref Av1EncoderBlockStruct block,
ref Av1EncoderPaletteInfo paletteInfo)
{
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8;
const int MaximumSampleCount = 8 * 8;
Buffer2DRegion<TSample> lumaSource = this.source.GetPlane(Av1Plane.Y);
Buffer2DRegion<TSample> lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y);
Point modeInfoPosition = new(
blockOrigin.X >> Av1Constants.ModeInfoSizeLog2,
blockOrigin.Y >> Av1Constants.ModeInfoSizeLog2);
Span<Av1MotionVector> referenceCandidates = stackalloc Av1MotionVector[8];
Span<int> referenceWeights = stackalloc int[8];
Av1MotionVector reference = Av1IntraBlockCopy.FindReference(
this.picture,
macroBlock,
modeInfoPosition,
BlockSize,
Av1PartitionType.None,
referenceCandidates,
referenceWeights);
Span<Av1MotionVector> candidates = stackalloc Av1MotionVector[4];
Av1IntraBlockCopySearchIndex search = this.picture.IntraBlockCopySearch;
int candidateCount = search.FindCandidates<TSample, TOperator>(
lumaSource,
lumaReconstruction,
blockOrigin,
macroBlock.Tile,
this.picture.Sequence.SequenceHeader,
writer,
reference,
this.rateMultiplier,
candidates);
candidateCount += search.FindPixelCandidates<TSample, TOperator>(
lumaSource,
lumaReconstruction,
blockOrigin,
macroBlock.Tile,
this.picture.Sequence.SequenceHeader,
writer,
reference,
this.quantization.QIndex[0],
this.rateMultiplier,
candidates[candidateCount..]);
// Hash and full-pixel searches can converge on the same vector. Preserve the first search-order
// occurrence so repeated vectors do not pay for duplicate transform searches or alter ties.
int uniqueCandidateCount = 0;
for (int candidateIndex = 0; candidateIndex < candidateCount; candidateIndex++)
{
Av1MotionVector candidate = candidates[candidateIndex];
bool duplicate = false;
for (int uniqueIndex = 0; uniqueIndex < uniqueCandidateCount; uniqueIndex++)
{
if (candidate == candidates[uniqueIndex])
{
duplicate = true;
break;
}
}
if (!duplicate)
{
candidates[uniqueCandidateCount++] = candidate;
}
}
if (uniqueCandidateCount == 0)
{
return;
}
int skipContext = Av1TileWriter.GetSkipContext(macroBlock);
int regularRateAdjustment = writer.GetUseIntraBlockCopyCost(false) +
writer.GetSkipCost(modeInfo.Block.Skip, skipContext);
if (modeInfo.Block.Skip)
{
regularRateAdjustment -= regularEmptyTransformRate;
}
long bestCost = regularModeCost + Av1RateDistortion.GetCost(this.rateMultiplier, regularRateAdjustment, 0);
bool hasSelectedCandidate = false;
bool selectedSkip = false;
Av1MotionVector selectedVector = default;
Av1EncoderTransformBlockState selectedLumaState = default;
Av1EncoderTransformBlockState selectedBlueState = default;
Av1EncoderTransformBlockState selectedRedState = default;
Span<TSample> selectedLumaReconstruction = stackalloc TSample[MaximumSampleCount];
Span<TSample> selectedBlueReconstruction = stackalloc TSample[MaximumSampleCount];
Span<TSample> selectedRedReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> selectedLumaCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedBlueCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedRedCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> lumaPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> lumaResidual = stackalloc short[MaximumSampleCount];
Span<TSample> lumaCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> lumaCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> bluePrediction = stackalloc TSample[MaximumSampleCount];
Span<short> blueResidual = stackalloc short[MaximumSampleCount];
Span<TSample> blueCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> blueCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> redPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> redResidual = stackalloc short[MaximumSampleCount];
Span<TSample> redCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> redCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> transformReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> transformCoefficients = stackalloc int[MaximumSampleCount];
Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Luminance,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
blockOrigin,
BlockSize,
LumaTransformSize);
ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig;
int subsamplingX = colorConfig.SubSamplingX ? 1 : 0;
int subsamplingY = colorConfig.SubSamplingY ? 1 : 0;
Point chromaOrigin = new(blockOrigin.X >> subsamplingX, blockOrigin.Y >> subsamplingY);
Av1TransformSize chromaTransformSize = BlockSize.GetMaxUvTransformSize(
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
Av1TransformBlockContext blueContext = default;
Av1TransformBlockContext redContext = default;
if (!this.source.IsMonochrome)
{
Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled(
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
blueContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Chroma,
this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex],
chromaOrigin,
chromaBlockSize,
chromaTransformSize);
redContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Chroma,
this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex],
chromaOrigin,
chromaBlockSize,
chromaTransformSize);
}
for (int candidateIndex = 0; candidateIndex < uniqueCandidateCount; candidateIndex++)
{
Av1MotionVector candidate = candidates[candidateIndex];
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.Y,
Av1ComponentType.Luminance,
blockOrigin,
0,
0,
LumaTransformSize,
lumaContext,
lumaPrediction,
lumaResidual,
transformReconstruction,
transformCoefficients,
lumaCandidateReconstruction,
lumaCandidateCoefficients,
out Av1EncoderTransformBlockState lumaCandidateState,
out int lumaRate,
out long lumaDistortion,
out bool hasEmptyLuma,
out Av1EncoderTransformBlockState emptyLumaState,
out long emptyLumaDistortion);
int blueRate = 0;
int redRate = 0;
long blueDistortion = 0;
long redDistortion = 0;
long emptyBlueDistortion = 0;
long emptyRedDistortion = 0;
bool hasEmptyBlue = true;
bool hasEmptyRed = true;
Av1EncoderTransformBlockState blueCandidateState = default;
Av1EncoderTransformBlockState redCandidateState = default;
Av1EncoderTransformBlockState emptyBlueState = default;
Av1EncoderTransformBlockState emptyRedState = default;
if (!this.source.IsMonochrome)
{
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.U,
Av1ComponentType.Chroma,
blockOrigin,
subsamplingX,
subsamplingY,
chromaTransformSize,
blueContext,
bluePrediction,
blueResidual,
transformReconstruction,
transformCoefficients,
blueCandidateReconstruction,
blueCandidateCoefficients,
out blueCandidateState,
out blueRate,
out blueDistortion,
out hasEmptyBlue,
out emptyBlueState,
out emptyBlueDistortion);
this.EvaluateIntraBlockCopyPlane(
writer,
candidate,
Av1Plane.V,
Av1ComponentType.Chroma,
blockOrigin,
subsamplingX,
subsamplingY,
chromaTransformSize,
redContext,
redPrediction,
redResidual,
transformReconstruction,
transformCoefficients,
redCandidateReconstruction,
redCandidateCoefficients,
out redCandidateState,
out redRate,
out redDistortion,
out hasEmptyRed,
out emptyRedState,
out emptyRedDistortion);
}
int displacementRate = writer.GetDisplacementVectorCost(candidate, reference);
int candidateRate = writer.GetUseIntraBlockCopyCost(true) +
displacementRate +
writer.GetSkipCost(false, skipContext) +
lumaRate +
blueRate +
redRate;
long candidateDistortion = lumaDistortion + blueDistortion + redDistortion;
long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, candidateDistortion);
bool candidateSkip = false;
if (hasEmptyLuma && hasEmptyBlue && hasEmptyRed)
{
int skipRate = writer.GetUseIntraBlockCopyCost(true) +
displacementRate +
writer.GetSkipCost(true, skipContext);
long skipDistortion = emptyLumaDistortion + emptyBlueDistortion + emptyRedDistortion;
long skipCost = Av1RateDistortion.GetCost(this.rateMultiplier, skipRate, skipDistortion);
if (skipCost < candidateCost)
{
candidateCost = skipCost;
candidateSkip = true;
}
}
// Conventional intra and earlier IBC vectors retain strict search-order precedence on equal RD.
if (candidateCost >= bestCost)
{
continue;
}
bestCost = candidateCost;
hasSelectedCandidate = true;
selectedSkip = candidateSkip;
selectedVector = candidate;
if (candidateSkip)
{
lumaPrediction.CopyTo(selectedLumaReconstruction);
selectedLumaCoefficients.Clear();
selectedLumaState = emptyLumaState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
bluePrediction[..chromaSampleCount].CopyTo(selectedBlueReconstruction);
redPrediction[..chromaSampleCount].CopyTo(selectedRedReconstruction);
selectedBlueCoefficients[..chromaSampleCount].Clear();
selectedRedCoefficients[..chromaSampleCount].Clear();
selectedBlueState = emptyBlueState;
selectedRedState = emptyRedState;
}
}
else
{
lumaCandidateReconstruction.CopyTo(selectedLumaReconstruction);
lumaCandidateCoefficients.CopyTo(selectedLumaCoefficients);
selectedLumaState = lumaCandidateState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
blueCandidateReconstruction[..chromaSampleCount].CopyTo(selectedBlueReconstruction);
redCandidateReconstruction[..chromaSampleCount].CopyTo(selectedRedReconstruction);
blueCandidateCoefficients[..chromaSampleCount].CopyTo(selectedBlueCoefficients);
redCandidateCoefficients[..chromaSampleCount].CopyTo(selectedRedCoefficients);
selectedBlueState = blueCandidateState;
selectedRedState = redCandidateState;
}
}
}
if (!hasSelectedCandidate)
{
return;
}
Span<int> retainedLumaCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.Y);
Span<Av1EncoderTransformBlockState> retainedLumaTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.Y);
int lumaTransformIndex = this.codedAreaLuma /
Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex];
CopyCandidate(
selectedLumaReconstruction,
selectedLumaCoefficients,
lumaReconstruction,
blockOrigin,
retainedLumaCoefficients[this.codedAreaLuma..],
LumaTransformSize,
selectedLumaState,
ref retainedLumaState);
if (!this.source.IsMonochrome)
{
Span<int> retainedBlueCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.U);
Span<int> retainedRedCoefficients = this.coefficientBuffer.GetPlaneSpan(this.superblock.Index, Av1Plane.V);
Span<Av1EncoderTransformBlockState> retainedBlueTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.U);
Span<Av1EncoderTransformBlockState> retainedRedTransformBlocks =
this.coefficientBuffer.GetTransformBlockSpan(this.superblock.Index, Av1Plane.V);
int chromaTransformIndex = this.codedAreaChroma /
Av1EncoderCoefficientBuffer.TransformBlockUnitCoefficientCount;
ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex];
ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex];
CopyCandidate(
selectedBlueReconstruction,
selectedBlueCoefficients,
this.reconstruction.GetPlane(Av1Plane.U),
chromaOrigin,
retainedBlueCoefficients[this.codedAreaChroma..],
chromaTransformSize,
selectedBlueState,
ref retainedBlueState);
CopyCandidate(
selectedRedReconstruction,
selectedRedCoefficients,
this.reconstruction.GetPlane(Av1Plane.V),
chromaOrigin,
retainedRedCoefficients[this.codedAreaChroma..],
chromaTransformSize,
selectedRedState,
ref retainedRedState);
}
modeInfo.Block.Mode = Av1PredictionMode.DC;
modeInfo.Block.UvMode = Av1ChromaPredictionMode.DC;
modeInfo.Block.Skip = selectedSkip;
modeInfo.Block.UseIntraBlockCopy = true;
block.FilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = 0;
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = 0;
block.PredictionUnit.ChromaFromLumaIndex = 0;
block.PredictionUnit.ChromaFromLumaSigns = 0;
paletteInfo = default;
this.picture.SetDisplacementVector(modeInfoPosition, selectedVector);
}
private void EvaluateIntraBlockCopyPlane(
Av1SymbolEncoder writer,
Av1MotionVector vector,
Av1Plane plane,
Av1ComponentType componentType,
Point lumaOrigin,
int subsamplingX,
int subsamplingY,
Av1TransformSize transformSize,
Av1TransformBlockContext blockContext,
Span<TSample> prediction,
Span<short> residual,
Span<TSample> transformReconstruction,
Span<int> transformCoefficients,
Span<TSample> selectedReconstruction,
Span<int> selectedCoefficients,
out Av1EncoderTransformBlockState selectedState,
out int selectedRate,
out long selectedDistortion,
out bool hasEmptyTransform,
out Av1EncoderTransformBlockState emptyState,
out long emptyDistortion)
{
Point planeOrigin = new(lumaOrigin.X >> subsamplingX, lumaOrigin.Y >> subsamplingY);
int sourceColumnQ4 = (planeOrigin.X << 4) + (vector.Column << (1 - subsamplingX));
int sourceRowQ4 = (planeOrigin.Y << 4) + (vector.Row << (1 - subsamplingY));
Point predictionOrigin = new(sourceColumnQ4 >> 4, sourceRowQ4 >> 4);
int sampleCount = transformSize.GetSize2d();
Buffer2DRegion<TSample> sourcePlane = this.source.GetPlane(plane);
Buffer2DRegion<TSample> reconstructionPlane = this.reconstruction.GetPlane(plane);
TOperator.PrepareIntraBlockCopy(
sourcePlane,
planeOrigin,
reconstructionPlane,
predictionOrigin,
(sourceColumnQ4 & 15) != 0,
(sourceRowQ4 & 15) != 0,
prediction[..sampleCount],
residual[..sampleCount],
transformSize);
Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType(
transformSize,
isInter: true,
this.picture.Parent.FrameHeader.UseReducedTransformSet);
long bestCost = long.MaxValue;
selectedState = default;
selectedRate = 0;
selectedDistortion = 0;
hasEmptyTransform = false;
emptyState = default;
emptyDistortion = 0;
for (Av1TransformType transformType = Av1TransformType.DctDct;
transformType < Av1TransformType.AllTransformTypes;
transformType++)
{
if (!transformType.IsExtendedSetUsed(transformSetType))
{
continue;
}
Av1EncoderTransformBlockState candidateState = default;
long candidateDistortion = TOperator.EncodePredictionCandidate(
this.blockWorkspace,
sourcePlane,
planeOrigin,
prediction[..sampleCount],
residual[..sampleCount],
transformReconstruction[..sampleCount],
transformCoefficients[..sampleCount],
transformSize,
transformType,
plane,
this.quantization.QIndex[0],
this.quantization.DeltaQDc[(int)plane],
this.quantization.DeltaQAc[(int)plane],
this.bitDepth,
ref candidateState);
int candidateRate = writer.GetCoefficientCost(
transformSize,
transformType,
Av1PredictionMode.DC,
transformCoefficients[..sampleCount],
componentType,
blockContext,
candidateState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: true);
long candidateCost = Av1RateDistortion.GetCost(
this.rateMultiplier,
candidateRate,
candidateDistortion);
if (candidateCost < bestCost)
{
transformReconstruction[..sampleCount].CopyTo(selectedReconstruction);
transformCoefficients[..sampleCount].CopyTo(selectedCoefficients);
bestCost = candidateCost;
selectedState = candidateState;
selectedRate = candidateRate;
selectedDistortion = candidateDistortion;
}
if (candidateState.EndOfBlock == 0 &&
(!hasEmptyTransform || candidateDistortion < emptyDistortion))
{
hasEmptyTransform = true;
emptyState = candidateState;
emptyDistortion = candidateDistortion;
}
}
}
}
}

74
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs

@ -195,30 +195,49 @@ internal static partial class Av1IntraSuperblockEncoder
ref lumaState,
ref paletteInfo,
out int lumaAngleDelta,
out Av1FilterIntraMode filterIntraMode);
out Av1FilterIntraMode filterIntraMode,
out long lumaCost);
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Y] = (sbyte)lumaAngleDelta;
block.FilterIntraMode = filterIntraMode;
this.codedAreaLuma += LumaTransformSize.GetSize2d();
bool lumaTransformEmpty = lumaState.EndOfBlock == 0;
if (this.source.IsMonochrome)
{
int emptyTransformRate = lumaTransformEmpty
? this.GetEmptyTransformRate(
writer,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Luminance,
blockOrigin,
BlockSize,
LumaTransformSize,
lumaState.TransformType,
modeInfo.Block.Mode,
block.FilterIntraMode)
: 0;
modeInfo.Block.Skip = lumaTransformEmpty &&
Av1TileWriter.ShouldSkipCoefficients(
writer,
Av1TileWriter.GetSkipContext(macroBlock),
this.GetEmptyTransformRate(
writer,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Luminance,
blockOrigin,
BlockSize,
LumaTransformSize,
lumaState.TransformType,
modeInfo.Block.Mode,
block.FilterIntraMode));
emptyTransformRate);
if (this.picture.Parent.FrameHeader.AllowIntraBlockCopy)
{
this.SelectIntraBlockCopy(
writer,
macroBlock,
blockOrigin,
tileIndex,
lumaCost,
emptyTransformRate,
ref modeInfo,
ref block,
ref paletteInfo);
}
this.codedAreaLuma += LumaTransformSize.GetSize2d();
return;
}
@ -258,20 +277,22 @@ internal static partial class Av1IntraSuperblockEncoder
ref paletteInfo,
out int chromaAngleDelta,
out byte chromaFromLumaIndex,
out sbyte chromaFromLumaSigns);
out sbyte chromaFromLumaSigns,
out long chromaCost);
block.PredictionUnit.AngleDelta[(int)Av1PlaneType.Uv] = (sbyte)chromaAngleDelta;
block.PredictionUnit.ChromaFromLumaIndex = chromaFromLumaIndex;
block.PredictionUnit.ChromaFromLumaSigns = chromaFromLumaSigns;
bool allTransformsEmpty = lumaTransformEmpty && blueState.EndOfBlock == 0 && redState.EndOfBlock == 0;
int regularEmptyTransformRate = 0;
if (allTransformsEmpty)
{
Av1BlockSize chromaBlockSize = BlockSize.GetSubsampled(
colorConfig.SubSamplingX,
colorConfig.SubSamplingY);
int emptyTransformRate = this.GetEmptyTransformRate(
regularEmptyTransformRate = this.GetEmptyTransformRate(
writer,
this.picture.LuminanceDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Luminance,
@ -282,7 +303,7 @@ internal static partial class Av1IntraSuperblockEncoder
modeInfo.Block.Mode,
block.FilterIntraMode);
emptyTransformRate += this.GetEmptyTransformRate(
regularEmptyTransformRate += this.GetEmptyTransformRate(
writer,
this.picture.CbDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Chroma,
@ -293,7 +314,7 @@ internal static partial class Av1IntraSuperblockEncoder
modeInfo.Block.Mode,
Av1FilterIntraMode.AllFilterIntraModes);
emptyTransformRate += this.GetEmptyTransformRate(
regularEmptyTransformRate += this.GetEmptyTransformRate(
writer,
this.picture.CrDcSignLevelCoefficientNeighbors[tileIndex],
Av1ComponentType.Chroma,
@ -307,9 +328,24 @@ internal static partial class Av1IntraSuperblockEncoder
modeInfo.Block.Skip = Av1TileWriter.ShouldSkipCoefficients(
writer,
Av1TileWriter.GetSkipContext(macroBlock),
emptyTransformRate);
regularEmptyTransformRate);
}
if (this.picture.Parent.FrameHeader.AllowIntraBlockCopy)
{
this.SelectIntraBlockCopy(
writer,
macroBlock,
blockOrigin,
tileIndex,
lumaCost + chromaCost,
regularEmptyTransformRate,
ref modeInfo,
ref block,
ref paletteInfo);
}
this.codedAreaLuma += LumaTransformSize.GetSize2d();
this.codedAreaChroma += chromaTransformSize.GetSize2d();
}
@ -352,7 +388,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderTransformBlockState retainedState,
ref Av1EncoderPaletteInfo paletteInfo,
out int selectedAngleDelta,
out Av1FilterIntraMode selectedFilterIntraMode)
out Av1FilterIntraMode selectedFilterIntraMode,
out long selectedCost)
{
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize TransformSize = Av1TransformSize.Size8x8;
@ -695,6 +732,7 @@ internal static partial class Av1IntraSuperblockEncoder
selectedFilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes;
}
selectedCost = bestTransformCost;
return bestMode;
}

94
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs

@ -6,6 +6,7 @@ using System.Runtime.Intrinsics;
using SixLabors.ImageSharp.Formats.Heif.Av1.Motion;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.IntraBlockCopy;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
using SixLabors.ImageSharp.Memory;
@ -215,6 +216,29 @@ internal static partial class Av1IntraSuperblockEncoder
Av1TransformSize transformSize,
Av1BitDepth bitDepth);
/// <summary>
/// Builds an intra-block-copy prediction and the matching source residual.
/// </summary>
/// <param name="source">The coded source plane.</param>
/// <param name="blockOrigin">The destination block origin in plane samples.</param>
/// <param name="reconstruction">The reconstructed plane containing the reference samples.</param>
/// <param name="predictionOrigin">The integer reference origin preceding any half-sample phase.</param>
/// <param name="halfX">Indicates whether the horizontal source phase is one half-sample.</param>
/// <param name="halfY">Indicates whether the vertical source phase is one half-sample.</param>
/// <param name="prediction">The contiguous prediction destination.</param>
/// <param name="residual">The contiguous source-minus-prediction destination.</param>
/// <param name="transformSize">The prediction dimensions.</param>
public static abstract void PrepareIntraBlockCopy(
Buffer2DRegion<TSample> source,
Point blockOrigin,
Buffer2DRegion<TSample> reconstruction,
Point predictionOrigin,
bool halfX,
bool halfY,
Span<TSample> prediction,
Span<short> residual,
Av1TransformSize transformSize);
/// <summary>
/// Encodes one prepared prediction with the selected transform into decision scratch.
/// </summary>
@ -715,6 +739,41 @@ internal static partial class Av1IntraSuperblockEncoder
height);
}
/// <inheritdoc/>
public static void PrepareIntraBlockCopy(
Buffer2DRegion<byte> source,
Point blockOrigin,
Buffer2DRegion<byte> reconstruction,
Point predictionOrigin,
bool halfX,
bool halfY,
Span<byte> prediction,
Span<short> residual,
Av1TransformSize transformSize)
{
int width = transformSize.GetWidth();
int height = transformSize.GetHeight();
Av1IntraBlockCopyPredictor.Predict(
Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin),
reconstruction.Stride,
prediction,
width,
width,
height,
halfX,
halfY);
Av1ResidualBuilder.Subtract(
Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin),
source.Stride,
prediction,
width,
residual,
width,
width,
height);
}
/// <inheritdoc/>
public static long EncodePredictionCandidate(
Av1EncoderBlockWorkspace workspace,
@ -1197,6 +1256,41 @@ internal static partial class Av1IntraSuperblockEncoder
height);
}
/// <inheritdoc/>
public static void PrepareIntraBlockCopy(
Buffer2DRegion<ushort> source,
Point blockOrigin,
Buffer2DRegion<ushort> reconstruction,
Point predictionOrigin,
bool halfX,
bool halfY,
Span<ushort> prediction,
Span<short> residual,
Av1TransformSize transformSize)
{
int width = transformSize.GetWidth();
int height = transformSize.GetHeight();
Av1IntraBlockCopyPredictor.Predict(
MemoryMarshal.Cast<ushort, short>(Av1TransformBlockEncoder.GetPlaneSpan(reconstruction, predictionOrigin)),
reconstruction.Stride,
MemoryMarshal.Cast<ushort, short>(prediction),
width,
width,
height,
halfX,
halfY);
Av1ResidualBuilder.Subtract(
Av1TransformBlockEncoder.GetPlaneSpan(source, blockOrigin),
source.Stride,
prediction,
width,
residual,
width,
width,
height);
}
/// <inheritdoc/>
public static long EncodePredictionCandidate(
Av1EncoderBlockWorkspace workspace,

287
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

@ -1864,6 +1864,290 @@ public class Av1IntraSuperblockEncoderTests
Assert.NotEqual(0, tileWriter.GetTileData(0).Length);
}
[Fact]
public void ProductionTileSelectsIntraBlockCopyByFullRateDistortion()
{
VerifyProductionTileSelectsIntraBlockCopy(
Av1BitDepth.EightBit,
8,
static value => (byte)value,
static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter(
Configuration.Default,
source,
reconstruction,
picture,
coefficients,
superblockWorkspace,
blockWorkspace,
initialSize: 4096));
VerifyProductionTileSelectsIntraBlockCopy(
Av1BitDepth.TwelveBit,
12,
static value => (ushort)(value << 4),
static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter(
Configuration.Default,
source,
reconstruction,
picture,
coefficients,
superblockWorkspace,
blockWorkspace,
initialSize: 4096));
}
private static void VerifyProductionTileSelectsIntraBlockCopy<TSample>(
Av1BitDepth bitDepth,
int bitDepthValue,
SampleFactory<TSample> createSample,
TileWriterFactory<TSample> createTileWriter)
where TSample : unmanaged
{
const int Width = 328;
const int Height = 8;
const int QIndex = 1;
const int ReferenceColumn = 0;
const int TargetColumn = 320;
ObuColorConfig colorConfig = new()
{
IsMonochrome = true,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = bitDepth
};
using Av1EncoderFrameBuffer<TSample> source = new(
Configuration.Default,
Width,
Height,
bitDepthValue,
Av1ColorFormat.Yuv400,
0,
0);
using Av1EncoderFrameBuffer<TSample> reconstruction = new(
Configuration.Default,
Width,
Height,
bitDepthValue,
Av1ColorFormat.Yuv400,
0,
0);
Buffer2DRegion<TSample> sourcePlane = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++)
{
Span<TSample> sourceRow = sourcePlane.DangerousGetRowSpan(row);
for (int column = 0; column < Width; column++)
{
sourceRow[column] = createSample(17 + (((column * 29) + (row * 43)) % 211));
}
for (int column = 0; column < 8; column++)
{
// The repeated high-contrast block has one legal hash match five completed 64-pixel regions earlier.
TSample sample = createSample(((column * 73) + (row * 109) + (((column + row) & 1) * 127)) & 255);
sourceRow[ReferenceColumn + column] = sample;
sourceRow[TargetColumn + column] = sample;
}
}
ClearPlane(reconstruction.Luma);
using Av1EncoderModeInfoBuffer modeInfo = new(
Configuration.Default,
Width,
Height,
disallow4x4AllFrames: true);
Av1PictureControlSet pictureTemplate = CreatePicture(
modeInfo,
colorConfig,
use128x128Superblock: false,
QIndex);
pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true;
pictureTemplate.Parent.FrameHeader.AllowIntraBlockCopy = true;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = Width;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = Height;
using Av1EncoderPictureBuffer picture = new(
Configuration.Default,
pictureTemplate.Sequence.SequenceHeader,
pictureTemplate.Parent.FrameHeader,
Width,
Height);
using Av1EncoderCoefficientBuffer coefficients = new(
Configuration.Default,
pictureTemplate.Sequence.SequenceHeader,
Width,
Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(Configuration.Default);
using Av1EncoderBlockWorkspace blockWorkspace = new(Configuration.Default);
using Av1IntraTileWriter tileWriter = createTileWriter(
source.Frame,
reconstruction.Frame,
picture.Picture,
coefficients,
superblockWorkspace,
blockWorkspace);
Point targetModeInfoPosition = new(TargetColumn >> Av1Constants.ModeInfoSizeLog2, 0);
ref Av1MacroBlockModeInfo targetMode = ref picture.Picture.GetMacroBlockModeInfo(targetModeInfoPosition);
Assert.True(targetMode.Block.UseIntraBlockCopy);
Assert.Equal(Av1PredictionMode.DC, targetMode.Block.Mode);
Assert.Equal(Av1ChromaPredictionMode.DC, targetMode.Block.UvMode);
var displacementVector = picture.Picture.GetDisplacementVector(targetModeInfoPosition);
Assert.Equal(0, displacementVector.Row);
Assert.Equal((ReferenceColumn - TargetColumn) * 8, displacementVector.Column);
Assert.Equal(Av1FilterIntraMode.AllFilterIntraModes, superblockWorkspace.FinalBlocks[0].FilterIntraMode);
Assert.Equal(0, superblockWorkspace.PaletteInfo.PaletteSizes[0]);
Assert.NotEqual(0, tileWriter.GetTileData(0).Length);
}
[Fact]
public void ProductionTileRetainsHalfSampleChromaIntraBlockCopy()
{
const int Width = 328;
const int Height = 8;
const int QIndex = 1;
const int ReferenceColumn = 1;
const int TargetColumn = 320;
ObuColorConfig colorConfig = new()
{
IsMonochrome = false,
SubSamplingX = true,
SubSamplingY = true,
BitDepth = Av1BitDepth.EightBit
};
using Av1EncoderFrameBuffer<byte> source = new(
Configuration.Default,
Width,
Height,
8,
Av1ColorFormat.Yuv420,
1,
1);
using Av1EncoderFrameBuffer<byte> reconstruction = new(
Configuration.Default,
Width,
Height,
8,
Av1ColorFormat.Yuv420,
1,
1);
Buffer2DRegion<byte> lumaSource = source.Frame.CodedView.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++)
{
Span<byte> lumaRow = lumaSource.DangerousGetRowSpan(row);
for (int column = 0; column < Width; column++)
{
lumaRow[column] = (byte)(23 + (((column * 31) + (row * 47)) % 197));
}
for (int column = 0; column < 8; column++)
{
byte sample = (byte)(((column * 79) + (row * 113) + (((column + row) & 1) * 127)) & 255);
lumaRow[ReferenceColumn + column] = sample;
lumaRow[TargetColumn + column] = sample;
}
}
int chromaTargetColumn = TargetColumn >> 1;
Buffer2DRegion<byte> blueSource = source.Frame.CodedView.GetPlane(Av1Plane.U);
Buffer2DRegion<byte> redSource = source.Frame.CodedView.GetPlane(Av1Plane.V);
for (int row = 0; row < Height >> 1; row++)
{
Span<byte> blueRow = blueSource.DangerousGetRowSpan(row);
Span<byte> redRow = redSource.DangerousGetRowSpan(row);
for (int column = 0; column < Width >> 1; column++)
{
blueRow[column] = (byte)(32 + (((column * 17) + (row * 29)) % 160));
redRow[column] = (byte)(40 + (((column * 23) + (row * 37)) % 152));
}
for (int column = 0; column < 4; column++)
{
// An odd luma displacement maps 4:2:0 chroma between adjacent reference samples.
blueRow[chromaTargetColumn + column] = (byte)((blueRow[column] + blueRow[column + 1] + 1) >> 1);
redRow[chromaTargetColumn + column] = (byte)((redRow[column] + redRow[column + 1] + 1) >> 1);
}
}
ClearPlane(reconstruction.Luma);
ClearPlane(Assert.IsType<Buffer2D<byte>>(reconstruction.ChromaBlue));
ClearPlane(Assert.IsType<Buffer2D<byte>>(reconstruction.ChromaRed));
using Av1EncoderModeInfoBuffer modeInfo = new(
Configuration.Default,
Width,
Height,
disallow4x4AllFrames: true);
Av1PictureControlSet pictureTemplate = CreatePicture(
modeInfo,
colorConfig,
use128x128Superblock: false,
QIndex);
pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true;
pictureTemplate.Parent.FrameHeader.AllowIntraBlockCopy = true;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = Width;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = Height;
using Av1EncoderPictureBuffer picture = new(
Configuration.Default,
pictureTemplate.Sequence.SequenceHeader,
pictureTemplate.Parent.FrameHeader,
Width,
Height);
using Av1EncoderCoefficientBuffer coefficients = new(
Configuration.Default,
pictureTemplate.Sequence.SequenceHeader,
Width,
Height);
using Av1EncoderSuperblockWorkspace superblockWorkspace = new(Configuration.Default);
using Av1EncoderBlockWorkspace blockWorkspace = new(Configuration.Default);
using Av1IntraTileWriter tileWriter = new(
Configuration.Default,
source.Frame,
reconstruction.Frame,
picture.Picture,
coefficients,
superblockWorkspace,
blockWorkspace,
initialSize: 4096);
Point targetModeInfoPosition = new(TargetColumn >> Av1Constants.ModeInfoSizeLog2, 0);
ref Av1MacroBlockModeInfo targetMode = ref picture.Picture.GetMacroBlockModeInfo(targetModeInfoPosition);
Assert.True(targetMode.Block.UseIntraBlockCopy);
var displacementVector = picture.Picture.GetDisplacementVector(targetModeInfoPosition);
Assert.Equal(0, displacementVector.Row);
Assert.Equal((ReferenceColumn - TargetColumn) * 8, displacementVector.Column);
Assert.Equal(8, displacementVector.Column & 15);
Av1TransformSetType interTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType(
Av1TransformSize.Size4x4,
isInter: true,
useReducedSet: false);
Assert.True(coefficients.GetTransformBlockSpan(5, Av1Plane.U)[0].TransformType.IsExtendedSetUsed(interTransformSet));
Assert.True(coefficients.GetTransformBlockSpan(5, Av1Plane.V)[0].TransformType.IsExtendedSetUsed(interTransformSet));
Assert.NotEqual(
(byte)0,
reconstruction.Frame.CodedView.GetPlane(Av1Plane.U).DangerousGetRowSpan(0)[chromaTargetColumn]);
Assert.NotEqual(
(byte)0,
reconstruction.Frame.CodedView.GetPlane(Av1Plane.V).DangerousGetRowSpan(0)[chromaTargetColumn]);
Assert.NotEqual(0, tileWriter.GetTileData(0).Length);
}
[Fact]
public void TileWriterMapsClippedRasterTraversalToEverySuperblockCoefficientSegment()
{
@ -2263,6 +2547,9 @@ public class Av1IntraSuperblockEncoderTests
Av1EncoderBlockWorkspace blockWorkspace)
where TSample : unmanaged;
private delegate TSample SampleFactory<TSample>(int value)
where TSample : unmanaged;
private delegate void FilterPrediction<TSample>(
Av1FilterIntraMode mode,
Span<TSample> destination,

Loading…
Cancel
Save