Browse Source

Complete palette transform-size selection

pull/2633/head
James Jackson-South 4 weeks ago
parent
commit
3f432401bc
  1. 4
      HEIF_IMPLEMENTATION_PLAN.md
  2. 8
      src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs
  3. 9
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs
  4. 6
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs
  5. 28
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs
  6. 157
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs
  7. 92
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs
  8. 5
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs
  9. 14
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs
  10. 7
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs
  11. 47
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs
  12. 272
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

4
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

8
src/ImageSharp/Formats/Heif/Av1/Entropy/Av1SymbolEncoder.cs

@ -785,7 +785,7 @@ internal class Av1SymbolEncoder : IDisposable
ushort endOfBlock, ushort endOfBlock,
bool useReducedTransformSet, bool useReducedTransformSet,
Av1FilterIntraMode filterIntraMode, Av1FilterIntraMode filterIntraMode,
bool usesInterTransformSet = false) bool usesInterTransformSet)
{ {
Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize); Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize);
@ -938,7 +938,7 @@ internal class Av1SymbolEncoder : IDisposable
ushort endOfBlock, ushort endOfBlock,
bool useReducedTransformSet, bool useReducedTransformSet,
Av1FilterIntraMode filterIntraMode, Av1FilterIntraMode filterIntraMode,
bool usesInterTransformSet = false) bool usesInterTransformSet)
{ {
Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize); Av1TransformSize transformSizeContext = Av1SymbolContextHelper.GetTransformSizeContext(transformSize);
@ -1384,7 +1384,7 @@ internal class Av1SymbolEncoder : IDisposable
int baseQIndex, int baseQIndex,
Av1FilterIntraMode filterIntraMode, Av1FilterIntraMode filterIntraMode,
Av1PredictionMode intraDirection, Av1PredictionMode intraDirection,
bool usesInterTransformSet = false) bool usesInterTransformSet)
=> this.ProcessTransformType<CoefficientCostOperation>( => this.ProcessTransformType<CoefficientCostOperation>(
transformType, transformType,
transformSize, transformSize,
@ -1411,7 +1411,7 @@ internal class Av1SymbolEncoder : IDisposable
int baseQIndex, int baseQIndex,
Av1FilterIntraMode filterIntraMode, Av1FilterIntraMode filterIntraMode,
Av1PredictionMode intraDirection, Av1PredictionMode intraDirection,
bool usesInterTransformSet = false) bool usesInterTransformSet)
{ {
_ = this.ProcessTransformType<CoefficientWriteOperation>( _ = this.ProcessTransformType<CoefficientWriteOperation>(
transformType, transformType,

9
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs

@ -527,7 +527,8 @@ internal static partial class Av1IntraSuperblockEncoder
context, context,
state.EndOfBlock, state.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
return distortion; return distortion;
} }
@ -631,7 +632,8 @@ internal static partial class Av1IntraSuperblockEncoder
blueContext, blueContext,
candidateBlueState.EndOfBlock, candidateBlueState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
rate += writer.GetCoefficientCost( rate += writer.GetCoefficientCost(
transformSize, transformSize,
@ -642,7 +644,8 @@ internal static partial class Av1IntraSuperblockEncoder
redContext, redContext,
candidateRedState.EndOfBlock, candidateRedState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion);
} }

6
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaPaletteModeDecision.cs

@ -311,7 +311,8 @@ internal static partial class Av1IntraSuperblockEncoder
blueContext, blueContext,
candidateBlueState.EndOfBlock, candidateBlueState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
rate += writer.GetCoefficientCost( rate += writer.GetCoefficientCost(
transformSize, transformSize,
@ -322,7 +323,8 @@ internal static partial class Av1IntraSuperblockEncoder
redContext, redContext,
candidateRedState.EndOfBlock, candidateRedState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion);
if (candidateCost < bestCost) if (candidateCost < bestCost)

28
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs

@ -188,6 +188,7 @@ internal static partial class Av1IntraSuperblockEncoder
0, 0,
0, 0,
LumaTransformSize, LumaTransformSize,
Av1TransformType.AllTransformTypes,
lumaContext, lumaContext,
workspace.LumaPrediction, workspace.LumaPrediction,
workspace.Residual, workspace.Residual,
@ -216,6 +217,19 @@ internal static partial class Av1IntraSuperblockEncoder
Av1EncoderTransformBlockState emptyRedState = default; Av1EncoderTransformBlockState emptyRedState = default;
if (!this.source.IsMonochrome) if (!this.source.IsMonochrome)
{ {
Av1TransformType chromaTransformType = lumaCandidateState.TransformType;
Av1TransformSetType chromaTransformSet = Av1SymbolContextHelper.GetExtendedTransformSetType(
chromaTransformSize,
isInter: true,
this.picture.Parent.FrameHeader.UseReducedTransformSet);
// Inter prediction does not signal an independent chroma transform type. Chroma reuses the
// selected luma type when that type belongs to its transform set and otherwise falls back to DCT.
if (!chromaTransformType.IsExtendedSetUsed(chromaTransformSet))
{
chromaTransformType = Av1TransformType.DctDct;
}
this.EvaluateIntraBlockCopyPlane( this.EvaluateIntraBlockCopyPlane(
writer, writer,
candidate, candidate,
@ -225,6 +239,7 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingX, subsamplingX,
subsamplingY, subsamplingY,
chromaTransformSize, chromaTransformSize,
chromaTransformType,
blueContext, blueContext,
workspace.BluePrediction, workspace.BluePrediction,
workspace.Residual, workspace.Residual,
@ -248,6 +263,7 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingX, subsamplingX,
subsamplingY, subsamplingY,
chromaTransformSize, chromaTransformSize,
chromaTransformType,
redContext, redContext,
workspace.RedPrediction, workspace.RedPrediction,
workspace.Residual, workspace.Residual,
@ -424,6 +440,7 @@ internal static partial class Av1IntraSuperblockEncoder
int subsamplingX, int subsamplingX,
int subsamplingY, int subsamplingY,
Av1TransformSize transformSize, Av1TransformSize transformSize,
Av1TransformType transformTypeSelection,
Av1TransformBlockContext blockContext, Av1TransformBlockContext blockContext,
Span<TSample> prediction, Span<TSample> prediction,
Span<short> residual, Span<short> residual,
@ -461,6 +478,13 @@ internal static partial class Av1IntraSuperblockEncoder
isInter: true, isInter: true,
this.picture.Parent.FrameHeader.UseReducedTransformSet); this.picture.Parent.FrameHeader.UseReducedTransformSet);
Av1TransformType firstTransformType = transformTypeSelection == Av1TransformType.AllTransformTypes
? Av1TransformType.DctDct
: transformTypeSelection;
Av1TransformType transformTypeLimit = transformTypeSelection == Av1TransformType.AllTransformTypes
? Av1TransformType.AllTransformTypes
: (Av1TransformType)((int)transformTypeSelection + 1);
long bestCost = long.MaxValue; long bestCost = long.MaxValue;
selectedState = default; selectedState = default;
selectedRate = 0; selectedRate = 0;
@ -468,8 +492,8 @@ internal static partial class Av1IntraSuperblockEncoder
hasEmptyTransform = false; hasEmptyTransform = false;
emptyState = default; emptyState = default;
emptyDistortion = 0; emptyDistortion = 0;
for (Av1TransformType transformType = Av1TransformType.DctDct; for (Av1TransformType transformType = firstTransformType;
transformType < Av1TransformType.AllTransformTypes; transformType < transformTypeLimit;
transformType++) transformType++)
{ {
if (!transformType.IsExtendedSetUsed(transformSetType)) if (!transformType.IsExtendedSetUsed(transformSetType))

157
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs

@ -389,7 +389,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
0, 0,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
} }
Span<byte> contexts = this.blockWorkspace Span<byte> contexts = this.blockWorkspace
@ -423,7 +424,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
0, 0,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
topContexts.Slice(blockColumn, transformWidth).Clear(); topContexts.Slice(blockColumn, transformWidth).Clear();
leftContexts.Slice(blockRow, transformHeight).Clear(); leftContexts.Slice(blockRow, transformHeight).Clear();
@ -788,6 +790,7 @@ internal static partial class Av1IntraSuperblockEncoder
} }
} }
selectedTransformSize = TransformSize;
if (this.effort >= 5 && if (this.effort >= 5 &&
this.picture.Parent.FrameHeader.AllowScreenContentTools && this.picture.Parent.FrameHeader.AllowScreenContentTools &&
this.SelectLumaPalette( this.SelectLumaPalette(
@ -800,19 +803,20 @@ internal static partial class Av1IntraSuperblockEncoder
transformSetType, transformSetType,
blockContext, blockContext,
largestTransformRate, largestTransformRate,
transformSizeContext,
candidateReconstruction, candidateReconstruction,
candidateCoefficients, candidateCoefficients,
retainedCoefficients, retainedCoefficients,
ref retainedStates[0], retainedStates,
ref bestTransformCost, ref bestTransformCost,
ref paletteInfo)) ref paletteInfo,
ref selectedTransformSize))
{ {
bestMode = Av1PredictionMode.DC; bestMode = Av1PredictionMode.DC;
selectedAngleDelta = 0; selectedAngleDelta = 0;
selectedFilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes; selectedFilterIntraMode = Av1FilterIntraMode.AllFilterIntraModes;
} }
selectedTransformSize = TransformSize;
if (this.effort >= 6 && if (this.effort >= 6 &&
this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select && this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select &&
paletteInfo.PaletteSizes[0] == 0) paletteInfo.PaletteSizes[0] == 0)
@ -827,6 +831,9 @@ internal static partial class Av1IntraSuperblockEncoder
bestMode, bestMode,
selectedAngleDelta, selectedAngleDelta,
selectedFilterIntraMode, selectedFilterIntraMode,
0,
ReadOnlySpan<ushort>.Empty,
0,
paletteDisabledCost, paletteDisabledCost,
transformSizeContext, transformSizeContext,
bestTransformCost, bestTransformCost,
@ -864,6 +871,9 @@ internal static partial class Av1IntraSuperblockEncoder
Av1PredictionMode mode, Av1PredictionMode mode,
int angleDelta, int angleDelta,
Av1FilterIntraMode filterIntraMode, Av1FilterIntraMode filterIntraMode,
int paletteSize,
scoped ReadOnlySpan<ushort> paletteColors,
int paletteHeaderRate,
int paletteDisabledCost, int paletteDisabledCost,
int transformSizeContext, int transformSizeContext,
long costLimit, long costLimit,
@ -902,19 +912,34 @@ internal static partial class Av1IntraSuperblockEncoder
TransformSize, TransformSize,
useReducedTransformSet); useReducedTransformSet);
int rate = Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta); int rate = writer.GetTransformSizeCost(BlockSize, TransformSize, transformSizeContext);
rate += writer.GetTransformSizeCost(BlockSize, TransformSize, transformSizeContext); if (paletteSize > 0)
if (mode == Av1PredictionMode.DC)
{ {
rate += paletteDisabledCost; rate += paletteHeaderRate;
if (this.picture.Sequence.SequenceHeader.EnableFilterIntra) }
else
{
rate += Av1TileWriter.GetLumaModeCost(writer, macroBlock, BlockSize, mode, angleDelta);
if (mode == Av1PredictionMode.DC)
{ {
rate += writer.GetFilterIntraModeCost( rate += paletteDisabledCost;
filterIntraMode, if (this.picture.Sequence.SequenceHeader.EnableFilterIntra)
BlockSize); {
rate += writer.GetFilterIntraModeCost(
filterIntraMode,
BlockSize);
}
} }
} }
Buffer2DRegion<byte> colorIndexMap = default;
if (paletteSize > 0)
{
colorIndexMap = this.superblock.Workspace
.GetPaletteMaps()
.GetMap(Av1PlaneType.Y, BlockWidth, BlockWidth);
}
long distortion = 0; long distortion = 0;
for (int transformRow = 0; transformRow < 2; transformRow++) for (int transformRow = 0; transformRow < 2; transformRow++)
{ {
@ -928,52 +953,73 @@ internal static partial class Av1IntraSuperblockEncoder
transformColumn * TransformWidth, transformColumn * TransformWidth,
transformRow * TransformWidth); transformRow * TransformWidth);
Span<TSample> aboveStorage = workspace.GetReferenceSamples(0); if (paletteSize > 0)
Span<TSample> leftStorage = workspace.GetReferenceSamples(1);
this.PrepareSplitLumaReferenceSamples(
reconstructionPlane,
blockOrigin,
macroBlock,
transformRow,
transformColumn,
candidateReconstruction,
aboveStorage,
leftStorage,
out bool hasLeft,
out bool hasAbove);
if (filterIntraMode == Av1FilterIntraMode.AllFilterIntraModes)
{ {
TOperator.PrepareIntra( // Palette prediction is block-local. A view over the retained map avoids copying indices or
this.blockWorkspace, // preparing reconstructed neighbor edges that this prediction mode cannot consume.
TOperator.PreparePalette(
sourcePlane, sourcePlane,
transformOrigin, transformOrigin,
paletteColors,
colorIndexMap.GetSubRegion(
new Rectangle(
transformColumn * TransformWidth,
transformRow * TransformWidth,
TransformWidth,
TransformWidth)),
prediction, prediction,
aboveStorage.Slice(1, TransformWidth * 2),
leftStorage.Slice(1, TransformWidth * 2),
hasLeft,
hasAbove,
mode,
angleDelta,
residual, residual,
TransformSize, TransformSize);
this.bitDepth);
} }
else else
{ {
// Filter-intra prediction is recursive within each transform unit, so rebuild it from Span<TSample> aboveStorage = workspace.GetReferenceSamples(0);
// the reconstructed edges established by the preceding 4x4 candidate. Span<TSample> leftStorage = workspace.GetReferenceSamples(1);
TOperator.PrepareFilterIntra( this.PrepareSplitLumaReferenceSamples(
this.blockWorkspace, reconstructionPlane,
sourcePlane, blockOrigin,
transformOrigin, macroBlock,
prediction, transformRow,
aboveStorage.Slice(1, TransformWidth * 2), transformColumn,
leftStorage.Slice(1, TransformWidth * 2), candidateReconstruction,
residual, aboveStorage,
filterIntraMode, leftStorage,
TransformSize, out bool hasLeft,
this.bitDepth); out bool hasAbove);
if (filterIntraMode == Av1FilterIntraMode.AllFilterIntraModes)
{
TOperator.PrepareIntra(
this.blockWorkspace,
sourcePlane,
transformOrigin,
prediction,
aboveStorage.Slice(1, TransformWidth * 2),
leftStorage.Slice(1, TransformWidth * 2),
hasLeft,
hasAbove,
mode,
angleDelta,
residual,
TransformSize,
this.bitDepth);
}
else
{
// Filter-intra prediction is recursive within each transform unit, so rebuild it from
// the reconstructed edges established by the preceding 4x4 candidate.
TOperator.PrepareFilterIntra(
this.blockWorkspace,
sourcePlane,
transformOrigin,
prediction,
aboveStorage.Slice(1, TransformWidth * 2),
leftStorage.Slice(1, TransformWidth * 2),
residual,
filterIntraMode,
TransformSize,
this.bitDepth);
}
} }
Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts( Av1TransformBlockContext blockContext = Av1TileWriter.GetTransformBlockContexts(
@ -1029,7 +1075,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
candidateState.EndOfBlock, candidateState.EndOfBlock,
useReducedTransformSet, useReducedTransformSet,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
long candidateCost = Av1RateDistortion.GetCost( long candidateCost = Av1RateDistortion.GetCost(
this.rateMultiplier, this.rateMultiplier,
@ -1302,7 +1349,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
candidateState.EndOfBlock, candidateState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion);
} }
@ -1362,7 +1410,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
candidateState.EndOfBlock, candidateState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion); return Av1RateDistortion.GetCost(this.rateMultiplier, rate, distortion);
} }

92
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.PaletteModeDecision.cs

@ -29,12 +29,14 @@ internal static partial class Av1IntraSuperblockEncoder
Av1TransformSetType transformSetType, Av1TransformSetType transformSetType,
Av1TransformBlockContext blockContext, Av1TransformBlockContext blockContext,
int transformSizeRate, int transformSizeRate,
int transformSizeContext,
Span<TSample> candidateReconstruction, Span<TSample> candidateReconstruction,
Span<int> candidateCoefficients, Span<int> candidateCoefficients,
Span<int> retainedCoefficients, Span<int> retainedCoefficients,
ref Av1EncoderTransformBlockState retainedState, Span<Av1EncoderTransformBlockState> retainedStates,
ref long bestCost, ref long bestCost,
ref Av1EncoderPaletteInfo paletteInfo) ref Av1EncoderPaletteInfo paletteInfo,
ref Av1TransformSize selectedTransformSize)
{ {
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const int BlockLength = 8; const int BlockLength = 8;
@ -137,9 +139,11 @@ internal static partial class Av1IntraSuperblockEncoder
writer, writer,
macroBlock, macroBlock,
blockOrigin, blockOrigin,
tileIndex,
transformSetType, transformSetType,
blockContext, blockContext,
transformSizeRate, transformSizeRate,
transformSizeContext,
samples, samples,
rows, rows,
columns, columns,
@ -151,11 +155,12 @@ internal static partial class Av1IntraSuperblockEncoder
candidateReconstruction, candidateReconstruction,
candidateCoefficients, candidateCoefficients,
retainedCoefficients, retainedCoefficients,
retainedStates,
retainedColorIndexMap, retainedColorIndexMap,
reconstructionPlane, reconstructionPlane,
ref retainedState,
ref bestCost, ref bestCost,
ref paletteInfo, ref paletteInfo,
ref selectedTransformSize,
ref paletteSelected); ref paletteSelected);
} }
@ -167,9 +172,11 @@ internal static partial class Av1IntraSuperblockEncoder
writer, writer,
macroBlock, macroBlock,
blockOrigin, blockOrigin,
tileIndex,
transformSetType, transformSetType,
blockContext, blockContext,
transformSizeRate, transformSizeRate,
transformSizeContext,
samples, samples,
rows, rows,
columns, columns,
@ -181,11 +188,12 @@ internal static partial class Av1IntraSuperblockEncoder
candidateReconstruction, candidateReconstruction,
candidateCoefficients, candidateCoefficients,
retainedCoefficients, retainedCoefficients,
retainedStates,
retainedColorIndexMap, retainedColorIndexMap,
reconstructionPlane, reconstructionPlane,
ref retainedState,
ref bestCost, ref bestCost,
ref paletteInfo, ref paletteInfo,
ref selectedTransformSize,
ref paletteSelected); ref paletteSelected);
} }
else else
@ -206,9 +214,11 @@ internal static partial class Av1IntraSuperblockEncoder
writer, writer,
macroBlock, macroBlock,
blockOrigin, blockOrigin,
tileIndex,
transformSetType, transformSetType,
blockContext, blockContext,
transformSizeRate, transformSizeRate,
transformSizeContext,
samples, samples,
rows, rows,
columns, columns,
@ -220,11 +230,12 @@ internal static partial class Av1IntraSuperblockEncoder
candidateReconstruction, candidateReconstruction,
candidateCoefficients, candidateCoefficients,
retainedCoefficients, retainedCoefficients,
retainedStates,
retainedColorIndexMap, retainedColorIndexMap,
reconstructionPlane, reconstructionPlane,
ref retainedState,
ref bestCost, ref bestCost,
ref paletteInfo, ref paletteInfo,
ref selectedTransformSize,
ref paletteSelected); ref paletteSelected);
} }
} }
@ -245,9 +256,11 @@ internal static partial class Av1IntraSuperblockEncoder
Av1SymbolEncoder writer, Av1SymbolEncoder writer,
Av1MacroBlockD macroBlock, Av1MacroBlockD macroBlock,
Point blockOrigin, Point blockOrigin,
ushort tileIndex,
Av1TransformSetType transformSetType, Av1TransformSetType transformSetType,
Av1TransformBlockContext blockContext, Av1TransformBlockContext blockContext,
int transformSizeRate, int transformSizeRate,
int transformSizeContext,
ReadOnlySpan<short> samples, ReadOnlySpan<short> samples,
int rows, int rows,
int columns, int columns,
@ -259,18 +272,21 @@ internal static partial class Av1IntraSuperblockEncoder
Span<TSample> candidateReconstruction, Span<TSample> candidateReconstruction,
Span<int> candidateCoefficients, Span<int> candidateCoefficients,
Span<int> retainedCoefficients, Span<int> retainedCoefficients,
Span<Av1EncoderTransformBlockState> retainedStates,
Span<byte> retainedColorIndexMap, Span<byte> retainedColorIndexMap,
Buffer2DRegion<TSample> reconstructionPlane, Buffer2DRegion<TSample> reconstructionPlane,
ref Av1EncoderTransformBlockState retainedState,
ref long bestCost, ref long bestCost,
ref Av1EncoderPaletteInfo paletteInfo, ref Av1EncoderPaletteInfo paletteInfo,
ref Av1TransformSize selectedTransformSize,
ref bool paletteSelected) ref bool paletteSelected)
{ {
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize TransformSize = Av1TransformSize.Size8x8; const Av1TransformSize TransformSize = Av1TransformSize.Size8x8;
const int BlockLength = 8; const int BlockLength = 8;
Av1EncoderPaletteWorkspace<TSample> workspace = Av1EncoderModeDecisionWorkspace<TSample> modeDecisionWorkspace =
this.blockWorkspace.GetModeDecisionWorkspace<TSample>().Palette; this.blockWorkspace.GetModeDecisionWorkspace<TSample>();
Av1EncoderPaletteWorkspace<TSample> workspace = modeDecisionWorkspace.Palette;
int bitDepth = this.bitDepth.GetBitCount(); int bitDepth = this.bitDepth.GetBitCount();
int cacheThreshold = 4 << (bitDepth - 8); int cacheThreshold = 4 << (bitDepth - 8);
@ -350,7 +366,6 @@ internal static partial class Av1IntraSuperblockEncoder
Av1PredictionMode.DC, Av1PredictionMode.DC,
0); 0);
rate += transformSizeRate;
rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext); rate += writer.GetPaletteYModeCost(true, blockSizeContext, neighborContext);
rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y); rate += writer.GetPaletteSizeCost(paletteSize, blockSizeContext, Av1PlaneType.Y);
rate += Av1SymbolEncoder.GetPaletteYColorCost(colorCache, paletteColors, bitDepth); rate += Av1SymbolEncoder.GetPaletteYColorCost(colorCache, paletteColors, bitDepth);
@ -389,7 +404,8 @@ internal static partial class Av1IntraSuperblockEncoder
this.bitDepth, this.bitDepth,
ref candidateState); ref candidateState);
int candidateRate = rate + writer.GetCoefficientCost( int candidateRate = rate + transformSizeRate;
candidateRate += writer.GetCoefficientCost(
TransformSize, TransformSize,
transformType, transformType,
Av1PredictionMode.DC, Av1PredictionMode.DC,
@ -398,7 +414,8 @@ internal static partial class Av1IntraSuperblockEncoder
blockContext, blockContext,
candidateState.EndOfBlock, candidateState.EndOfBlock,
this.picture.Parent.FrameHeader.UseReducedTransformSet, this.picture.Parent.FrameHeader.UseReducedTransformSet,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, distortion); long candidateCost = Av1RateDistortion.GetCost(this.rateMultiplier, candidateRate, distortion);
if (candidateCost < bestCost) if (candidateCost < bestCost)
@ -411,7 +428,7 @@ internal static partial class Av1IntraSuperblockEncoder
retainedCoefficients, retainedCoefficients,
TransformSize, TransformSize,
candidateState, candidateState,
ref retainedState); ref retainedStates[0]);
for (int row = 0; row < BlockLength; row++) for (int row = 0; row < BlockLength; row++)
{ {
@ -421,10 +438,61 @@ internal static partial class Av1IntraSuperblockEncoder
paletteInfo.PaletteSizes[0] = (byte)paletteSize; paletteInfo.PaletteSizes[0] = (byte)paletteSize;
paletteInfo.SetColors(Av1Plane.Y, paletteColors); paletteInfo.SetColors(Av1Plane.Y, paletteColors);
selectedTransformSize = TransformSize;
bestCost = candidateCost; bestCost = candidateCost;
paletteSelected = true; paletteSelected = true;
} }
} }
if (this.effort >= 6 &&
this.picture.Parent.FrameHeader.TransformMode == Av1TransformMode.Select)
{
// Transform size is part of each palette candidate's RD result. Searching it here preserves
// candidates whose 4x4 residual partition wins even when their 8x8 result does not.
long splitCost = this.GetSplitLumaCandidateCost(
writer,
macroBlock,
this.source.GetPlane(Av1Plane.Y),
reconstructionPlane,
blockOrigin,
tileIndex,
Av1PredictionMode.DC,
0,
Av1FilterIntraMode.AllFilterIntraModes,
paletteSize,
paletteColors,
rate,
0,
transformSizeContext,
bestCost,
candidateReconstruction,
candidateCoefficients,
modeDecisionWorkspace.CandidateTransformBlocks);
if (splitCost < bestCost)
{
CopySplitCandidate(
candidateReconstruction,
candidateCoefficients,
modeDecisionWorkspace.CandidateTransformBlocks,
reconstructionPlane,
blockOrigin,
retainedCoefficients,
retainedStates);
for (int row = 0; row < BlockLength; row++)
{
colorIndexMap.DangerousGetRowSpan(row)[..BlockLength]
.CopyTo(retainedColorIndexMap[(row * BlockLength)..]);
}
paletteInfo.PaletteSizes[0] = (byte)paletteSize;
paletteInfo.SetColors(Av1Plane.Y, paletteColors);
selectedTransformSize = Av1TransformSize.Size4x4;
bestCost = splitCost;
paletteSelected = true;
}
}
} }
} }
} }

5
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1EncoderPictureBuffer.cs

@ -207,6 +207,11 @@ internal sealed class Av1EncoderPictureBuffer : IDisposable
GranularityNormalLog2 = Av1Constants.ModeInfoSizeLog2 GranularityNormalLog2 = Av1Constants.ModeInfoSizeLog2
}; };
// Variable-transform contexts consult both edges without separate availability flags. The largest
// transform makes an unavailable edge compare as unsplit until a coded neighbor publishes its size.
this.transformContexts[tileIndex].Left.Fill((byte)Av1Constants.MaxTransformSize);
this.transformContexts[tileIndex].Top.Fill((byte)Av1Constants.MaxTransformSize);
if (frameHeader.AllowScreenContentTools) if (frameHeader.AllowScreenContentTools)
{ {
this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>( this.paletteContexts[tileIndex] = new Av1NeighborArrayUnit<Av1EncoderPaletteInfo>(

14
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileWriter.cs

@ -983,7 +983,8 @@ internal partial class Av1TileWriter
bool writesVariableTransformSize = !isLossless && bool writesVariableTransformSize = !isLossless &&
frameHeader.TransformMode == Av1TransformMode.Select && frameHeader.TransformMode == Av1TransformMode.Select &&
isInter && isInter &&
!macroBlockModeInfo.Block.Skip; !macroBlockModeInfo.Block.Skip &&
blockSize > Av1BlockSize.Block4x4;
Av1TransformSize transformSize = isLossless Av1TransformSize transformSize = isLossless
? Av1TransformSize.Size4x4 ? Av1TransformSize.Size4x4
@ -1650,6 +1651,7 @@ internal partial class Av1TileWriter
Av1NeighborArrayUnit<byte> luma_dc_sign_level_coeff_na) Av1NeighborArrayUnit<byte> luma_dc_sign_level_coeff_na)
{ {
ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; ObuFrameHeader frameHeader = pcs.Parent.FrameHeader;
bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy;
Span<int> lumaCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.Y); Span<int> lumaCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.Y);
Span<Av1EncoderTransformBlockState> lumaTransformBlocks = Span<Av1EncoderTransformBlockState> lumaTransformBlocks =
coefficientBuffer.GetTransformBlockSpan(superblockIndex, Av1Plane.Y); coefficientBuffer.GetTransformBlockSpan(superblockIndex, Av1Plane.Y);
@ -1720,7 +1722,8 @@ internal partial class Av1TileWriter
blockContext, blockContext,
endOfBlock, endOfBlock,
frameHeader.UseReducedTransformSet, frameHeader.UseReducedTransformSet,
blk_ptr.FilterIntraMode); blk_ptr.FilterIntraMode,
usesInterTransformSet);
int transformWidth = transformSize.GetWidth(); int transformWidth = transformSize.GetWidth();
int transformHeight = transformSize.GetHeight(); int transformHeight = transformSize.GetHeight();
@ -1771,6 +1774,7 @@ internal partial class Av1TileWriter
} }
ObuFrameHeader frameHeader = pcs.Parent.FrameHeader; ObuFrameHeader frameHeader = pcs.Parent.FrameHeader;
bool usesInterTransformSet = entropyCodingContext.MacroBlockModeInfo.Block.UseIntraBlockCopy;
Span<int> blueCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.U); Span<int> blueCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.U);
Span<int> redCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.V); Span<int> redCoefficients = coefficientBuffer.GetPlaneSpan(superblockIndex, Av1Plane.V);
Span<Av1EncoderTransformBlockState> blueTransformBlocks = Span<Av1EncoderTransformBlockState> blueTransformBlocks =
@ -1847,7 +1851,8 @@ internal partial class Av1TileWriter
blockContext, blockContext,
blueTransformBlock.EndOfBlock, blueTransformBlock.EndOfBlock,
frameHeader.UseReducedTransformSet, frameHeader.UseReducedTransformSet,
blk_ptr.FilterIntraMode); blk_ptr.FilterIntraMode,
usesInterTransformSet);
coefficients = redCoefficients[entropyCodingContext.CodedAreaSuperblockUv..]; coefficients = redCoefficients[entropyCodingContext.CodedAreaSuperblockUv..];
blockContext = GetTransformBlockContexts( blockContext = GetTransformBlockContexts(
@ -1866,7 +1871,8 @@ internal partial class Av1TileWriter
blockContext, blockContext,
redTransformBlock.EndOfBlock, redTransformBlock.EndOfBlock,
frameHeader.UseReducedTransformSet, frameHeader.UseReducedTransformSet,
blk_ptr.FilterIntraMode); blk_ptr.FilterIntraMode,
usesInterTransformSet);
cb_dc_sign_level_coeff_na.UnitModeWrite( cb_dc_sign_level_coeff_na.UnitModeWrite(
(byte)culLevelCb, (byte)culLevelCb,

7
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1CoefficientsEntropyTests.cs

@ -1128,7 +1128,7 @@ public class Av1CoefficientsEntropyTests
Span<int> actuals = new int[16]; Span<int> actuals = new int[16];
// Act // Act
encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode); encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode, usesInterTransformSet: false);
using IMemoryOwner<byte> encoded = encoder.Exit(); using IMemoryOwner<byte> encoded = encoder.Exit();
@ -1203,7 +1203,7 @@ public class Av1CoefficientsEntropyTests
Span<int> actuals = new int[16 + 1]; Span<int> actuals = new int[16 + 1];
// Act // Act
encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode); encoder.WriteCoefficients(transformSize, transformType, intraDirection, coefficientsBuffer, componentType, transformBlockContext, endOfBlock, true, filterIntraMode, usesInterTransformSet: false);
using IMemoryOwner<byte> encoded = encoder.Exit(); using IMemoryOwner<byte> encoded = encoder.Exit();
@ -1331,7 +1331,8 @@ public class Av1CoefficientsEntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
useReducedTransformSet, useReducedTransformSet,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
using IMemoryOwner<byte> encoded = encoder.Exit(); using IMemoryOwner<byte> encoded = encoder.Exit();

47
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EntropyTests.cs

@ -330,7 +330,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
0, 0,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
Assert.Equal(Av1ProbabilityCost.GetSymbolCost(transformSkip, 1), emptyCost); Assert.Equal(Av1ProbabilityCost.GetSymbolCost(transformSkip, 1), emptyCost);
@ -346,7 +347,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
16, 16,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
coefficients.Clear(); coefficients.Clear();
coefficients[0] = 1; coefficients[0] = 1;
@ -371,7 +373,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
1, 1,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
Assert.Equal(expectedDcCost, dcCost); Assert.Equal(expectedDcCost, dcCost);
} }
@ -417,7 +420,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
1, 1,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
Assert.Equal(expected, actual); Assert.Equal(expected, actual);
} }
@ -540,7 +544,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
filterIntraMode); filterIntraMode,
usesInterTransformSet: false);
Assert.Equal(expected, actual); Assert.Equal(expected, actual);
} }
@ -570,7 +575,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
Assert.Equal( Assert.Equal(
initialCost, initialCost,
@ -583,7 +589,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes)); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false));
int actualContext = actualEncoder.WriteCoefficients( int actualContext = actualEncoder.WriteCoefficients(
transformSize, transformSize,
@ -594,7 +601,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
int adaptedCost = actualEncoder.GetCoefficientCost( int adaptedCost = actualEncoder.GetCoefficientCost(
transformSize, transformSize,
@ -605,7 +613,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
int expectedContext = expectedEncoder.WriteCoefficients( int expectedContext = expectedEncoder.WriteCoefficients(
transformSize, transformSize,
@ -616,7 +625,8 @@ public class Av1EntropyTests
transformBlockContext, transformBlockContext,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
using IMemoryOwner<byte> actual = actualEncoder.Exit(); using IMemoryOwner<byte> actual = actualEncoder.Exit();
using IMemoryOwner<byte> expected = expectedEncoder.Exit(); using IMemoryOwner<byte> expected = expectedEncoder.Exit();
@ -648,7 +658,8 @@ public class Av1EntropyTests
default, default,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
long before = GC.GetAllocatedBytesForCurrentThread(); long before = GC.GetAllocatedBytesForCurrentThread();
for (int i = 0; i < 1000; i++) for (int i = 0; i < 1000; i++)
@ -662,7 +673,8 @@ public class Av1EntropyTests
default, default,
endOfBlock, endOfBlock,
false, false,
Av1FilterIntraMode.AllFilterIntraModes); Av1FilterIntraMode.AllFilterIntraModes,
usesInterTransformSet: false);
} }
long after = GC.GetAllocatedBytesForCurrentThread(); long after = GC.GetAllocatedBytesForCurrentThread();
@ -825,7 +837,8 @@ public class Av1EntropyTests
default, default,
0, 0,
false, false,
Av1FilterIntraMode.DC); Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(0, emptyContext); Assert.Equal(0, emptyContext);
Assert.Single(allocator.AllocationLog); Assert.Single(allocator.AllocationLog);
@ -840,7 +853,8 @@ public class Av1EntropyTests
default, default,
1, 1,
false, false,
Av1FilterIntraMode.DC); Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(3, allocator.AllocationLog.Count); Assert.Equal(3, allocator.AllocationLog.Count);
@ -853,7 +867,8 @@ public class Av1EntropyTests
default, default,
1, 1,
false, false,
Av1FilterIntraMode.DC); Av1FilterIntraMode.DC,
usesInterTransformSet: false);
Assert.Equal(3, allocator.AllocationLog.Count); Assert.Equal(3, allocator.AllocationLog.Count);
TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1]; TestMemoryAllocator.AllocationRequest levelScratch = allocator.AllocationLog[1];
@ -1634,7 +1649,7 @@ public class Av1EntropyTests
// Act // Act
foreach (Av1TransformType value in values) foreach (Av1TransformType value in values)
{ {
encoder.WriteTransformType(value, transformSizeContext, true, BaseQIndex, filterIntraMode, intraDirection); encoder.WriteTransformType(value, transformSizeContext, true, BaseQIndex, filterIntraMode, intraDirection, usesInterTransformSet: false);
} }
using IMemoryOwner<byte> encoded = encoder.Exit(); using IMemoryOwner<byte> encoded = encoder.Exit();

272
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs

@ -728,6 +728,7 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
8, 8,
8, 8,
false,
(byte)32, (byte)32,
(byte)224, (byte)224,
32, 32,
@ -749,6 +750,7 @@ public class Av1IntraSuperblockEncoderTests
12, 12,
8, 8,
8, 8,
false,
(ushort)512, (ushort)512,
(ushort)3584, (ushort)3584,
512, 512,
@ -770,6 +772,7 @@ public class Av1IntraSuperblockEncoderTests
8, 8,
5, 5,
3, 3,
false,
(byte)48, (byte)48,
(byte)208, (byte)208,
48, 48,
@ -787,6 +790,54 @@ public class Av1IntraSuperblockEncoderTests
effort: 5)); effort: 5));
} }
[Fact]
public void ProductionTileSelectsLumaPaletteWithFourByFourTransforms()
{
AssertProductionTileSelectsExactLumaPalette(
Av1BitDepth.EightBit,
8,
8,
8,
true,
(byte)64,
(byte)192,
64,
192,
static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter(
Configuration.Default,
source,
reconstruction,
picture,
coefficients,
superblockWorkspace,
blockWorkspace,
initialSize: 256,
effort: 6));
AssertProductionTileSelectsExactLumaPalette(
Av1BitDepth.TwelveBit,
12,
8,
8,
true,
(ushort)1024,
(ushort)3072,
1024,
3072,
static (source, reconstruction, picture, coefficients, superblockWorkspace, blockWorkspace) =>
new Av1IntraTileWriter(
Configuration.Default,
source,
reconstruction,
picture,
coefficients,
superblockWorkspace,
blockWorkspace,
initialSize: 256,
effort: 6));
}
[Fact] [Fact]
public void ProductionTileSelectsExactPairedChromaPalette() public void ProductionTileSelectsExactPairedChromaPalette()
{ {
@ -1926,49 +1977,7 @@ public class Av1IntraSuperblockEncoderTests
long predictionOnlyError = 64L * 40 * 40 * sampleScale * sampleScale; long predictionOnlyError = 64L * 40 * 40 * sampleScale * sampleScale;
Assert.InRange(reconstructionError, 1, predictionOnlyError - 1); Assert.InRange(reconstructionError, 1, predictionOnlyError - 1);
// Tile fixtures initialize only entropy state. Complete the same still-picture headers as the frame byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, Width, Height);
// encoder before serializing so the independent decoder validates the real OBU syntax.
ObuSequenceHeader sequenceHeader = pictureTemplate.Sequence.SequenceHeader;
sequenceHeader.IsStillPicture = true;
sequenceHeader.IsReducedStillPictureHeader = true;
sequenceHeader.SequenceProfile = bitDepth == 12
? ObuSequenceProfile.Professional
: ObuSequenceProfile.Main;
sequenceHeader.OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }];
sequenceHeader.FrameWidthBits = 4;
sequenceHeader.FrameHeightBits = 4;
sequenceHeader.MaxFrameWidth = Width;
sequenceHeader.MaxFrameHeight = Height;
sequenceHeader.ForceScreenContentTools = 2;
sequenceHeader.ForceIntegerMotionVector = 2;
ObuFrameHeader frameHeader = pictureTemplate.Parent.FrameHeader;
frameHeader.FrameType = ObuFrameType.KeyFrame;
frameHeader.ShowFrame = true;
frameHeader.ErrorResilientMode = true;
frameHeader.RefreshFrameFlags = byte.MaxValue;
frameHeader.DisableFrameEndUpdateCdf = true;
frameHeader.FrameSize = new ObuFrameSize
{
FrameWidth = Width,
FrameHeight = Height,
SuperResolutionDenominator = Av1Constants.ScaleNumerator,
SuperResolutionUpscaledWidth = Width,
RenderWidth = Width,
RenderHeight = Height
};
frameHeader.TilesInfo.HasUniformTileSpacing = true;
using MemoryStream stream = new();
new ObuWriter().WriteAll(
Configuration.Default,
stream,
sequenceHeader,
frameHeader,
tileWriter);
byte[] payload = stream.ToArray();
using Av1Decoder decoder = new(Configuration.Default); using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload); using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.NotNull(decoder.FrameInfo); Assert.NotNull(decoder.FrameInfo);
@ -2517,6 +2526,62 @@ public class Av1IntraSuperblockEncoderTests
Assert.NotEqual(0, tileWriter.GetTileData(0).Length); Assert.NotEqual(0, tileWriter.GetTileData(0).Length);
} }
private static byte[] WriteCompleteTileObu(
Av1PictureControlSet pictureTemplate,
IAv1TileWriter tileWriter,
int width,
int height)
{
// Tile fixtures initialize only entropy state. Complete the same still-picture headers as the frame
// encoder before serializing so independent decoders validate the real OBU syntax.
ObuSequenceHeader sequenceHeader = pictureTemplate.Sequence.SequenceHeader;
ObuColorConfig colorConfig = sequenceHeader.ColorConfig;
Av1ColorFormat colorFormat = colorConfig.GetColorFormat();
sequenceHeader.IsStillPicture = true;
sequenceHeader.IsReducedStillPictureHeader = true;
sequenceHeader.SequenceProfile = colorConfig.BitDepth == Av1BitDepth.TwelveBit ||
colorFormat == Av1ColorFormat.Yuv422
? ObuSequenceProfile.Professional
: colorFormat == Av1ColorFormat.Yuv444
? ObuSequenceProfile.High
: ObuSequenceProfile.Main;
sequenceHeader.OperatingPoint = [new ObuOperatingPoint { SequenceLevelIndex = 31 }];
sequenceHeader.FrameWidthBits = width > 1 ? Av1Math.MostSignificantBit((uint)(width - 1)) + 1 : 1;
sequenceHeader.FrameHeightBits = height > 1 ? Av1Math.MostSignificantBit((uint)(height - 1)) + 1 : 1;
sequenceHeader.MaxFrameWidth = width;
sequenceHeader.MaxFrameHeight = height;
sequenceHeader.ForceScreenContentTools = 2;
sequenceHeader.ForceIntegerMotionVector = 2;
ObuFrameHeader frameHeader = pictureTemplate.Parent.FrameHeader;
frameHeader.FrameType = ObuFrameType.KeyFrame;
frameHeader.ShowFrame = true;
frameHeader.ErrorResilientMode = true;
frameHeader.RefreshFrameFlags = byte.MaxValue;
frameHeader.DisableFrameEndUpdateCdf = true;
frameHeader.FrameSize = new ObuFrameSize
{
FrameWidth = width,
FrameHeight = height,
SuperResolutionDenominator = Av1Constants.ScaleNumerator,
SuperResolutionUpscaledWidth = width,
RenderWidth = width,
RenderHeight = height
};
frameHeader.TilesInfo.HasUniformTileSpacing = true;
using MemoryStream stream = new();
new ObuWriter().WriteAll(
Configuration.Default,
stream,
sequenceHeader,
frameHeader,
tileWriter);
return stream.ToArray();
}
private static Av1PictureControlSet CreatePicture( private static Av1PictureControlSet CreatePicture(
Av1EncoderModeInfoBuffer modeInfo, Av1EncoderModeInfoBuffer modeInfo,
ObuColorConfig colorConfig, ObuColorConfig colorConfig,
@ -2581,12 +2646,13 @@ public class Av1IntraSuperblockEncoderTests
int bitDepthValue, int bitDepthValue,
int width, int width,
int height, int height,
bool useSplitTransform,
TSample lowerColor, TSample lowerColor,
TSample upperColor, TSample upperColor,
ushort expectedLowerColor, ushort expectedLowerColor,
ushort expectedUpperColor, ushort expectedUpperColor,
TileWriterFactory<TSample> createTileWriter) TileWriterFactory<TSample> createTileWriter)
where TSample : unmanaged where TSample : unmanaged, IBinaryInteger<TSample>
{ {
const int QIndex = 37; const int QIndex = 37;
ObuColorConfig colorConfig = new() ObuColorConfig colorConfig = new()
@ -2619,7 +2685,32 @@ public class Av1IntraSuperblockEncoderTests
for (int row = 0; row < sourcePlane.Height; row++) for (int row = 0; row < sourcePlane.Height; row++)
{ {
int visibleRow = Math.Min(row, height - 1); int visibleRow = Math.Min(row, height - 1);
sourcePlane.DangerousGetRowSpan(row).Fill(visibleRow < height / 2 ? lowerColor : upperColor); Span<TSample> sourceRow = sourcePlane.DangerousGetRowSpan(row);
if (!useSplitTransform)
{
sourceRow.Fill(visibleRow < height / 2 ? lowerColor : upperColor);
continue;
}
int transformRow = visibleRow >> 2;
int localRow = visibleRow & 3;
for (int column = 0; column < sourceRow.Length; column++)
{
int transformColumn = column >> 2;
int localColumn = column & 3;
int transformIndex = (transformRow * 2) + transformColumn;
int residual = transformIndex switch
{
0 => (localRow * 2) - 3,
1 => (localColumn * 2) - 3,
2 => (localRow + localColumn) - 3,
_ => localRow - localColumn
};
int baseColor = int.CreateChecked(visibleRow < height / 2 ? lowerColor : upperColor);
sourceRow[column] = TSample.CreateChecked(
baseColor + (residual * 4 * (1 << (bitDepthValue - 8))));
}
} }
ClearPlane(reconstruction.Luma); ClearPlane(reconstruction.Luma);
@ -2636,6 +2727,10 @@ public class Av1IntraSuperblockEncoderTests
QIndex); QIndex);
pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true; pictureTemplate.Parent.FrameHeader.AllowScreenContentTools = true;
pictureTemplate.Parent.FrameHeader.TransformMode = useSplitTransform
? Av1TransformMode.Select
: Av1TransformMode.Largest;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = width; pictureTemplate.Parent.FrameHeader.FrameSize.FrameWidth = width;
pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = height; pictureTemplate.Parent.FrameHeader.FrameSize.FrameHeight = height;
using Av1EncoderPictureBuffer picture = new( using Av1EncoderPictureBuffer picture = new(
@ -2664,27 +2759,96 @@ public class Av1IntraSuperblockEncoderTests
ref Av1MacroBlockModeInfo mode = ref picture.Picture.GetMacroBlockModeInfo(default); ref Av1MacroBlockModeInfo mode = ref picture.Picture.GetMacroBlockModeInfo(default);
Assert.Equal(Av1PredictionMode.DC, mode.Block.Mode); Assert.Equal(Av1PredictionMode.DC, mode.Block.Mode);
Assert.Equal(Av1FilterIntraMode.AllFilterIntraModes, superblockWorkspace.FinalBlocks[0].FilterIntraMode); Assert.Equal(Av1FilterIntraMode.AllFilterIntraModes, superblockWorkspace.FinalBlocks[0].FilterIntraMode);
Assert.Equal((ushort)0, coefficients.GetTransformBlockSpan(0, Av1Plane.Y)[0].EndOfBlock);
Assert.Equal(2, superblockWorkspace.PaletteInfo.PaletteSizes[0]);
Assert.Equal( Assert.Equal(
[expectedLowerColor, expectedUpperColor], useSplitTransform ? Av1TransformSize.Size4x4 : Av1TransformSize.Size8x8,
superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y).ToArray()); mode.Block.TransformSize);
Span<Av1EncoderTransformBlockState> transformStates = coefficients
.GetTransformBlockSpan(0, Av1Plane.Y)[..(useSplitTransform ? 4 : 1)];
if (useSplitTransform)
{
int coefficientBearingTransformCount = 0;
foreach (Av1EncoderTransformBlockState transformState in transformStates)
{
if (transformState.EndOfBlock > 0)
{
coefficientBearingTransformCount++;
}
}
Assert.InRange(coefficientBearingTransformCount, 1, transformStates.Length);
Assert.InRange(superblockWorkspace.PaletteInfo.PaletteSizes[0], 2, Av1Constants.PaletteMaxSize);
}
else
{
Assert.Equal((ushort)0, transformStates[0].EndOfBlock);
Assert.Equal(2, superblockWorkspace.PaletteInfo.PaletteSizes[0]);
Assert.Equal(
[expectedLowerColor, expectedUpperColor],
superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y).ToArray());
}
Buffer2DRegion<byte> colorIndexMap = superblockWorkspace Buffer2DRegion<byte> colorIndexMap = superblockWorkspace
.GetPaletteMaps() .GetPaletteMaps()
.GetMap(Av1PlaneType.Y, 8, 8); .GetMap(Av1PlaneType.Y, 8, 8);
Buffer2DRegion<TSample> reconstructionPlane = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> reconstructionPlane = reconstruction.Frame.CodedView.GetPlane(Av1Plane.Y);
ReadOnlySpan<ushort> selectedPaletteColors = superblockWorkspace.PaletteInfo.GetColors(Av1Plane.Y);
long predictionOnlyError = 0;
long reconstructionError = 0;
for (int row = 0; row < reconstructionPlane.Height; row++) for (int row = 0; row < reconstructionPlane.Height; row++)
{ {
int visibleRow = Math.Min(row, height - 1); if (useSplitTransform)
byte expectedIndex = (byte)(visibleRow < height / 2 ? 0 : 1);
foreach (byte index in colorIndexMap.DangerousGetRowSpan(row))
{ {
Assert.Equal(expectedIndex, index); ReadOnlySpan<TSample> sourceRow = sourcePlane.DangerousGetRowSpan(row);
ReadOnlySpan<TSample> reconstructionRow = reconstructionPlane.DangerousGetRowSpan(row);
ReadOnlySpan<byte> mapRow = colorIndexMap.DangerousGetRowSpan(row);
for (int column = 0; column < reconstructionRow.Length; column++)
{
long sourceSample = long.CreateChecked(sourceRow[column]);
long predictionDifference = sourceSample - selectedPaletteColors[mapRow[column]];
long reconstructionDifference = sourceSample - long.CreateChecked(reconstructionRow[column]);
predictionOnlyError += predictionDifference * predictionDifference;
reconstructionError += reconstructionDifference * reconstructionDifference;
}
} }
else
{
int visibleRow = Math.Min(row, height - 1);
byte expectedIndex = (byte)(visibleRow < height / 2 ? 0 : 1);
foreach (byte index in colorIndexMap.DangerousGetRowSpan(row))
{
Assert.Equal(expectedIndex, index);
}
Assert.True(sourcePlane.DangerousGetRowSpan(row).SequenceEqual(reconstructionPlane.DangerousGetRowSpan(row))); Assert.True(sourcePlane.DangerousGetRowSpan(row).SequenceEqual(reconstructionPlane.DangerousGetRowSpan(row)));
}
}
if (useSplitTransform)
{
Assert.True(predictionOnlyError > 0);
Assert.True(reconstructionError < predictionOnlyError);
byte[] payload = WriteCompleteTileObu(pictureTemplate, tileWriter, width, height);
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.NotNull(decoder.FrameInfo);
Av1BlockModeInfo decodedBlock = decoder.FrameInfo.GetModeInfoAt(default);
Assert.True(decodedBlock.GetPaletteSize(Av1Plane.Y) > 0);
Assert.Equal(4, decodedBlock.GetTransformUnitCount(Av1Plane.Y));
Assert.Equal(new Size(width, height), decoded.Size);
string outputDirectory = Path.Combine(
TestEnvironment.ActualOutputDirectoryFullPath,
"Formats",
"Heif",
"Av1");
Directory.CreateDirectory(outputDirectory);
File.WriteAllBytes(
Path.Combine(outputDirectory, $"encoder-palette-transform-size-select-{bitDepthValue}b.obu"),
payload);
} }
Assert.NotEqual(0, tileWriter.GetTileData(0).Length); Assert.NotEqual(0, tileWriter.GetTileData(0).Length);

Loading…
Cancel
Save