Browse Source

Enable adaptive AV1 intra-block copy

pull/2633/head
James Jackson-South 1 month ago
parent
commit
1c342699d6
  1. 6
      HEIF_IMPLEMENTATION_PLAN.md
  2. 61
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs
  3. 143
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs
  4. 14
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  5. 104
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs
  6. 88
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs
  7. 138
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

6
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

61
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs

@ -26,12 +26,40 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
/// <summary>
/// The complete workspace length in signed-integer storage elements.
/// </summary>
public const int StorageLength = ResidualStorageLength + MaximumCoefficientCount + MaximumCoefficientCount + Av1TransformWorkspace.MaximumLength;
public const int StorageLength =
ResidualStorageLength +
MaximumCoefficientCount +
MaximumCoefficientCount +
Av1TransformWorkspace.MaximumLength +
IntraBlockCopySampleStorageLength +
IntraBlockCopyResidualStorageLength +
IntraBlockCopyCoefficientStorageLength;
private const int ResidualStorageLength = MaximumResidualCount / 2;
private const int TransformCoefficientOffset = ResidualStorageLength;
private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount;
private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount;
private const int IntraBlockCopySampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength;
private const int IntraBlockCopySampleStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount *
sizeof(ushort) /
sizeof(int);
private const int IntraBlockCopyResidualStorageOffset =
IntraBlockCopySampleStorageOffset + IntraBlockCopySampleStorageLength;
private const int IntraBlockCopyResidualStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount *
sizeof(short) /
sizeof(int);
private const int IntraBlockCopyCoefficientStorageOffset =
IntraBlockCopyResidualStorageOffset + IntraBlockCopyResidualStorageLength;
private const int IntraBlockCopyCoefficientStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.CoefficientBufferCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount;
/// <summary>
/// Owns the complete reusable block workspace in 32-bit elements so every transform region is naturally aligned.
@ -69,6 +97,37 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
public Span<int> TransformWorkspace
=> this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength);
/// <summary>
/// Gets the reusable storage used while comparing intra-block-copy candidates.
/// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
/// <returns>The typed intra-block-copy workspace.</returns>
public Av1EncoderIntraBlockCopyWorkspace<TSample> GetIntraBlockCopyWorkspace<TSample>()
where TSample : unmanaged
{
Span<int> storage = this.owner.Memory.Span;
Span<TSample> sampleStorage = MemoryMarshal
.Cast<int, TSample>(storage.Slice(IntraBlockCopySampleStorageOffset, IntraBlockCopySampleStorageLength));
sampleStorage = sampleStorage[
..(Av1EncoderIntraBlockCopyWorkspace<TSample>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount)];
Span<short> residualStorage = MemoryMarshal
.Cast<int, short>(storage.Slice(IntraBlockCopyResidualStorageOffset, IntraBlockCopyResidualStorageLength));
residualStorage = residualStorage[..Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount];
Span<int> coefficientStorage = storage.Slice(
IntraBlockCopyCoefficientStorageOffset,
IntraBlockCopyCoefficientStorageLength);
return new Av1EncoderIntraBlockCopyWorkspace<TSample>(
sampleStorage,
residualStorage,
coefficientStorage);
}
/// <summary>
/// Releases the reusable block workspace.
/// </summary>

143
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs

@ -0,0 +1,143 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Provides disjoint reusable buffers for intra-block-copy mode decisions.
/// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace<TSample>
where TSample : unmanaged
{
/// <summary>
/// The number of samples in the fixed 8x8 intra-block-copy transform.
/// </summary>
public const int MaximumSampleCount = 8 * 8;
/// <summary>
/// The number of sample buffers retained by one mode decision.
/// </summary>
public const int SampleBufferCount = 10;
/// <summary>
/// The number of coefficient buffers retained by one mode decision.
/// </summary>
public const int CoefficientBufferCount = 7;
private readonly Span<TSample> samples;
private readonly Span<short> residual;
private readonly Span<int> coefficients;
/// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderIntraBlockCopyWorkspace{TSample}"/> struct.
/// </summary>
/// <param name="samples">The sample storage.</param>
/// <param name="residual">The residual storage shared by sequential plane evaluations.</param>
/// <param name="coefficients">The coefficient storage.</param>
public Av1EncoderIntraBlockCopyWorkspace(
Span<TSample> samples,
Span<short> residual,
Span<int> coefficients)
{
this.samples = samples;
this.residual = residual;
this.coefficients = coefficients;
}
/// <summary>
/// Gets the selected luma reconstruction.
/// </summary>
public Span<TSample> SelectedLumaReconstruction => this.GetSamples(0);
/// <summary>
/// Gets the selected blue-difference chroma reconstruction.
/// </summary>
public Span<TSample> SelectedBlueReconstruction => this.GetSamples(1);
/// <summary>
/// Gets the selected red-difference chroma reconstruction.
/// </summary>
public Span<TSample> SelectedRedReconstruction => this.GetSamples(2);
/// <summary>
/// Gets the current luma prediction.
/// </summary>
public Span<TSample> LumaPrediction => this.GetSamples(3);
/// <summary>
/// Gets the current blue-difference chroma prediction.
/// </summary>
public Span<TSample> BluePrediction => this.GetSamples(4);
/// <summary>
/// Gets the current red-difference chroma prediction.
/// </summary>
public Span<TSample> RedPrediction => this.GetSamples(5);
/// <summary>
/// Gets the current luma candidate reconstruction.
/// </summary>
public Span<TSample> LumaCandidateReconstruction => this.GetSamples(6);
/// <summary>
/// Gets the current blue-difference chroma candidate reconstruction.
/// </summary>
public Span<TSample> BlueCandidateReconstruction => this.GetSamples(7);
/// <summary>
/// Gets the current red-difference chroma candidate reconstruction.
/// </summary>
public Span<TSample> RedCandidateReconstruction => this.GetSamples(8);
/// <summary>
/// Gets the reconstruction scratch overwritten by each transform trial.
/// </summary>
public Span<TSample> TransformReconstruction => this.GetSamples(9);
/// <summary>
/// Gets the residual scratch shared by sequential plane evaluations.
/// </summary>
public Span<short> Residual => this.residual;
/// <summary>
/// Gets the selected luma coefficients.
/// </summary>
public Span<int> SelectedLumaCoefficients => this.GetCoefficients(0);
/// <summary>
/// Gets the selected blue-difference chroma coefficients.
/// </summary>
public Span<int> SelectedBlueCoefficients => this.GetCoefficients(1);
/// <summary>
/// Gets the selected red-difference chroma coefficients.
/// </summary>
public Span<int> SelectedRedCoefficients => this.GetCoefficients(2);
/// <summary>
/// Gets the current luma candidate coefficients.
/// </summary>
public Span<int> LumaCandidateCoefficients => this.GetCoefficients(3);
/// <summary>
/// Gets the current blue-difference chroma candidate coefficients.
/// </summary>
public Span<int> BlueCandidateCoefficients => this.GetCoefficients(4);
/// <summary>
/// Gets the current red-difference chroma candidate coefficients.
/// </summary>
public Span<int> RedCandidateCoefficients => this.GetCoefficients(5);
/// <summary>
/// Gets the coefficient scratch overwritten by each transform trial.
/// </summary>
public Span<int> TransformCoefficients => this.GetCoefficients(6);
private Span<TSample> GetSamples(int index)
=> this.samples.Slice(index * MaximumSampleCount, MaximumSampleCount);
private Span<int> GetCoefficients(int index)
=> this.coefficients.Slice(index * MaximumSampleCount, MaximumSampleCount);
}

14
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs

@ -235,8 +235,13 @@ internal static class Av1FrameEncoder
where TPixel : unmanaged, IPixel<TPixel>
{
PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig);
frameHeader.AllowScreenContentTools = Av1ScreenContentDetector.IsPaletteLikely(source.Frame);
Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
using Av1EncoderPictureBuffer picture = new(
configuration,
sequenceHeader,
@ -278,8 +283,13 @@ internal static class Av1FrameEncoder
where TPixel : unmanaged, IPixel<TPixel>
{
PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig);
frameHeader.AllowScreenContentTools = Av1ScreenContentDetector.IsPaletteLikely(source.Frame);
Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
using Av1EncoderPictureBuffer picture = new(
configuration,
sequenceHeader,

104
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs

@ -33,7 +33,6 @@ internal static partial class Av1IntraSuperblockEncoder
{
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8;
const int MaximumSampleCount = 8 * 8;
Buffer2DRegion<TSample> lumaSource = this.source.GetPlane(Av1Plane.Y);
Buffer2DRegion<TSample> lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y);
Point modeInfoPosition = new(
@ -119,27 +118,8 @@ internal static partial class Av1IntraSuperblockEncoder
Av1EncoderTransformBlockState selectedLumaState = default;
Av1EncoderTransformBlockState selectedBlueState = default;
Av1EncoderTransformBlockState selectedRedState = default;
Span<TSample> selectedLumaReconstruction = stackalloc TSample[MaximumSampleCount];
Span<TSample> selectedBlueReconstruction = stackalloc TSample[MaximumSampleCount];
Span<TSample> selectedRedReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> selectedLumaCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedBlueCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedRedCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> lumaPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> lumaResidual = stackalloc short[MaximumSampleCount];
Span<TSample> lumaCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> lumaCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> bluePrediction = stackalloc TSample[MaximumSampleCount];
Span<short> blueResidual = stackalloc short[MaximumSampleCount];
Span<TSample> blueCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> blueCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> redPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> redResidual = stackalloc short[MaximumSampleCount];
Span<TSample> redCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> redCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> transformReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> transformCoefficients = stackalloc int[MaximumSampleCount];
Av1EncoderIntraBlockCopyWorkspace<TSample> workspace =
this.blockWorkspace.GetIntraBlockCopyWorkspace<TSample>();
Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Luminance,
@ -192,12 +172,12 @@ internal static partial class Av1IntraSuperblockEncoder
0,
LumaTransformSize,
lumaContext,
lumaPrediction,
lumaResidual,
transformReconstruction,
transformCoefficients,
lumaCandidateReconstruction,
lumaCandidateCoefficients,
workspace.LumaPrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.LumaCandidateReconstruction,
workspace.LumaCandidateCoefficients,
out Av1EncoderTransformBlockState lumaCandidateState,
out int lumaRate,
out long lumaDistortion,
@ -229,12 +209,12 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingY,
chromaTransformSize,
blueContext,
bluePrediction,
blueResidual,
transformReconstruction,
transformCoefficients,
blueCandidateReconstruction,
blueCandidateCoefficients,
workspace.BluePrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.BlueCandidateReconstruction,
workspace.BlueCandidateCoefficients,
out blueCandidateState,
out blueRate,
out blueDistortion,
@ -252,12 +232,12 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingY,
chromaTransformSize,
redContext,
redPrediction,
redResidual,
transformReconstruction,
transformCoefficients,
redCandidateReconstruction,
redCandidateCoefficients,
workspace.RedPrediction,
workspace.Residual,
workspace.TransformReconstruction,
workspace.TransformCoefficients,
workspace.RedCandidateReconstruction,
workspace.RedCandidateCoefficients,
out redCandidateState,
out redRate,
out redDistortion,
@ -304,32 +284,40 @@ internal static partial class Av1IntraSuperblockEncoder
selectedVector = candidate;
if (candidateSkip)
{
lumaPrediction.CopyTo(selectedLumaReconstruction);
selectedLumaCoefficients.Clear();
workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction);
workspace.SelectedLumaCoefficients.Clear();
selectedLumaState = emptyLumaState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
bluePrediction[..chromaSampleCount].CopyTo(selectedBlueReconstruction);
redPrediction[..chromaSampleCount].CopyTo(selectedRedReconstruction);
selectedBlueCoefficients[..chromaSampleCount].Clear();
selectedRedCoefficients[..chromaSampleCount].Clear();
workspace.BluePrediction[..chromaSampleCount].CopyTo(workspace.SelectedBlueReconstruction);
workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction);
workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear();
workspace.SelectedRedCoefficients[..chromaSampleCount].Clear();
selectedBlueState = emptyBlueState;
selectedRedState = emptyRedState;
}
}
else
{
lumaCandidateReconstruction.CopyTo(selectedLumaReconstruction);
lumaCandidateCoefficients.CopyTo(selectedLumaCoefficients);
workspace.LumaCandidateReconstruction.CopyTo(workspace.SelectedLumaReconstruction);
workspace.LumaCandidateCoefficients.CopyTo(workspace.SelectedLumaCoefficients);
selectedLumaState = lumaCandidateState;
if (!this.source.IsMonochrome)
{
int chromaSampleCount = chromaTransformSize.GetSize2d();
blueCandidateReconstruction[..chromaSampleCount].CopyTo(selectedBlueReconstruction);
redCandidateReconstruction[..chromaSampleCount].CopyTo(selectedRedReconstruction);
blueCandidateCoefficients[..chromaSampleCount].CopyTo(selectedBlueCoefficients);
redCandidateCoefficients[..chromaSampleCount].CopyTo(selectedRedCoefficients);
workspace.BlueCandidateReconstruction[..chromaSampleCount]
.CopyTo(workspace.SelectedBlueReconstruction);
workspace.RedCandidateReconstruction[..chromaSampleCount]
.CopyTo(workspace.SelectedRedReconstruction);
workspace.BlueCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedBlueCoefficients);
workspace.RedCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedRedCoefficients);
selectedBlueState = blueCandidateState;
selectedRedState = redCandidateState;
}
@ -350,8 +338,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex];
CopyCandidate(
selectedLumaReconstruction,
selectedLumaCoefficients,
workspace.SelectedLumaReconstruction,
workspace.SelectedLumaCoefficients,
lumaReconstruction,
blockOrigin,
retainedLumaCoefficients[this.codedAreaLuma..],
@ -375,8 +363,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex];
ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex];
CopyCandidate(
selectedBlueReconstruction,
selectedBlueCoefficients,
workspace.SelectedBlueReconstruction,
workspace.SelectedBlueCoefficients,
this.reconstruction.GetPlane(Av1Plane.U),
chromaOrigin,
retainedBlueCoefficients[this.codedAreaChroma..],
@ -385,8 +373,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref retainedBlueState);
CopyCandidate(
selectedRedReconstruction,
selectedRedCoefficients,
workspace.SelectedRedReconstruction,
workspace.SelectedRedCoefficients,
this.reconstruction.GetPlane(Av1Plane.V),
chromaOrigin,
retainedRedCoefficients[this.codedAreaChroma..],

88
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs

@ -20,12 +20,11 @@ internal static class Av1ScreenContentDetector
where TSample : unmanaged
{
/// <summary>
/// Converts one native sample to eight-bit precision.
/// Converts one native sample to an integer without changing its precision.
/// </summary>
/// <param name="value">The source sample.</param>
/// <param name="bitDepthShift">The number of low bits removed from high-bit-depth samples.</param>
/// <returns>The normalized sample.</returns>
public static abstract int ToEightBit(TSample value, int bitDepthShift);
/// <returns>The native sample value.</returns>
public static abstract int ToInt32(TSample value);
}
/// <summary>
@ -34,7 +33,10 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param>
/// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns>
public static bool IsPaletteLikely(Av1EncoderFrame<byte> source)
=> IsPaletteLikely<byte, ByteSampleOperator>(source);
{
Detect(source, out bool allowScreenContentTools, out _);
return allowScreenContentTools;
}
/// <summary>
/// Detects palette-friendly content in a high-bit-depth source frame.
@ -42,9 +44,39 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param>
/// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns>
public static bool IsPaletteLikely(Av1EncoderFrame<ushort> source)
=> IsPaletteLikely<ushort, UShortSampleOperator>(source);
{
Detect(source, out bool allowScreenContentTools, out _);
return allowScreenContentTools;
}
private static bool IsPaletteLikely<TSample, TOperator>(Av1EncoderFrame<TSample> source)
/// <summary>
/// Detects palette and intra-block-copy content in an eight-bit source frame.
/// </summary>
/// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect(
Av1EncoderFrame<byte> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
=> Detect<byte, ByteSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy);
/// <summary>
/// Detects palette and intra-block-copy content in a high-bit-depth source frame.
/// </summary>
/// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect(
Av1EncoderFrame<ushort> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
=> Detect<ushort, UShortSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy);
private static void Detect<TSample, TOperator>(
Av1EncoderFrame<TSample> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
where TSample : unmanaged
where TOperator : struct, ISampleOperator<TSample>
{
@ -54,7 +86,10 @@ internal static class Av1ScreenContentDetector
long frameArea = (long)width * height;
int bitDepthShift = source.LumaBitDepth - 8;
int paletteBlockCount = 0;
int intraBlockCopyBlockCount = 0;
Span<ulong> seenColors = stackalloc ulong[4];
allowScreenContentTools = false;
allowIntraBlockCopy = false;
// Complete 16x16 blocks and the strict frame-area threshold preserve the reference detector's decision.
for (int blockRow = 0; blockRow + DetectionBlockLength <= height; blockRow += DetectionBlockLength)
@ -63,6 +98,8 @@ internal static class Av1ScreenContentDetector
{
seenColors.Clear();
int colorCount = 0;
long sum = 0;
long sumOfSquares = 0;
for (int row = 0; row < DetectionBlockLength && colorCount <= MaximumPaletteColorCount; row++)
{
ReadOnlySpan<TSample> samples = view
@ -72,10 +109,14 @@ internal static class Av1ScreenContentDetector
// Histogram updates depend on each sample value, so a compact scalar bitset avoids gather/scatter overhead.
for (int column = 0; column < samples.Length; column++)
{
int value = TOperator.ToEightBit(samples[column], bitDepthShift);
int nativeValue = TOperator.ToInt32(samples[column]);
int value = nativeValue >> bitDepthShift;
int wordIndex = value >> 6;
ulong mask = 1UL << (value & 63);
ref ulong word = ref seenColors[wordIndex];
int centeredValue = nativeValue - (128 << bitDepthShift);
sum += centeredValue;
sumOfSquares += (long)centeredValue * centeredValue;
if ((word & mask) == 0)
{
word |= mask;
@ -91,24 +132,43 @@ internal static class Av1ScreenContentDetector
if (colorCount > 1 && colorCount <= MaximumPaletteColorCount)
{
paletteBlockCount++;
if ((long)paletteBlockCount * DetectionBlockArea * 10 > frameArea)
long normalizedSum = sum;
long normalizedSumOfSquares = sumOfSquares;
if (bitDepthShift != 0)
{
normalizedSum = RoundPowerOfTwo(sum, bitDepthShift);
normalizedSumOfSquares = RoundPowerOfTwo(sumOfSquares, bitDepthShift * 2);
}
long variance = normalizedSumOfSquares - ((normalizedSum * normalizedSum) >> 8);
if (variance >= DetectionBlockArea / 2)
{
intraBlockCopyBlockCount++;
}
allowScreenContentTools = (long)paletteBlockCount * DetectionBlockArea * 10 > frameArea;
allowIntraBlockCopy = allowScreenContentTools &&
(long)intraBlockCopyBlockCount * DetectionBlockArea * 12 > frameArea;
if (allowIntraBlockCopy)
{
return true;
return;
}
}
}
}
return false;
}
private static long RoundPowerOfTwo(long value, int shift)
=> (value + (1L << (shift - 1))) >> shift;
/// <summary>
/// Preserves native eight-bit samples.
/// </summary>
private readonly struct ByteSampleOperator : ISampleOperator<byte>
{
/// <inheritdoc/>
public static int ToEightBit(byte value, int bitDepthShift) => value;
public static int ToInt32(byte value) => value;
}
/// <summary>
@ -117,6 +177,6 @@ internal static class Av1ScreenContentDetector
private readonly struct UShortSampleOperator : ISampleOperator<ushort>
{
/// <inheritdoc/>
public static int ToEightBit(ushort value, int bitDepthShift) => value >> bitDepthShift;
public static int ToInt32(ushort value) => value;
}
}

138
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

@ -4,6 +4,7 @@
using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Tests.Memory;
@ -147,7 +148,7 @@ public class Av1EncoderFrameTests
}
[Fact]
public void ScreenContentDetectorMatchesLibaomPaletteThreshold()
public void ScreenContentDetectorMatchesLibaomFeatureThresholds()
{
const int width = 160;
const int height = 16;
@ -172,6 +173,13 @@ public class Av1EncoderFrameTests
// One qualifying block is exactly ten percent of this frame, and the reference threshold is strict.
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame));
Av1ScreenContentDetector.Detect(
byteFrame.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
for (int row = 0; row < height; row++)
{
Span<byte> samples = byteFrame.Frame.View.GetLumaRowSpan(row);
@ -182,6 +190,13 @@ public class Av1EncoderFrameTests
}
Assert.True(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame));
Av1ScreenContentDetector.Detect(
byteFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
using Av1EncoderFrameBuffer<ushort> highBitDepthFrame = new(
Configuration.Default,
16,
@ -201,6 +216,13 @@ public class Av1EncoderFrameTests
}
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
for (int row = 0; row < 16; row++)
{
Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row);
@ -208,6 +230,13 @@ public class Av1EncoderFrameTests
}
Assert.True(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
for (int row = 0; row < 16; row++)
{
Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row);
@ -218,10 +247,58 @@ public class Av1EncoderFrameTests
}
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
}
[Fact]
public void EncodeActivatesPaletteToolsForScreenContent()
public void ScreenContentDetectorMatchesLibaomIntraBlockCopyVarianceThreshold()
{
const int Width = 16;
const int Height = 16;
using Av1EncoderFrameBuffer<byte> frame = new(
Configuration.Default,
Width,
Height,
8,
Av1ColorFormat.Yuv400,
0,
0);
Buffer2DRegion<byte> luma = frame.Frame.View.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++)
{
luma.DangerousGetRowSpan(row).Fill(96);
}
// A single delta of eleven leaves total variance below half a sample after per-pixel rounding.
luma.DangerousGetRowSpan(0)[0] = 107;
Av1ScreenContentDetector.Detect(
frame.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
// Raising that delta to twelve crosses the exact integer rounding boundary used by libaom.
luma.DangerousGetRowSpan(0)[0] = 108;
Av1ScreenContentDetector.Detect(
frame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
}
[Fact]
public void EncodeActivatesScreenContentTools()
{
const int width = 16;
const int height = 16;
@ -253,7 +330,7 @@ public class Av1EncoderFrameTests
obuReader.ReadAll(ref reader, payload.Length, () => tileReader);
ObuFrameHeader frameHeader = Assert.IsType<ObuFrameHeader>(obuReader.FrameHeader);
Assert.True(frameHeader.AllowScreenContentTools);
Assert.False(frameHeader.AllowIntraBlockCopy);
Assert.True(frameHeader.AllowIntraBlockCopy);
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.Equal(new Size(width, height), decoded.Size);
@ -268,6 +345,61 @@ public class Av1EncoderFrameTests
File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-16x16-8b-444-palette.obu"), payload);
}
[Fact]
public void EncodeSelectsIntraBlockCopyForRepeatedScreenContent()
{
const int Width = 328;
const int Height = 16;
const ulong Pattern = 0xD6A5_3C97_E18B_4F20UL;
using Image<Rgba32> source = new(Width, Height);
for (int row = 0; row < Height; row++)
{
Span<Rgba32> pixels = source.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row);
for (int column = 0; column < Width; column++)
{
int patternIndex = ((row & 7) * 8) + (column & 7);
pixels[column] = ((Pattern >> patternIndex) & 1) == 0
? new Rgba32(224, 32, 32)
: new Rgba32(32, 32, 224);
}
}
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv444);
using MemoryStream stream = new();
_ = Av1FrameEncoder.Encode(
Configuration.Default,
source.Frames.RootFrame,
stream,
colorConfig,
qIndex: 37);
byte[] payload = stream.ToArray();
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.NotNull(decoder.FrameHeader);
Assert.True(decoder.FrameHeader.AllowScreenContentTools);
Assert.True(decoder.FrameHeader.AllowIntraBlockCopy);
Assert.NotNull(decoder.FrameInfo);
Av1SuperblockInfo targetSuperblock = decoder.FrameInfo.GetSuperblock(new Point(5, 0));
bool usesIntraBlockCopy = false;
foreach (Av1BlockModeInfo modeInfo in targetSuperblock.GetModeInfos())
{
usesIntraBlockCopy |= modeInfo.UseIntraBlockCopy;
}
Assert.True(usesIntraBlockCopy);
Assert.Equal(new Size(Width, Height), decoded.Size);
string outputDirectory = Path.Combine(
TestEnvironment.ActualOutputDirectoryFullPath,
"Formats",
"Heif",
"Av1");
Directory.CreateDirectory(outputDirectory);
File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-328x16-8b-444-intrabc.obu"), payload);
}
[Fact]
public void PrepareSourceConvertsRgba32DirectlyIntoBorderedEightBitPlane()
{

Loading…
Cancel
Save