Browse Source

Enable adaptive AV1 intra-block copy

pull/2633/head
James Jackson-South 1 month ago
parent
commit
1c342699d6
  1. 6
      HEIF_IMPLEMENTATION_PLAN.md
  2. 61
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs
  3. 143
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs
  4. 14
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  5. 104
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs
  6. 88
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs
  7. 138
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

6
HEIF_IMPLEMENTATION_PLAN.md

File diff suppressed because one or more lines are too long

61
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderBlockWorkspace.cs

@ -26,12 +26,40 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
/// <summary> /// <summary>
/// The complete workspace length in signed-integer storage elements. /// The complete workspace length in signed-integer storage elements.
/// </summary> /// </summary>
public const int StorageLength = ResidualStorageLength + MaximumCoefficientCount + MaximumCoefficientCount + Av1TransformWorkspace.MaximumLength; public const int StorageLength =
ResidualStorageLength +
MaximumCoefficientCount +
MaximumCoefficientCount +
Av1TransformWorkspace.MaximumLength +
IntraBlockCopySampleStorageLength +
IntraBlockCopyResidualStorageLength +
IntraBlockCopyCoefficientStorageLength;
private const int ResidualStorageLength = MaximumResidualCount / 2; private const int ResidualStorageLength = MaximumResidualCount / 2;
private const int TransformCoefficientOffset = ResidualStorageLength; private const int TransformCoefficientOffset = ResidualStorageLength;
private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount; private const int DequantizedCoefficientOffset = TransformCoefficientOffset + MaximumCoefficientCount;
private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount; private const int TransformWorkspaceOffset = DequantizedCoefficientOffset + MaximumCoefficientCount;
private const int IntraBlockCopySampleStorageOffset = TransformWorkspaceOffset + Av1TransformWorkspace.MaximumLength;
private const int IntraBlockCopySampleStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount *
sizeof(ushort) /
sizeof(int);
private const int IntraBlockCopyResidualStorageOffset =
IntraBlockCopySampleStorageOffset + IntraBlockCopySampleStorageLength;
private const int IntraBlockCopyResidualStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount *
sizeof(short) /
sizeof(int);
private const int IntraBlockCopyCoefficientStorageOffset =
IntraBlockCopyResidualStorageOffset + IntraBlockCopyResidualStorageLength;
private const int IntraBlockCopyCoefficientStorageLength =
Av1EncoderIntraBlockCopyWorkspace<ushort>.CoefficientBufferCount *
Av1EncoderIntraBlockCopyWorkspace<ushort>.MaximumSampleCount;
/// <summary> /// <summary>
/// Owns the complete reusable block workspace in 32-bit elements so every transform region is naturally aligned. /// Owns the complete reusable block workspace in 32-bit elements so every transform region is naturally aligned.
@ -69,6 +97,37 @@ internal sealed class Av1EncoderBlockWorkspace : IDisposable
public Span<int> TransformWorkspace public Span<int> TransformWorkspace
=> this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength); => this.owner.Memory.Span.Slice(TransformWorkspaceOffset, Av1TransformWorkspace.MaximumLength);
/// <summary>
/// Gets the reusable storage used while comparing intra-block-copy candidates.
/// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
/// <returns>The typed intra-block-copy workspace.</returns>
public Av1EncoderIntraBlockCopyWorkspace<TSample> GetIntraBlockCopyWorkspace<TSample>()
where TSample : unmanaged
{
Span<int> storage = this.owner.Memory.Span;
Span<TSample> sampleStorage = MemoryMarshal
.Cast<int, TSample>(storage.Slice(IntraBlockCopySampleStorageOffset, IntraBlockCopySampleStorageLength));
sampleStorage = sampleStorage[
..(Av1EncoderIntraBlockCopyWorkspace<TSample>.SampleBufferCount *
Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount)];
Span<short> residualStorage = MemoryMarshal
.Cast<int, short>(storage.Slice(IntraBlockCopyResidualStorageOffset, IntraBlockCopyResidualStorageLength));
residualStorage = residualStorage[..Av1EncoderIntraBlockCopyWorkspace<TSample>.MaximumSampleCount];
Span<int> coefficientStorage = storage.Slice(
IntraBlockCopyCoefficientStorageOffset,
IntraBlockCopyCoefficientStorageLength);
return new Av1EncoderIntraBlockCopyWorkspace<TSample>(
sampleStorage,
residualStorage,
coefficientStorage);
}
/// <summary> /// <summary>
/// Releases the reusable block workspace. /// Releases the reusable block workspace.
/// </summary> /// </summary>

143
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1EncoderIntraBlockCopyWorkspace.cs

@ -0,0 +1,143 @@
// Copyright (c) Six Labors.
// Licensed under the Six Labors Split License.
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Provides disjoint reusable buffers for intra-block-copy mode decisions.
/// </summary>
/// <typeparam name="TSample">The native sample type selected by the encoder pipeline.</typeparam>
internal readonly ref struct Av1EncoderIntraBlockCopyWorkspace<TSample>
where TSample : unmanaged
{
/// <summary>
/// The number of samples in the fixed 8x8 intra-block-copy transform.
/// </summary>
public const int MaximumSampleCount = 8 * 8;
/// <summary>
/// The number of sample buffers retained by one mode decision.
/// </summary>
public const int SampleBufferCount = 10;
/// <summary>
/// The number of coefficient buffers retained by one mode decision.
/// </summary>
public const int CoefficientBufferCount = 7;
private readonly Span<TSample> samples;
private readonly Span<short> residual;
private readonly Span<int> coefficients;
/// <summary>
/// Initializes a new instance of the <see cref="Av1EncoderIntraBlockCopyWorkspace{TSample}"/> struct.
/// </summary>
/// <param name="samples">The sample storage.</param>
/// <param name="residual">The residual storage shared by sequential plane evaluations.</param>
/// <param name="coefficients">The coefficient storage.</param>
public Av1EncoderIntraBlockCopyWorkspace(
Span<TSample> samples,
Span<short> residual,
Span<int> coefficients)
{
this.samples = samples;
this.residual = residual;
this.coefficients = coefficients;
}
/// <summary>
/// Gets the selected luma reconstruction.
/// </summary>
public Span<TSample> SelectedLumaReconstruction => this.GetSamples(0);
/// <summary>
/// Gets the selected blue-difference chroma reconstruction.
/// </summary>
public Span<TSample> SelectedBlueReconstruction => this.GetSamples(1);
/// <summary>
/// Gets the selected red-difference chroma reconstruction.
/// </summary>
public Span<TSample> SelectedRedReconstruction => this.GetSamples(2);
/// <summary>
/// Gets the current luma prediction.
/// </summary>
public Span<TSample> LumaPrediction => this.GetSamples(3);
/// <summary>
/// Gets the current blue-difference chroma prediction.
/// </summary>
public Span<TSample> BluePrediction => this.GetSamples(4);
/// <summary>
/// Gets the current red-difference chroma prediction.
/// </summary>
public Span<TSample> RedPrediction => this.GetSamples(5);
/// <summary>
/// Gets the current luma candidate reconstruction.
/// </summary>
public Span<TSample> LumaCandidateReconstruction => this.GetSamples(6);
/// <summary>
/// Gets the current blue-difference chroma candidate reconstruction.
/// </summary>
public Span<TSample> BlueCandidateReconstruction => this.GetSamples(7);
/// <summary>
/// Gets the current red-difference chroma candidate reconstruction.
/// </summary>
public Span<TSample> RedCandidateReconstruction => this.GetSamples(8);
/// <summary>
/// Gets the reconstruction scratch overwritten by each transform trial.
/// </summary>
public Span<TSample> TransformReconstruction => this.GetSamples(9);
/// <summary>
/// Gets the residual scratch shared by sequential plane evaluations.
/// </summary>
public Span<short> Residual => this.residual;
/// <summary>
/// Gets the selected luma coefficients.
/// </summary>
public Span<int> SelectedLumaCoefficients => this.GetCoefficients(0);
/// <summary>
/// Gets the selected blue-difference chroma coefficients.
/// </summary>
public Span<int> SelectedBlueCoefficients => this.GetCoefficients(1);
/// <summary>
/// Gets the selected red-difference chroma coefficients.
/// </summary>
public Span<int> SelectedRedCoefficients => this.GetCoefficients(2);
/// <summary>
/// Gets the current luma candidate coefficients.
/// </summary>
public Span<int> LumaCandidateCoefficients => this.GetCoefficients(3);
/// <summary>
/// Gets the current blue-difference chroma candidate coefficients.
/// </summary>
public Span<int> BlueCandidateCoefficients => this.GetCoefficients(4);
/// <summary>
/// Gets the current red-difference chroma candidate coefficients.
/// </summary>
public Span<int> RedCandidateCoefficients => this.GetCoefficients(5);
/// <summary>
/// Gets the coefficient scratch overwritten by each transform trial.
/// </summary>
public Span<int> TransformCoefficients => this.GetCoefficients(6);
private Span<TSample> GetSamples(int index)
=> this.samples.Slice(index * MaximumSampleCount, MaximumSampleCount);
private Span<int> GetCoefficients(int index)
=> this.coefficients.Slice(index * MaximumSampleCount, MaximumSampleCount);
}

14
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs

@ -235,8 +235,13 @@ internal static class Av1FrameEncoder
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig); PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig);
frameHeader.AllowScreenContentTools = Av1ScreenContentDetector.IsPaletteLikely(source.Frame); Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
using Av1EncoderPictureBuffer picture = new( using Av1EncoderPictureBuffer picture = new(
configuration, configuration,
sequenceHeader, sequenceHeader,
@ -278,8 +283,13 @@ internal static class Av1FrameEncoder
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig); PrepareSource(configuration, image, source.Frame, sequenceHeader.ColorConfig);
frameHeader.AllowScreenContentTools = Av1ScreenContentDetector.IsPaletteLikely(source.Frame); Av1ScreenContentDetector.Detect(
source.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
frameHeader.AllowScreenContentTools = allowScreenContentTools;
frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
using Av1EncoderPictureBuffer picture = new( using Av1EncoderPictureBuffer picture = new(
configuration, configuration,
sequenceHeader, sequenceHeader,

104
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.IntraBlockCopyModeDecision.cs

@ -33,7 +33,6 @@ internal static partial class Av1IntraSuperblockEncoder
{ {
const Av1BlockSize BlockSize = Av1BlockSize.Block8x8; const Av1BlockSize BlockSize = Av1BlockSize.Block8x8;
const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8; const Av1TransformSize LumaTransformSize = Av1TransformSize.Size8x8;
const int MaximumSampleCount = 8 * 8;
Buffer2DRegion<TSample> lumaSource = this.source.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> lumaSource = this.source.GetPlane(Av1Plane.Y);
Buffer2DRegion<TSample> lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y); Buffer2DRegion<TSample> lumaReconstruction = this.reconstruction.GetPlane(Av1Plane.Y);
Point modeInfoPosition = new( Point modeInfoPosition = new(
@ -119,27 +118,8 @@ internal static partial class Av1IntraSuperblockEncoder
Av1EncoderTransformBlockState selectedLumaState = default; Av1EncoderTransformBlockState selectedLumaState = default;
Av1EncoderTransformBlockState selectedBlueState = default; Av1EncoderTransformBlockState selectedBlueState = default;
Av1EncoderTransformBlockState selectedRedState = default; Av1EncoderTransformBlockState selectedRedState = default;
Span<TSample> selectedLumaReconstruction = stackalloc TSample[MaximumSampleCount]; Av1EncoderIntraBlockCopyWorkspace<TSample> workspace =
Span<TSample> selectedBlueReconstruction = stackalloc TSample[MaximumSampleCount]; this.blockWorkspace.GetIntraBlockCopyWorkspace<TSample>();
Span<TSample> selectedRedReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> selectedLumaCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedBlueCoefficients = stackalloc int[MaximumSampleCount];
Span<int> selectedRedCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> lumaPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> lumaResidual = stackalloc short[MaximumSampleCount];
Span<TSample> lumaCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> lumaCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> bluePrediction = stackalloc TSample[MaximumSampleCount];
Span<short> blueResidual = stackalloc short[MaximumSampleCount];
Span<TSample> blueCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> blueCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> redPrediction = stackalloc TSample[MaximumSampleCount];
Span<short> redResidual = stackalloc short[MaximumSampleCount];
Span<TSample> redCandidateReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> redCandidateCoefficients = stackalloc int[MaximumSampleCount];
Span<TSample> transformReconstruction = stackalloc TSample[MaximumSampleCount];
Span<int> transformCoefficients = stackalloc int[MaximumSampleCount];
Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts( Av1TransformBlockContext lumaContext = Av1TileWriter.GetTransformBlockContexts(
Av1ComponentType.Luminance, Av1ComponentType.Luminance,
@ -192,12 +172,12 @@ internal static partial class Av1IntraSuperblockEncoder
0, 0,
LumaTransformSize, LumaTransformSize,
lumaContext, lumaContext,
lumaPrediction, workspace.LumaPrediction,
lumaResidual, workspace.Residual,
transformReconstruction, workspace.TransformReconstruction,
transformCoefficients, workspace.TransformCoefficients,
lumaCandidateReconstruction, workspace.LumaCandidateReconstruction,
lumaCandidateCoefficients, workspace.LumaCandidateCoefficients,
out Av1EncoderTransformBlockState lumaCandidateState, out Av1EncoderTransformBlockState lumaCandidateState,
out int lumaRate, out int lumaRate,
out long lumaDistortion, out long lumaDistortion,
@ -229,12 +209,12 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingY, subsamplingY,
chromaTransformSize, chromaTransformSize,
blueContext, blueContext,
bluePrediction, workspace.BluePrediction,
blueResidual, workspace.Residual,
transformReconstruction, workspace.TransformReconstruction,
transformCoefficients, workspace.TransformCoefficients,
blueCandidateReconstruction, workspace.BlueCandidateReconstruction,
blueCandidateCoefficients, workspace.BlueCandidateCoefficients,
out blueCandidateState, out blueCandidateState,
out blueRate, out blueRate,
out blueDistortion, out blueDistortion,
@ -252,12 +232,12 @@ internal static partial class Av1IntraSuperblockEncoder
subsamplingY, subsamplingY,
chromaTransformSize, chromaTransformSize,
redContext, redContext,
redPrediction, workspace.RedPrediction,
redResidual, workspace.Residual,
transformReconstruction, workspace.TransformReconstruction,
transformCoefficients, workspace.TransformCoefficients,
redCandidateReconstruction, workspace.RedCandidateReconstruction,
redCandidateCoefficients, workspace.RedCandidateCoefficients,
out redCandidateState, out redCandidateState,
out redRate, out redRate,
out redDistortion, out redDistortion,
@ -304,32 +284,40 @@ internal static partial class Av1IntraSuperblockEncoder
selectedVector = candidate; selectedVector = candidate;
if (candidateSkip) if (candidateSkip)
{ {
lumaPrediction.CopyTo(selectedLumaReconstruction); workspace.LumaPrediction.CopyTo(workspace.SelectedLumaReconstruction);
selectedLumaCoefficients.Clear(); workspace.SelectedLumaCoefficients.Clear();
selectedLumaState = emptyLumaState; selectedLumaState = emptyLumaState;
if (!this.source.IsMonochrome) if (!this.source.IsMonochrome)
{ {
int chromaSampleCount = chromaTransformSize.GetSize2d(); int chromaSampleCount = chromaTransformSize.GetSize2d();
bluePrediction[..chromaSampleCount].CopyTo(selectedBlueReconstruction); workspace.BluePrediction[..chromaSampleCount].CopyTo(workspace.SelectedBlueReconstruction);
redPrediction[..chromaSampleCount].CopyTo(selectedRedReconstruction); workspace.RedPrediction[..chromaSampleCount].CopyTo(workspace.SelectedRedReconstruction);
selectedBlueCoefficients[..chromaSampleCount].Clear(); workspace.SelectedBlueCoefficients[..chromaSampleCount].Clear();
selectedRedCoefficients[..chromaSampleCount].Clear(); workspace.SelectedRedCoefficients[..chromaSampleCount].Clear();
selectedBlueState = emptyBlueState; selectedBlueState = emptyBlueState;
selectedRedState = emptyRedState; selectedRedState = emptyRedState;
} }
} }
else else
{ {
lumaCandidateReconstruction.CopyTo(selectedLumaReconstruction); workspace.LumaCandidateReconstruction.CopyTo(workspace.SelectedLumaReconstruction);
lumaCandidateCoefficients.CopyTo(selectedLumaCoefficients); workspace.LumaCandidateCoefficients.CopyTo(workspace.SelectedLumaCoefficients);
selectedLumaState = lumaCandidateState; selectedLumaState = lumaCandidateState;
if (!this.source.IsMonochrome) if (!this.source.IsMonochrome)
{ {
int chromaSampleCount = chromaTransformSize.GetSize2d(); int chromaSampleCount = chromaTransformSize.GetSize2d();
blueCandidateReconstruction[..chromaSampleCount].CopyTo(selectedBlueReconstruction); workspace.BlueCandidateReconstruction[..chromaSampleCount]
redCandidateReconstruction[..chromaSampleCount].CopyTo(selectedRedReconstruction); .CopyTo(workspace.SelectedBlueReconstruction);
blueCandidateCoefficients[..chromaSampleCount].CopyTo(selectedBlueCoefficients);
redCandidateCoefficients[..chromaSampleCount].CopyTo(selectedRedCoefficients); workspace.RedCandidateReconstruction[..chromaSampleCount]
.CopyTo(workspace.SelectedRedReconstruction);
workspace.BlueCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedBlueCoefficients);
workspace.RedCandidateCoefficients[..chromaSampleCount]
.CopyTo(workspace.SelectedRedCoefficients);
selectedBlueState = blueCandidateState; selectedBlueState = blueCandidateState;
selectedRedState = redCandidateState; selectedRedState = redCandidateState;
} }
@ -350,8 +338,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex]; ref Av1EncoderTransformBlockState retainedLumaState = ref retainedLumaTransformBlocks[lumaTransformIndex];
CopyCandidate( CopyCandidate(
selectedLumaReconstruction, workspace.SelectedLumaReconstruction,
selectedLumaCoefficients, workspace.SelectedLumaCoefficients,
lumaReconstruction, lumaReconstruction,
blockOrigin, blockOrigin,
retainedLumaCoefficients[this.codedAreaLuma..], retainedLumaCoefficients[this.codedAreaLuma..],
@ -375,8 +363,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex]; ref Av1EncoderTransformBlockState retainedBlueState = ref retainedBlueTransformBlocks[chromaTransformIndex];
ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex]; ref Av1EncoderTransformBlockState retainedRedState = ref retainedRedTransformBlocks[chromaTransformIndex];
CopyCandidate( CopyCandidate(
selectedBlueReconstruction, workspace.SelectedBlueReconstruction,
selectedBlueCoefficients, workspace.SelectedBlueCoefficients,
this.reconstruction.GetPlane(Av1Plane.U), this.reconstruction.GetPlane(Av1Plane.U),
chromaOrigin, chromaOrigin,
retainedBlueCoefficients[this.codedAreaChroma..], retainedBlueCoefficients[this.codedAreaChroma..],
@ -385,8 +373,8 @@ internal static partial class Av1IntraSuperblockEncoder
ref retainedBlueState); ref retainedBlueState);
CopyCandidate( CopyCandidate(
selectedRedReconstruction, workspace.SelectedRedReconstruction,
selectedRedCoefficients, workspace.SelectedRedCoefficients,
this.reconstruction.GetPlane(Av1Plane.V), this.reconstruction.GetPlane(Av1Plane.V),
chromaOrigin, chromaOrigin,
retainedRedCoefficients[this.codedAreaChroma..], retainedRedCoefficients[this.codedAreaChroma..],

88
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1ScreenContentDetector.cs

@ -20,12 +20,11 @@ internal static class Av1ScreenContentDetector
where TSample : unmanaged where TSample : unmanaged
{ {
/// <summary> /// <summary>
/// Converts one native sample to eight-bit precision. /// Converts one native sample to an integer without changing its precision.
/// </summary> /// </summary>
/// <param name="value">The source sample.</param> /// <param name="value">The source sample.</param>
/// <param name="bitDepthShift">The number of low bits removed from high-bit-depth samples.</param> /// <returns>The native sample value.</returns>
/// <returns>The normalized sample.</returns> public static abstract int ToInt32(TSample value);
public static abstract int ToEightBit(TSample value, int bitDepthShift);
} }
/// <summary> /// <summary>
@ -34,7 +33,10 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param> /// <param name="source">The converted source frame.</param>
/// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns> /// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns>
public static bool IsPaletteLikely(Av1EncoderFrame<byte> source) public static bool IsPaletteLikely(Av1EncoderFrame<byte> source)
=> IsPaletteLikely<byte, ByteSampleOperator>(source); {
Detect(source, out bool allowScreenContentTools, out _);
return allowScreenContentTools;
}
/// <summary> /// <summary>
/// Detects palette-friendly content in a high-bit-depth source frame. /// Detects palette-friendly content in a high-bit-depth source frame.
@ -42,9 +44,39 @@ internal static class Av1ScreenContentDetector
/// <param name="source">The converted source frame.</param> /// <param name="source">The converted source frame.</param>
/// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns> /// <returns><see langword="true"/> when palette tools should be enabled; otherwise, <see langword="false"/>.</returns>
public static bool IsPaletteLikely(Av1EncoderFrame<ushort> source) public static bool IsPaletteLikely(Av1EncoderFrame<ushort> source)
=> IsPaletteLikely<ushort, UShortSampleOperator>(source); {
Detect(source, out bool allowScreenContentTools, out _);
return allowScreenContentTools;
}
private static bool IsPaletteLikely<TSample, TOperator>(Av1EncoderFrame<TSample> source) /// <summary>
/// Detects palette and intra-block-copy content in an eight-bit source frame.
/// </summary>
/// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect(
Av1EncoderFrame<byte> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
=> Detect<byte, ByteSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy);
/// <summary>
/// Detects palette and intra-block-copy content in a high-bit-depth source frame.
/// </summary>
/// <param name="source">The converted source frame.</param>
/// <param name="allowScreenContentTools">Receives whether palette syntax should be enabled.</param>
/// <param name="allowIntraBlockCopy">Receives whether intra-block copy should be enabled.</param>
public static void Detect(
Av1EncoderFrame<ushort> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
=> Detect<ushort, UShortSampleOperator>(source, out allowScreenContentTools, out allowIntraBlockCopy);
private static void Detect<TSample, TOperator>(
Av1EncoderFrame<TSample> source,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy)
where TSample : unmanaged where TSample : unmanaged
where TOperator : struct, ISampleOperator<TSample> where TOperator : struct, ISampleOperator<TSample>
{ {
@ -54,7 +86,10 @@ internal static class Av1ScreenContentDetector
long frameArea = (long)width * height; long frameArea = (long)width * height;
int bitDepthShift = source.LumaBitDepth - 8; int bitDepthShift = source.LumaBitDepth - 8;
int paletteBlockCount = 0; int paletteBlockCount = 0;
int intraBlockCopyBlockCount = 0;
Span<ulong> seenColors = stackalloc ulong[4]; Span<ulong> seenColors = stackalloc ulong[4];
allowScreenContentTools = false;
allowIntraBlockCopy = false;
// Complete 16x16 blocks and the strict frame-area threshold preserve the reference detector's decision. // Complete 16x16 blocks and the strict frame-area threshold preserve the reference detector's decision.
for (int blockRow = 0; blockRow + DetectionBlockLength <= height; blockRow += DetectionBlockLength) for (int blockRow = 0; blockRow + DetectionBlockLength <= height; blockRow += DetectionBlockLength)
@ -63,6 +98,8 @@ internal static class Av1ScreenContentDetector
{ {
seenColors.Clear(); seenColors.Clear();
int colorCount = 0; int colorCount = 0;
long sum = 0;
long sumOfSquares = 0;
for (int row = 0; row < DetectionBlockLength && colorCount <= MaximumPaletteColorCount; row++) for (int row = 0; row < DetectionBlockLength && colorCount <= MaximumPaletteColorCount; row++)
{ {
ReadOnlySpan<TSample> samples = view ReadOnlySpan<TSample> samples = view
@ -72,10 +109,14 @@ internal static class Av1ScreenContentDetector
// Histogram updates depend on each sample value, so a compact scalar bitset avoids gather/scatter overhead. // Histogram updates depend on each sample value, so a compact scalar bitset avoids gather/scatter overhead.
for (int column = 0; column < samples.Length; column++) for (int column = 0; column < samples.Length; column++)
{ {
int value = TOperator.ToEightBit(samples[column], bitDepthShift); int nativeValue = TOperator.ToInt32(samples[column]);
int value = nativeValue >> bitDepthShift;
int wordIndex = value >> 6; int wordIndex = value >> 6;
ulong mask = 1UL << (value & 63); ulong mask = 1UL << (value & 63);
ref ulong word = ref seenColors[wordIndex]; ref ulong word = ref seenColors[wordIndex];
int centeredValue = nativeValue - (128 << bitDepthShift);
sum += centeredValue;
sumOfSquares += (long)centeredValue * centeredValue;
if ((word & mask) == 0) if ((word & mask) == 0)
{ {
word |= mask; word |= mask;
@ -91,24 +132,43 @@ internal static class Av1ScreenContentDetector
if (colorCount > 1 && colorCount <= MaximumPaletteColorCount) if (colorCount > 1 && colorCount <= MaximumPaletteColorCount)
{ {
paletteBlockCount++; paletteBlockCount++;
if ((long)paletteBlockCount * DetectionBlockArea * 10 > frameArea) long normalizedSum = sum;
long normalizedSumOfSquares = sumOfSquares;
if (bitDepthShift != 0)
{
normalizedSum = RoundPowerOfTwo(sum, bitDepthShift);
normalizedSumOfSquares = RoundPowerOfTwo(sumOfSquares, bitDepthShift * 2);
}
long variance = normalizedSumOfSquares - ((normalizedSum * normalizedSum) >> 8);
if (variance >= DetectionBlockArea / 2)
{
intraBlockCopyBlockCount++;
}
allowScreenContentTools = (long)paletteBlockCount * DetectionBlockArea * 10 > frameArea;
allowIntraBlockCopy = allowScreenContentTools &&
(long)intraBlockCopyBlockCount * DetectionBlockArea * 12 > frameArea;
if (allowIntraBlockCopy)
{ {
return true; return;
} }
} }
} }
} }
return false;
} }
private static long RoundPowerOfTwo(long value, int shift)
=> (value + (1L << (shift - 1))) >> shift;
/// <summary> /// <summary>
/// Preserves native eight-bit samples. /// Preserves native eight-bit samples.
/// </summary> /// </summary>
private readonly struct ByteSampleOperator : ISampleOperator<byte> private readonly struct ByteSampleOperator : ISampleOperator<byte>
{ {
/// <inheritdoc/> /// <inheritdoc/>
public static int ToEightBit(byte value, int bitDepthShift) => value; public static int ToInt32(byte value) => value;
} }
/// <summary> /// <summary>
@ -117,6 +177,6 @@ internal static class Av1ScreenContentDetector
private readonly struct UShortSampleOperator : ISampleOperator<ushort> private readonly struct UShortSampleOperator : ISampleOperator<ushort>
{ {
/// <inheritdoc/> /// <inheritdoc/>
public static int ToEightBit(ushort value, int bitDepthShift) => value >> bitDepthShift; public static int ToInt32(ushort value) => value;
} }
} }

138
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

@ -4,6 +4,7 @@
using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1;
using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Tests.Memory; using SixLabors.ImageSharp.Tests.Memory;
@ -147,7 +148,7 @@ public class Av1EncoderFrameTests
} }
[Fact] [Fact]
public void ScreenContentDetectorMatchesLibaomPaletteThreshold() public void ScreenContentDetectorMatchesLibaomFeatureThresholds()
{ {
const int width = 160; const int width = 160;
const int height = 16; const int height = 16;
@ -172,6 +173,13 @@ public class Av1EncoderFrameTests
// One qualifying block is exactly ten percent of this frame, and the reference threshold is strict. // One qualifying block is exactly ten percent of this frame, and the reference threshold is strict.
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame)); Assert.False(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame));
Av1ScreenContentDetector.Detect(
byteFrame.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
for (int row = 0; row < height; row++) for (int row = 0; row < height; row++)
{ {
Span<byte> samples = byteFrame.Frame.View.GetLumaRowSpan(row); Span<byte> samples = byteFrame.Frame.View.GetLumaRowSpan(row);
@ -182,6 +190,13 @@ public class Av1EncoderFrameTests
} }
Assert.True(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame)); Assert.True(Av1ScreenContentDetector.IsPaletteLikely(byteFrame.Frame));
Av1ScreenContentDetector.Detect(
byteFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
using Av1EncoderFrameBuffer<ushort> highBitDepthFrame = new( using Av1EncoderFrameBuffer<ushort> highBitDepthFrame = new(
Configuration.Default, Configuration.Default,
16, 16,
@ -201,6 +216,13 @@ public class Av1EncoderFrameTests
} }
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame)); Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
for (int row = 0; row < 16; row++) for (int row = 0; row < 16; row++)
{ {
Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row); Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row);
@ -208,6 +230,13 @@ public class Av1EncoderFrameTests
} }
Assert.True(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame)); Assert.True(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
for (int row = 0; row < 16; row++) for (int row = 0; row < 16; row++)
{ {
Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row); Span<ushort> samples = highBitDepthFrame.Frame.View.GetLumaRowSpan(row);
@ -218,10 +247,58 @@ public class Av1EncoderFrameTests
} }
Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame)); Assert.False(Av1ScreenContentDetector.IsPaletteLikely(highBitDepthFrame.Frame));
Av1ScreenContentDetector.Detect(
highBitDepthFrame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.False(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
} }
[Fact] [Fact]
public void EncodeActivatesPaletteToolsForScreenContent() public void ScreenContentDetectorMatchesLibaomIntraBlockCopyVarianceThreshold()
{
const int Width = 16;
const int Height = 16;
using Av1EncoderFrameBuffer<byte> frame = new(
Configuration.Default,
Width,
Height,
8,
Av1ColorFormat.Yuv400,
0,
0);
Buffer2DRegion<byte> luma = frame.Frame.View.GetPlane(Av1Plane.Y);
for (int row = 0; row < Height; row++)
{
luma.DangerousGetRowSpan(row).Fill(96);
}
// A single delta of eleven leaves total variance below half a sample after per-pixel rounding.
luma.DangerousGetRowSpan(0)[0] = 107;
Av1ScreenContentDetector.Detect(
frame.Frame,
out bool allowScreenContentTools,
out bool allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.False(allowIntraBlockCopy);
// Raising that delta to twelve crosses the exact integer rounding boundary used by libaom.
luma.DangerousGetRowSpan(0)[0] = 108;
Av1ScreenContentDetector.Detect(
frame.Frame,
out allowScreenContentTools,
out allowIntraBlockCopy);
Assert.True(allowScreenContentTools);
Assert.True(allowIntraBlockCopy);
}
[Fact]
public void EncodeActivatesScreenContentTools()
{ {
const int width = 16; const int width = 16;
const int height = 16; const int height = 16;
@ -253,7 +330,7 @@ public class Av1EncoderFrameTests
obuReader.ReadAll(ref reader, payload.Length, () => tileReader); obuReader.ReadAll(ref reader, payload.Length, () => tileReader);
ObuFrameHeader frameHeader = Assert.IsType<ObuFrameHeader>(obuReader.FrameHeader); ObuFrameHeader frameHeader = Assert.IsType<ObuFrameHeader>(obuReader.FrameHeader);
Assert.True(frameHeader.AllowScreenContentTools); Assert.True(frameHeader.AllowScreenContentTools);
Assert.False(frameHeader.AllowIntraBlockCopy); Assert.True(frameHeader.AllowIntraBlockCopy);
using Av1Decoder decoder = new(Configuration.Default); using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload); using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.Equal(new Size(width, height), decoded.Size); Assert.Equal(new Size(width, height), decoded.Size);
@ -268,6 +345,61 @@ public class Av1EncoderFrameTests
File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-16x16-8b-444-palette.obu"), payload); File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-16x16-8b-444-palette.obu"), payload);
} }
[Fact]
public void EncodeSelectsIntraBlockCopyForRepeatedScreenContent()
{
const int Width = 328;
const int Height = 16;
const ulong Pattern = 0xD6A5_3C97_E18B_4F20UL;
using Image<Rgba32> source = new(Width, Height);
for (int row = 0; row < Height; row++)
{
Span<Rgba32> pixels = source.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row);
for (int column = 0; column < Width; column++)
{
int patternIndex = ((row & 7) * 8) + (column & 7);
pixels[column] = ((Pattern >> patternIndex) & 1) == 0
? new Rgba32(224, 32, 32)
: new Rgba32(32, 32, 224);
}
}
ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv444);
using MemoryStream stream = new();
_ = Av1FrameEncoder.Encode(
Configuration.Default,
source.Frames.RootFrame,
stream,
colorConfig,
qIndex: 37);
byte[] payload = stream.ToArray();
using Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
Assert.NotNull(decoder.FrameHeader);
Assert.True(decoder.FrameHeader.AllowScreenContentTools);
Assert.True(decoder.FrameHeader.AllowIntraBlockCopy);
Assert.NotNull(decoder.FrameInfo);
Av1SuperblockInfo targetSuperblock = decoder.FrameInfo.GetSuperblock(new Point(5, 0));
bool usesIntraBlockCopy = false;
foreach (Av1BlockModeInfo modeInfo in targetSuperblock.GetModeInfos())
{
usesIntraBlockCopy |= modeInfo.UseIntraBlockCopy;
}
Assert.True(usesIntraBlockCopy);
Assert.Equal(new Size(Width, Height), decoded.Size);
string outputDirectory = Path.Combine(
TestEnvironment.ActualOutputDirectoryFullPath,
"Formats",
"Heif",
"Av1");
Directory.CreateDirectory(outputDirectory);
File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-328x16-8b-444-intrabc.obu"), payload);
}
[Fact] [Fact]
public void PrepareSourceConvertsRgba32DirectlyIntoBorderedEightBitPlane() public void PrepareSourceConvertsRgba32DirectlyIntoBorderedEightBitPlane()
{ {

Loading…
Cancel
Save