Honor AV1 encoding effort

src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs

@@ -26,15 +26,17 @@ internal static class Av1FrameEncoder
     /// <param name="stream">The destination receiving the complete AV1 item payload.</param>
     /// <param name="colorConfig">The resolved native color and precision configuration.</param>
     /// <param name="qIndex">The frame quantizer index.</param>
+    /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
     /// <returns>The sequence header describing the encoded payload.</returns>
     public static ObuSequenceHeader Encode<TPixel>(
         Configuration configuration,
         ImageFrame<TPixel> image,
         Stream stream,
         ObuColorConfig colorConfig,
-        int qIndex)
+        int qIndex,
+        int effort = 5)
         where TPixel : unmanaged, IPixel<TPixel>
-        => Encode(configuration, image, stream, colorConfig, qIndex, false);
+        => Encode(configuration, image, stream, colorConfig, qIndex, effort, false);
 
     /// <summary>
     /// Encodes one packed alpha channel as a reduced-still-picture monochrome AV1 frame.
@@ -45,15 +47,17 @@ internal static class Av1FrameEncoder
     /// <param name="stream">The destination receiving the complete AV1 item payload.</param>
     /// <param name="colorConfig">The resolved monochrome precision configuration.</param>
     /// <param name="qIndex">The frame quantizer index.</param>
+    /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
     /// <returns>The sequence header describing the encoded payload.</returns>
     public static ObuSequenceHeader EncodeAlpha<TPixel>(
         Configuration configuration,
         ImageFrame<TPixel> image,
         Stream stream,
         ObuColorConfig colorConfig,
-        int qIndex)
+        int qIndex,
+        int effort = 5)
         where TPixel : unmanaged, IPixel<TPixel>
-        => Encode(configuration, image, stream, colorConfig, qIndex, true);
+        => Encode(configuration, image, stream, colorConfig, qIndex, effort, true);
 
     private static ObuSequenceHeader Encode<TPixel>(
         Configuration configuration,
@@ -61,6 +65,7 @@ internal static class Av1FrameEncoder
         Stream stream,
         ObuColorConfig colorConfig,
         int qIndex,
+        int effort,
         bool encodeAlpha)
         where TPixel : unmanaged, IPixel<TPixel>
     {
@@ -87,7 +92,7 @@ internal static class Av1FrameEncoder
             Use128x128Superblock = false,
             ForceScreenContentTools = 2,
             ForceIntegerMotionVector = 2,
-            EnableFilterIntra = true,
+            EnableFilterIntra = effort >= 4,
             EnableIntraEdgeFilter = false,
             EnableSuperResolution = false,
             EnableCdef = false,
@@ -148,11 +153,11 @@ internal static class Av1FrameEncoder
         int initialTileSize = checked((int)Math.Max(8192L, (sampleCount * sampleSize * 5) / 2));
         if (colorConfig.BitDepth == Av1BitDepth.EightBit)
         {
-            EncodeByte(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, encodeAlpha);
+            EncodeByte(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, effort, encodeAlpha);
         }
         else
         {
-            EncodeHighBitDepth(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, encodeAlpha);
+            EncodeHighBitDepth(configuration, image, stream, sequenceHeader, frameHeader, colorFormat, initialTileSize, effort, encodeAlpha);
         }
 
         return sequenceHeader;
@@ -198,6 +203,7 @@ internal static class Av1FrameEncoder
         ObuFrameHeader frameHeader,
         Av1ColorFormat colorFormat,
         int initialTileSize,
+        int effort,
         bool encodeAlpha)
         where TPixel : unmanaged, IPixel<TPixel>
     {
@@ -219,7 +225,7 @@ internal static class Av1FrameEncoder
             chromaPositionX: 1,
             chromaPositionY: 1);
 
-        Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, encodeAlpha);
+        Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, effort, encodeAlpha);
     }
 
     private static void EncodeHighBitDepth<TPixel>(
@@ -230,6 +236,7 @@ internal static class Av1FrameEncoder
         ObuFrameHeader frameHeader,
         Av1ColorFormat colorFormat,
         int initialTileSize,
+        int effort,
         bool encodeAlpha)
         where TPixel : unmanaged, IPixel<TPixel>
     {
@@ -252,7 +259,7 @@ internal static class Av1FrameEncoder
             chromaPositionX: 1,
             chromaPositionY: 1);
 
-        Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, encodeAlpha);
+        Encode(configuration, image, stream, sequenceHeader, frameHeader, source, reconstruction, initialTileSize, effort, encodeAlpha);
     }
 
     private static void Encode<TPixel>(
@@ -264,6 +271,7 @@ internal static class Av1FrameEncoder
         Av1EncoderFrameBuffer<byte> source,
         Av1EncoderFrameBuffer<byte> reconstruction,
         int initialTileSize,
+        int effort,
         bool encodeAlpha)
         where TPixel : unmanaged, IPixel<TPixel>
     {
@@ -274,10 +282,16 @@ internal static class Av1FrameEncoder
             sequenceHeader.ColorConfig,
             encodeAlpha);
 
-        Av1ScreenContentDetector.Detect(
+        bool allowScreenContentTools = false;
-            source.Frame,
+        bool allowIntraBlockCopy = false;
-            out bool allowScreenContentTools,
+        if (effort >= 5)
-            out bool allowIntraBlockCopy);
+        {
+            // Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan.
+            Av1ScreenContentDetector.Detect(
+                source.Frame,
+                out allowScreenContentTools,
+                out allowIntraBlockCopy);
+        }
 
         frameHeader.AllowScreenContentTools = allowScreenContentTools;
         frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
@@ -304,7 +318,8 @@ internal static class Av1FrameEncoder
             coefficients,
             superblockWorkspace,
             blockWorkspace,
-            initialTileSize);
+            initialTileSize,
+            effort);
 
         ObuWriter writer = new();
         writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter);
@@ -319,6 +334,7 @@ internal static class Av1FrameEncoder
         Av1EncoderFrameBuffer<ushort> source,
         Av1EncoderFrameBuffer<ushort> reconstruction,
         int initialTileSize,
+        int effort,
         bool encodeAlpha)
         where TPixel : unmanaged, IPixel<TPixel>
     {
@@ -329,10 +345,16 @@ internal static class Av1FrameEncoder
             sequenceHeader.ColorConfig,
             encodeAlpha);
 
-        Av1ScreenContentDetector.Detect(
+        bool allowScreenContentTools = false;
-            source.Frame,
+        bool allowIntraBlockCopy = false;
-            out bool allowScreenContentTools,
+        if (effort >= 5)
-            out bool allowIntraBlockCopy);
+        {
+            // Lower effort levels omit palette and intra-block-copy searches, so they do not need the whole-frame suitability scan.
+            Av1ScreenContentDetector.Detect(
+                source.Frame,
+                out allowScreenContentTools,
+                out allowIntraBlockCopy);
+        }
 
         frameHeader.AllowScreenContentTools = allowScreenContentTools;
         frameHeader.AllowIntraBlockCopy = allowIntraBlockCopy;
@@ -359,7 +381,8 @@ internal static class Av1FrameEncoder
             coefficients,
             superblockWorkspace,
             blockWorkspace,
-            initialTileSize);
+            initialTileSize,
+            effort);
 
         ObuWriter writer = new();
         writer.WriteAll(configuration, stream, sequenceHeader, frameHeader, tileWriter);

src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs

@@ -164,7 +164,15 @@ internal static partial class Av1IntraSuperblockEncoder
             int baseModeCount = ChromaModeSearchOrder.Length;
             int deltaCount = AngleDeltaSearchOrder.Length;
             int directionalModeCount = (int)Av1ChromaPredictionMode.Directional67Degrees - (int)Av1ChromaPredictionMode.Vertical + 1;
-            int candidateCount = baseModeCount + (directionalModeCount * deltaCount);
+
+            // Effort zero evaluates DC only, effort one adds every zero-angle mode, and higher levels add all directional adjustments.
+            int candidateCount = this.effort switch
+            {
+                0 => 1,
+                1 => baseModeCount,
+                _ => baseModeCount + (directionalModeCount * deltaCount)
+            };
+
             bool hasLumaPalette = paletteInfo.PaletteSizes[0] != 0;
             int paletteDisabledCost = this.picture.Parent.FrameHeader.AllowScreenContentTools
                 ? writer.GetPaletteUvModeCost(false, hasLumaPalette)
@@ -249,7 +257,7 @@ internal static partial class Av1IntraSuperblockEncoder
                 colorConfig.SubSamplingX,
                 colorConfig.SubSamplingY);
 
-            if (chromaFromLumaAllowed)
+            if (this.effort >= 4 && chromaFromLumaAllowed)
             {
                 Span<short> lumaQ3 = stackalloc short[Av1ChromaFromLumaContext.BufferLine * 8];
                 TOperator.PrepareChromaFromLuma(
@@ -441,7 +449,8 @@ internal static partial class Av1IntraSuperblockEncoder
                 }
             }
 
-            if (this.picture.Parent.FrameHeader.AllowScreenContentTools &&
+            if (this.effort >= 5 &&
+                this.picture.Parent.FrameHeader.AllowScreenContentTools &&
                 this.SelectChromaPalette(
                     writer,
                     macroBlock,

src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs

@@ -117,6 +117,7 @@ internal static partial class Av1IntraSuperblockEncoder
         private readonly ObuQuantizationParameters quantization;
         private readonly Av1BitDepth bitDepth;
         private readonly int rateMultiplier;
+        private readonly int effort;
         private int codedAreaLuma;
         private int codedAreaChroma;
 
@@ -129,13 +130,15 @@ internal static partial class Av1IntraSuperblockEncoder
         /// <param name="superblock">The current superblock.</param>
         /// <param name="coefficientBuffer">The frame-owned quantized coefficient and transform state.</param>
         /// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
+        /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
         public ModeDecision(
             Av1EncoderFrame<TSample> source,
             Av1EncoderFrame<TSample> reconstruction,
             Av1PictureControlSet picture,
             Av1Superblock superblock,
             Av1EncoderCoefficientBuffer coefficientBuffer,
-            Av1EncoderBlockWorkspace blockWorkspace)
+            Av1EncoderBlockWorkspace blockWorkspace,
+            int effort = 5)
         {
             this.source = source.CodedView;
             this.reconstruction = reconstruction.CodedView;
@@ -146,6 +149,7 @@ internal static partial class Av1IntraSuperblockEncoder
             this.quantization = picture.Parent.FrameHeader.QuantizationParameters;
             this.bitDepth = picture.Sequence.SequenceHeader.ColorConfig.BitDepth;
             this.rateMultiplier = Av1RateDistortion.GetKeyFrameRateMultiplier(this.quantization.QIndex[0], this.bitDepth);
+            this.effort = effort;
             this.codedAreaLuma = 0;
             this.codedAreaChroma = 0;
         }
@@ -528,7 +532,15 @@ internal static partial class Av1IntraSuperblockEncoder
             int baseModeCount = LumaModeSearchOrder.Length;
             int deltaCount = AngleDeltaSearchOrder.Length;
             int directionalModeCount = (int)Av1PredictionMode.Directional67Degrees - (int)Av1PredictionMode.Vertical + 1;
-            int candidateCount = baseModeCount + (directionalModeCount * deltaCount);
+
+            // Effort zero evaluates DC only, effort one adds every zero-angle mode, and higher levels add all directional adjustments.
+            int candidateCount = this.effort switch
+            {
+                0 => 1,
+                1 => baseModeCount,
+                _ => baseModeCount + (directionalModeCount * deltaCount)
+            };
+
             bool useReducedTransformSet = this.picture.Parent.FrameHeader.UseReducedTransformSet;
             Av1TransformSetType transformSetType = Av1SymbolContextHelper.GetExtendedTransformSetType(
                 TransformSize,
@@ -594,11 +606,11 @@ internal static partial class Av1IntraSuperblockEncoder
                 }
             }
 
-            // Mode selection uses the mode-derived default transform, then the winning mode alone pays for
+            // Lower effort levels retain the mode-derived default transform. Higher levels refine only the winning
-            // an exhaustive transform refinement. This preserves reference tie order without a 61-by-7 search.
+            // mode across the legal transform set, preserving search order without evaluating every mode-transform pair.
-            long bestTransformCost = long.MaxValue;
+            long bestTransformCost = bestCost;
             for (Av1TransformType transformType = Av1TransformType.DctDct;
-                transformType < Av1TransformType.AllTransformTypes;
+                this.effort >= 3 && transformType < Av1TransformType.AllTransformTypes;
                 transformType++)
             {
                 if (!transformType.IsExtendedSetUsed(transformSetType))
@@ -641,7 +653,7 @@ internal static partial class Av1IntraSuperblockEncoder
                 }
             }
 
-            if (this.picture.Sequence.SequenceHeader.EnableFilterIntra)
+            if (this.effort >= 4 && this.picture.Sequence.SequenceHeader.EnableFilterIntra)
             {
                 Span<TSample> filterPrediction = stackalloc TSample[SampleCount];
                 Span<short> filterResidual = stackalloc short[SampleCount];
@@ -710,7 +722,8 @@ internal static partial class Av1IntraSuperblockEncoder
                 }
             }
 
-            if (this.picture.Parent.FrameHeader.AllowScreenContentTools &&
+            if (this.effort >= 5 &&
+                this.picture.Parent.FrameHeader.AllowScreenContentTools &&
                 this.SelectLumaPalette(
                     writer,
                     macroBlock,

src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraTileWriter.cs

@@ -28,6 +28,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
     /// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
     /// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
     /// <param name="initialSize">The estimated encoded tile size in bytes.</param>
+    /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
     public Av1IntraTileWriter(
         Configuration configuration,
         Av1EncoderFrame<byte> source,
@@ -36,7 +37,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
         Av1EncoderCoefficientBuffer coefficientBuffer,
         Av1EncoderSuperblockWorkspace superblockWorkspace,
         Av1EncoderBlockWorkspace blockWorkspace,
-        int initialSize)
+        int initialSize,
+        int effort = 5)
     {
         this.tileData = Encode<byte, Av1IntraSuperblockEncoder.ByteOperator>(
             configuration,
@@ -46,6 +48,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
             coefficientBuffer,
             superblockWorkspace,
             blockWorkspace,
+            effort,
             initialSize,
             out this.tileDataLength);
     }
@@ -61,6 +64,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
     /// <param name="superblockWorkspace">The reusable partition and final-block decision workspace.</param>
     /// <param name="blockWorkspace">The reusable block arithmetic workspace.</param>
     /// <param name="initialSize">The estimated encoded tile size in bytes.</param>
+    /// <param name="effort">The mode-search effort in the inclusive range zero through ten.</param>
     public Av1IntraTileWriter(
         Configuration configuration,
         Av1EncoderFrame<ushort> source,
@@ -69,7 +73,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
         Av1EncoderCoefficientBuffer coefficientBuffer,
         Av1EncoderSuperblockWorkspace superblockWorkspace,
         Av1EncoderBlockWorkspace blockWorkspace,
-        int initialSize)
+        int initialSize,
+        int effort = 5)
     {
         this.tileData = Encode<ushort, Av1IntraSuperblockEncoder.UInt16Operator>(
             configuration,
@@ -79,6 +84,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
             coefficientBuffer,
             superblockWorkspace,
             blockWorkspace,
+            effort,
             initialSize,
             out this.tileDataLength);
     }
@@ -107,6 +113,7 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
         Av1EncoderCoefficientBuffer coefficientBuffer,
         Av1EncoderSuperblockWorkspace superblockWorkspace,
         Av1EncoderBlockWorkspace blockWorkspace,
+        int effort,
         int initialSize,
         out int tileDataLength)
         where TSample : unmanaged
@@ -171,7 +178,8 @@ internal sealed partial class Av1IntraTileWriter : IAv1TileWriter, IDisposable
                     picture,
                     superblock,
                     coefficientBuffer,
-                    blockWorkspace);
+                    blockWorkspace,
+                    effort);
 
                 Av1TileWriter.WriteSuperblock(
                     picture,

tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs

@@ -5,6 +5,7 @@ using SixLabors.ImageSharp.Formats;
 using SixLabors.ImageSharp.Formats.Heif.Av1;
 using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
 using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
 using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
 using SixLabors.ImageSharp.Formats.Heif.Components;
 using SixLabors.ImageSharp.Formats.Heif.Components.Alpha;
@@ -482,6 +483,91 @@ public class Av1EncoderFrameTests
         File.WriteAllBytes(Path.Combine(outputDirectory, "encoder-frame-16x16-8b-444-palette.obu"), payload);
     }
 
+    [Theory]
+    [InlineData(0, false, false)]
+    [InlineData(1, false, false)]
+    [InlineData(2, false, false)]
+    [InlineData(3, false, false)]
+    [InlineData(4, true, false)]
+    [InlineData(5, true, true)]
+    public void EncodeEffortControlsSearchFeatures(int effort, bool enableFilterIntra, bool enableScreenContentTools)
+    {
+        const int width = 16;
+        const int height = 16;
+        using Image<Rgba32> source = new(width, height);
+        for (int row = 0; row < height; row++)
+        {
+            Span<Rgba32> pixels = source.Frames.RootFrame.PixelBuffer.DangerousGetRowSpan(row);
+            for (int column = 0; column < width; column++)
+            {
+                pixels[column] = (((column >> 2) + (row >> 2)) & 1) == 0
+                    ? new Rgba32(224, 32, 32)
+                    : new Rgba32(32, 32, 224);
+            }
+        }
+
+        ObuColorConfig colorConfig = CreateColorConfig(Av1BitDepth.EightBit, Av1ColorFormat.Yuv444);
+        using MemoryStream stream = new();
+        _ = Av1FrameEncoder.Encode(
+            Configuration.Default,
+            source.Frames.RootFrame,
+            stream,
+            colorConfig,
+            qIndex: 37,
+            effort);
+
+        byte[] payload = stream.ToArray();
+        using Av1Decoder decoder = new(Configuration.Default);
+        using Image<Rgba32> decoded = decoder.Decode<Rgba32>(payload);
+        ObuSequenceHeader sequenceHeader = Assert.IsType<ObuSequenceHeader>(decoder.SequenceHeader);
+        ObuFrameHeader frameHeader = Assert.IsType<ObuFrameHeader>(decoder.FrameHeader);
+        Av1FrameInfo frameInfo = Assert.IsType<Av1FrameInfo>(decoder.FrameInfo);
+        Assert.Equal(enableFilterIntra, sequenceHeader.EnableFilterIntra);
+        Assert.Equal(enableScreenContentTools, frameHeader.AllowScreenContentTools);
+        Assert.Equal(enableScreenContentTools, frameHeader.AllowIntraBlockCopy);
+        Assert.Equal(new Size(width, height), decoded.Size);
+
+        int modeCount = 0;
+        foreach (Av1BlockModeInfo modeInfo in frameInfo.GetSuperblock(Point.Empty).GetModeInfos())
+        {
+            modeCount++;
+            if (effort == 0)
+            {
+                Assert.Equal(Av1PredictionMode.DC, modeInfo.YMode);
+                Assert.Equal(Av1ChromaPredictionMode.DC, modeInfo.UvMode);
+            }
+
+            if (effort <= 1)
+            {
+                Assert.Equal(0, modeInfo.GetAngleDelta(Av1Plane.Y));
+                Assert.Equal(0, modeInfo.GetAngleDelta(Av1Plane.U));
+            }
+
+            if (effort < 4)
+            {
+                Assert.False(modeInfo.UseFilterIntra);
+            }
+
+            if (effort < 5)
+            {
+                Assert.False(modeInfo.UseIntraBlockCopy);
+                Assert.Equal(0, modeInfo.GetPaletteSize(Av1Plane.Y));
+                Assert.Equal(0, modeInfo.GetPaletteSize(Av1Plane.U));
+            }
+        }
+
+        Assert.NotEqual(0, modeCount);
+
+        string outputDirectory = Path.Combine(
+            TestEnvironment.ActualOutputDirectoryFullPath,
+            "Formats",
+            "Heif",
+            "Av1");
+
+        Directory.CreateDirectory(outputDirectory);
+        File.WriteAllBytes(Path.Combine(outputDirectory, $"encoder-frame-16x16-8b-444-effort-{effort}.obu"), payload);
+    }
+
     [Fact]
     public void EncodeSelectsIntraBlockCopyForRepeatedScreenContent()
     {