Implement AV1 filter intra prediction

1 week ago · abe0f70ca9
5 changed files with 244 additions and 8 deletions
--- a/HEIF_IMPLEMENTATION_PLAN.md
+++ b/HEIF_IMPLEMENTATION_PLAN.md
@ -58,7 +58,7 @@ This snapshot pins or classifies the available references and failures; it does
 | --- | --- | --- | --- |
 | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. |
 | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. |
-| `Av1PredictionDecoder` and the scalar DC, directional, Paeth, and smooth intra predictors | AV1 section 7.11.2 intra prediction | libaom `aom_dsp/intrapred.c` and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for neighbor addressing, directional upsampling, Paeth selection, one-axis smooth normalization, and chroma-from-luma row strides. Existing managed scalar predictors remain the implementation base. The WIP rectangular smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. |
+| `Av1PredictionDecoder` and the scalar DC, directional, Paeth, smooth, and filter-intra predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c` and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, and chroma-from-luma row strides. Existing managed scalar predictors remain the implementation base. The WIP rectangular smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. |

 This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped.

--- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs
+++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs
@ -0,0 +1,150 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
+
+internal static class Av1FilterIntraPredictor
+{
+    private const int BufferStride = 33;
+    private const int BufferLength = BufferStride * BufferStride;
+    private const int TapsPerPixel = 7;
+    private const int PixelsPerGroup = 8;
+    private const int TapsPerMode = TapsPerPixel * PixelsPerGroup;
+
+    // AV1 7.11.2.3 defines five sets of eight filters over the same seven
+    // already-reconstructed neighbors. The omitted eighth libaom tap is zero.
+    private static readonly sbyte[] Taps =
+    [
+
+        // DC
+        -6, 10, 0, 0, 0, 12, 0,
+        -5, 2, 10, 0, 0, 9, 0,
+        -3, 1, 1, 10, 0, 7, 0,
+        -3, 1, 1, 2, 10, 5, 0,
+        -4, 6, 0, 0, 0, 2, 12,
+        -3, 2, 6, 0, 0, 2, 9,
+        -3, 2, 2, 6, 0, 2, 7,
+        -3, 1, 2, 2, 6, 3, 5,
+
+        // Vertical
+        -10, 16, 0, 0, 0, 10, 0,
+        -6, 0, 16, 0, 0, 6, 0,
+        -4, 0, 0, 16, 0, 4, 0,
+        -2, 0, 0, 0, 16, 2, 0,
+        -10, 16, 0, 0, 0, 0, 10,
+        -6, 0, 16, 0, 0, 0, 6,
+        -4, 0, 0, 16, 0, 0, 4,
+        -2, 0, 0, 0, 16, 0, 2,
+
+        // Horizontal
+        -8, 8, 0, 0, 0, 16, 0,
+        -8, 0, 8, 0, 0, 16, 0,
+        -8, 0, 0, 8, 0, 16, 0,
+        -8, 0, 0, 0, 8, 16, 0,
+        -4, 4, 0, 0, 0, 0, 16,
+        -4, 0, 4, 0, 0, 0, 16,
+        -4, 0, 0, 4, 0, 0, 16,
+        -4, 0, 0, 0, 4, 0, 16,
+
+        // Directional 157 degrees
+        -2, 8, 0, 0, 0, 10, 0,
+        -1, 3, 8, 0, 0, 6, 0,
+        -1, 2, 3, 8, 0, 4, 0,
+        0, 1, 2, 3, 8, 2, 0,
+        -1, 4, 0, 0, 0, 3, 10,
+        -1, 3, 4, 0, 0, 4, 6,
+        -1, 2, 3, 4, 0, 4, 4,
+        -1, 2, 2, 3, 4, 3, 3,
+
+        // Paeth
+        -12, 14, 0, 0, 0, 14, 0,
+        -10, 0, 14, 0, 0, 12, 0,
+        -9, 0, 0, 14, 0, 11, 0,
+        -8, 0, 0, 0, 14, 10, 0,
+        -10, 12, 0, 0, 0, 0, 14,
+        -9, 1, 12, 0, 0, 0, 12,
+        -8, 0, 0, 12, 0, 1, 11,
+        -7, 0, 0, 1, 12, 1, 9,
+    ];
+
+    internal static void Predict(
+        Span<byte> destination,
+        nuint destinationStride,
+        Av1TransformSize transformSize,
+        Span<byte> above,
+        Span<byte> left,
+        Av1FilterIntraMode mode)
+    {
+        int width = transformSize.GetWidth();
+        int height = transformSize.GetHeight();
+        DebugGuard.MustBeLessThanOrEqualTo(width, 32, nameof(width));
+        DebugGuard.MustBeLessThanOrEqualTo(height, 32, nameof(height));
+        Guard.MustBeGreaterThanOrEqualTo(destinationStride, (nuint)width, nameof(destinationStride));
+        Guard.MustBeSizedAtLeast(destination, (int)destinationStride * height, nameof(destination));
+        Guard.MustBeSizedAtLeast(above, width, nameof(above));
+        Guard.MustBeSizedAtLeast(left, height, nameof(left));
+
+        Span<byte> buffer = stackalloc byte[BufferLength];
+        ref byte bufferRef = ref buffer[0];
+        ref byte aboveRef = ref above[0];
+        ref byte leftRef = ref left[0];
+
+        // Row zero includes the top-left sample followed by the top neighbors.
+        bufferRef = Unsafe.Subtract(ref aboveRef, 1);
+        above[..width].CopyTo(buffer[1..]);
+        for (int row = 0; row < height; row++)
+        {
+            Unsafe.Add(ref bufferRef, (row + 1) * BufferStride) = Unsafe.Add(ref leftRef, row);
+        }
+
+        int modeOffset = (int)mode * TapsPerMode;
+        ref sbyte tapsRef = ref Taps[modeOffset];
+        for (int row = 1; row <= height; row += 2)
+        {
+            for (int column = 1; column <= width; column += 4)
+            {
+                int sourceOffset = ((row - 1) * BufferStride) + column - 1;
+                int p0 = Unsafe.Add(ref bufferRef, sourceOffset);
+                int p1 = Unsafe.Add(ref bufferRef, sourceOffset + 1);
+                int p2 = Unsafe.Add(ref bufferRef, sourceOffset + 2);
+                int p3 = Unsafe.Add(ref bufferRef, sourceOffset + 3);
+                int p4 = Unsafe.Add(ref bufferRef, sourceOffset + 4);
+                int p5 = Unsafe.Add(ref bufferRef, sourceOffset + BufferStride);
+                int p6 = Unsafe.Add(ref bufferRef, sourceOffset + (2 * BufferStride));
+
+                for (int pixel = 0; pixel < PixelsPerGroup; pixel++)
+                {
+                    int tapOffset = pixel * TapsPerPixel;
+                    int prediction =
+                        (Unsafe.Add(ref tapsRef, tapOffset) * p0)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 1) * p1)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 2) * p2)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 3) * p3)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 4) * p4)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 5) * p5)
+                        + (Unsafe.Add(ref tapsRef, tapOffset + 6) * p6);
+
+                    int rowOffset = pixel >> 2;
+                    int columnOffset = pixel & 3;
+                    int destinationOffset = ((row + rowOffset) * BufferStride) + column + columnOffset;
+                    Unsafe.Add(ref bufferRef, destinationOffset) =
+                        (byte)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 4), 0, 255);
+                }
+            }
+        }
+
+        ref byte destinationRef = ref destination[0];
+        for (int row = 0; row < height; row++)
+        {
+            buffer.Slice(((row + 1) * BufferStride) + 1, width).CopyTo(
+                MemoryMarshal.CreateSpan(ref destinationRef, width));
+
+            destinationRef = ref Unsafe.Add(ref destinationRef, destinationStride);
+        }
+    }
+}
--- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs
+++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs
@ -744,24 +744,25 @@ internal class Av1PredictionDecoder

        if (needAboveLeft)
        {
+            ref byte aboveLeft = ref Unsafe.Subtract(ref aboveRow[0], 1);
            if (topPixelCount > 0 && leftPixelCount > 0)
            {
-                aboveRow[-1] = aboveNeighbor[-1];
+                aboveLeft = Unsafe.Subtract(ref aboveNeighbor[0], 1);
            }
            else if (topPixelCount > 0)
            {
-                aboveRow[-1] = aboveNeighbor[0];
+                aboveLeft = aboveNeighbor[0];
            }
            else if (leftPixelCount > 0)
            {
-                aboveRow[-1] = leftNeighbor[0];
+                aboveLeft = leftNeighbor[0];
            }
            else
            {
-                aboveRow[-1] = 128;
+                aboveLeft = 128;
            }

-            leftColumn[-1] = aboveRow[-1];
+            Unsafe.Subtract(ref leftColumn[0], 1) = aboveLeft;
        }

        if (useFilterIntra)
@ -857,7 +858,7 @@ internal class Av1PredictionDecoder
        input[count + 2] = buffer[count - 1];

        // interpolate half-sample edge positions
-        buffer[-2] = input[0];
+        Unsafe.Subtract(ref buffer[0], 2) = input[0];
        for (int i = 0; i < count; i++)
        {
            int s = -input[i] + (9 * input[i + 1]) + (9 * input[i + 2]) - input[i + 3];
--- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs
+++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs
@ -102,7 +102,8 @@ internal class Av1PredictorFactory
        }
    }

-    internal static void FilterIntraPredictor(Span<byte> destination, nuint destinationStride, Av1TransformSize transformSize, Span<byte> aboveRow, Span<byte> leftColumn, Av1FilterIntraMode filterIntraMode) => throw new NotImplementedException();
+    internal static void FilterIntraPredictor(Span<byte> destination, nuint destinationStride, Av1TransformSize transformSize, Span<byte> aboveRow, Span<byte> leftColumn, Av1FilterIntraMode filterIntraMode)
+        => Av1FilterIntraPredictor.Predict(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode);

    internal static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span<byte> destination, nuint destinationStride, Span<byte> aboveRow, Span<byte> leftColumn)
    {
--- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs
@ -3,6 +3,7 @@

 using SixLabors.ImageSharp.Formats.Heif.Av1;
 using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
+using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
 using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;

 namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
@ -368,6 +369,50 @@ public class Av1PredictorTests
        Assert.Equal(expected, actual);
    }

+    [Theory]
+    [MemberData(nameof(GetFilterIntraPredictions))]
+    public void FilterIntraMatchesLibaomScalarVector(int mode, byte[] expected)
+    {
+        byte[] destination = new byte[16];
+        byte[] aboveData = [17, 30, 70, 110, 150];
+        Span<byte> above = aboveData.AsSpan(1);
+        byte[] left = [40, 80, 120, 160];
+
+        Av1PredictorFactory.FilterIntraPredictor(
+            destination,
+            4,
+            Av1TransformSize.Size4x4,
+            above,
+            left,
+            (Av1FilterIntraMode)mode);
+
+        Assert.Equal(expected, destination);
+    }
+
+    [Theory]
+    [MemberData(nameof(GetFilterIntraTransformSizes))]
+    public void FilterIntraSupportsEveryPermittedTransformSize(int transformSizeIndex)
+    {
+        Av1TransformSize transformSize = (Av1TransformSize)transformSizeIndex;
+        int width = transformSize.GetWidth();
+        int height = transformSize.GetHeight();
+        byte[] destination = new byte[width * height];
+        byte[] aboveData = new byte[width + 1];
+        byte[] left = new byte[height];
+        Array.Fill(aboveData, (byte)73);
+        Array.Fill(left, (byte)73);
+
+        Av1PredictorFactory.FilterIntraPredictor(
+            destination,
+            (nuint)width,
+            transformSize,
+            aboveData.AsSpan(1),
+            left,
+            Av1FilterIntraMode.DC);
+
+        Assert.All(destination, value => Assert.Equal(73, value));
+    }
+
    private static void AssertValue(byte expected, byte actual)
    {
        Assert.NotEqual(0, actual);
@ -399,6 +444,45 @@ public class Av1PredictorTests
        return combinations;
    }

+    public static TheoryData<int, byte[]> GetFilterIntraPredictions() => new()
+    {
+        {
+            (int)Av1FilterIntraMode.DC,
+            [42, 65, 89, 123, 72, 77, 91, 110, 105, 100, 104, 112, 142, 128, 123, 124]
+        },
+        {
+            (int)Av1FilterIntraMode.Vertical,
+            [44, 79, 116, 153, 69, 94, 126, 158, 94, 109, 136, 163, 119, 124, 146, 168]
+        },
+        {
+            (int)Av1FilterIntraMode.Horizontal,
+            [47, 67, 87, 107, 83, 93, 103, 113, 122, 127, 132, 137, 161, 163, 166, 168]
+        },
+        {
+            (int)Av1FilterIntraMode.Directional157,
+            [38, 55, 81, 111, 64, 62, 73, 92, 97, 83, 81, 86, 134, 113, 103, 100]
+        },
+        {
+            (int)Av1FilterIntraMode.Paeth,
+            [49, 81, 114, 148, 82, 105, 132, 159, 117, 132, 153, 174, 152, 159, 177, 190]
+        },
+    };
+
+    public static TheoryData<int> GetFilterIntraTransformSizes()
+    {
+        TheoryData<int> transformSizes = [];
+        for (int i = 0; i < (int)Av1TransformSize.AllSizes; i++)
+        {
+            Av1TransformSize transformSize = (Av1TransformSize)i;
+            if (transformSize.GetWidth() <= 32 && transformSize.GetHeight() <= 32)
+            {
+                transformSizes.Add(i);
+            }
+        }
+
+        return transformSizes;
+    }
+
    private static string GetExpectedDigext(Av1TransformSize size, Av1PredictionMode mode) => size switch
    {
        Av1TransformSize.Size4x4 => Digests4x4[(int)mode],