From abe0f70ca9ff8b752db89610187ed569b71a72c1 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Mon, 24 Aug 2026 17:52:30 +1000 Subject: [PATCH] Implement AV1 filter intra prediction --- HEIF_IMPLEMENTATION_PLAN.md | 2 +- .../Av1/Prediction/Av1FilterIntraPredictor.cs | 150 ++++++++++++++++++ .../Av1/Prediction/Av1PredictionDecoder.cs | 13 +- .../Av1/Prediction/Av1PredictorFactory.cs | 3 +- .../Formats/Heif/Av1/Av1PredictorTests.cs | 84 ++++++++++ 5 files changed, 244 insertions(+), 8 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 043b2909c..07f0d5bef 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -58,7 +58,7 @@ This snapshot pins or classifies the available references and failures; it does | --- | --- | --- | --- | | `Av1YuvConverter.ConvertToRgb`, `ConvertFromRgb`, scalar row conversion, and chroma reconstruction | H.273 formulas 20-31 and the identity, YCgCo, and non-constant-luminance matrix formulas; AV1 section 6.4.2 chroma sample positions | libavif `src/reformat.c` and `src/colr.c` at `092276ce89098ead06db80975173191e5fee1826`; libaom `aom/aom_image.h` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Scalar behavioral oracle for 8-bit full/limited-range conversion. Decode covers monochrome, YUV 4:2:0, 4:2:2, and 4:4:4 with AV1 chroma sample positioning; encode remains YUV 4:4:4 at this snapshot. Later high-bit-depth and SIMD paths must match it. | | `Av1FrameBuffer` high-bit-depth sample layout and `Av1YuvConverter` 10/12-bit output conversion | AV1 section 6.4.1 bit depth and H.273 sample-range scaling | libaom `aom_scale/yv12config.h`, `av1/common/idct.c`, and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f`; libavif `src/avif.c` and `src/reformat.c` at `092276ce89098ead06db80975173191e5fee1826` | Establish two-byte native sample storage with sample-unit strides for 10/12-bit reconstruction and use the same scalar color model at every supported bit depth. | -| `Av1PredictionDecoder` and the scalar DC, directional, Paeth, and smooth intra predictors | AV1 section 7.11.2 intra prediction | libaom `aom_dsp/intrapred.c` and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for neighbor addressing, directional upsampling, Paeth selection, one-axis smooth normalization, and chroma-from-luma row strides. Existing managed scalar predictors remain the implementation base. The WIP rectangular smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | +| `Av1PredictionDecoder` and the scalar DC, directional, Paeth, smooth, and filter-intra predictors | AV1 sections 7.11.2 and 7.11.2.3 intra prediction | libaom `aom_dsp/intrapred.c` and `av1/common/reconintra.c` at `03087864cf4bea6abb0d28f95cf7843511413d8f` | Behavioral oracle for neighbor addressing, directional upsampling, Paeth selection, smooth normalization, filter-intra taps, and chroma-from-luma row strides. Existing managed scalar predictors remain the implementation base. The WIP rectangular smooth digest expectations encode width/height-swapped weights and must be replaced only from an independently generated oracle, not regenerated from this implementation. | This table is intentionally incomplete. Add a row before each additional AV1 or HEVC algorithm is ported or materially reshaped. diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs new file mode 100644 index 000000000..243d0e822 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.cs @@ -0,0 +1,150 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; +using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static class Av1FilterIntraPredictor +{ + private const int BufferStride = 33; + private const int BufferLength = BufferStride * BufferStride; + private const int TapsPerPixel = 7; + private const int PixelsPerGroup = 8; + private const int TapsPerMode = TapsPerPixel * PixelsPerGroup; + + // AV1 7.11.2.3 defines five sets of eight filters over the same seven + // already-reconstructed neighbors. The omitted eighth libaom tap is zero. + private static readonly sbyte[] Taps = + [ + + // DC + -6, 10, 0, 0, 0, 12, 0, + -5, 2, 10, 0, 0, 9, 0, + -3, 1, 1, 10, 0, 7, 0, + -3, 1, 1, 2, 10, 5, 0, + -4, 6, 0, 0, 0, 2, 12, + -3, 2, 6, 0, 0, 2, 9, + -3, 2, 2, 6, 0, 2, 7, + -3, 1, 2, 2, 6, 3, 5, + + // Vertical + -10, 16, 0, 0, 0, 10, 0, + -6, 0, 16, 0, 0, 6, 0, + -4, 0, 0, 16, 0, 4, 0, + -2, 0, 0, 0, 16, 2, 0, + -10, 16, 0, 0, 0, 0, 10, + -6, 0, 16, 0, 0, 0, 6, + -4, 0, 0, 16, 0, 0, 4, + -2, 0, 0, 0, 16, 0, 2, + + // Horizontal + -8, 8, 0, 0, 0, 16, 0, + -8, 0, 8, 0, 0, 16, 0, + -8, 0, 0, 8, 0, 16, 0, + -8, 0, 0, 0, 8, 16, 0, + -4, 4, 0, 0, 0, 0, 16, + -4, 0, 4, 0, 0, 0, 16, + -4, 0, 0, 4, 0, 0, 16, + -4, 0, 0, 0, 4, 0, 16, + + // Directional 157 degrees + -2, 8, 0, 0, 0, 10, 0, + -1, 3, 8, 0, 0, 6, 0, + -1, 2, 3, 8, 0, 4, 0, + 0, 1, 2, 3, 8, 2, 0, + -1, 4, 0, 0, 0, 3, 10, + -1, 3, 4, 0, 0, 4, 6, + -1, 2, 3, 4, 0, 4, 4, + -1, 2, 2, 3, 4, 3, 3, + + // Paeth + -12, 14, 0, 0, 0, 14, 0, + -10, 0, 14, 0, 0, 12, 0, + -9, 0, 0, 14, 0, 11, 0, + -8, 0, 0, 0, 14, 10, 0, + -10, 12, 0, 0, 0, 0, 14, + -9, 1, 12, 0, 0, 0, 12, + -8, 0, 0, 12, 0, 1, 11, + -7, 0, 0, 1, 12, 1, 9, + ]; + + internal static void Predict( + Span destination, + nuint destinationStride, + Av1TransformSize transformSize, + Span above, + Span left, + Av1FilterIntraMode mode) + { + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + DebugGuard.MustBeLessThanOrEqualTo(width, 32, nameof(width)); + DebugGuard.MustBeLessThanOrEqualTo(height, 32, nameof(height)); + Guard.MustBeGreaterThanOrEqualTo(destinationStride, (nuint)width, nameof(destinationStride)); + Guard.MustBeSizedAtLeast(destination, (int)destinationStride * height, nameof(destination)); + Guard.MustBeSizedAtLeast(above, width, nameof(above)); + Guard.MustBeSizedAtLeast(left, height, nameof(left)); + + Span buffer = stackalloc byte[BufferLength]; + ref byte bufferRef = ref buffer[0]; + ref byte aboveRef = ref above[0]; + ref byte leftRef = ref left[0]; + + // Row zero includes the top-left sample followed by the top neighbors. + bufferRef = Unsafe.Subtract(ref aboveRef, 1); + above[..width].CopyTo(buffer[1..]); + for (int row = 0; row < height; row++) + { + Unsafe.Add(ref bufferRef, (row + 1) * BufferStride) = Unsafe.Add(ref leftRef, row); + } + + int modeOffset = (int)mode * TapsPerMode; + ref sbyte tapsRef = ref Taps[modeOffset]; + for (int row = 1; row <= height; row += 2) + { + for (int column = 1; column <= width; column += 4) + { + int sourceOffset = ((row - 1) * BufferStride) + column - 1; + int p0 = Unsafe.Add(ref bufferRef, sourceOffset); + int p1 = Unsafe.Add(ref bufferRef, sourceOffset + 1); + int p2 = Unsafe.Add(ref bufferRef, sourceOffset + 2); + int p3 = Unsafe.Add(ref bufferRef, sourceOffset + 3); + int p4 = Unsafe.Add(ref bufferRef, sourceOffset + 4); + int p5 = Unsafe.Add(ref bufferRef, sourceOffset + BufferStride); + int p6 = Unsafe.Add(ref bufferRef, sourceOffset + (2 * BufferStride)); + + for (int pixel = 0; pixel < PixelsPerGroup; pixel++) + { + int tapOffset = pixel * TapsPerPixel; + int prediction = + (Unsafe.Add(ref tapsRef, tapOffset) * p0) + + (Unsafe.Add(ref tapsRef, tapOffset + 1) * p1) + + (Unsafe.Add(ref tapsRef, tapOffset + 2) * p2) + + (Unsafe.Add(ref tapsRef, tapOffset + 3) * p3) + + (Unsafe.Add(ref tapsRef, tapOffset + 4) * p4) + + (Unsafe.Add(ref tapsRef, tapOffset + 5) * p5) + + (Unsafe.Add(ref tapsRef, tapOffset + 6) * p6); + + int rowOffset = pixel >> 2; + int columnOffset = pixel & 3; + int destinationOffset = ((row + rowOffset) * BufferStride) + column + columnOffset; + Unsafe.Add(ref bufferRef, destinationOffset) = + (byte)Av1Math.Clamp(Av1Math.RoundPowerOf2(prediction, 4), 0, 255); + } + } + } + + ref byte destinationRef = ref destination[0]; + for (int row = 0; row < height; row++) + { + buffer.Slice(((row + 1) * BufferStride) + 1, width).CopyTo( + MemoryMarshal.CreateSpan(ref destinationRef, width)); + + destinationRef = ref Unsafe.Add(ref destinationRef, destinationStride); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index 68cd1e608..70803a4c6 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -744,24 +744,25 @@ internal class Av1PredictionDecoder if (needAboveLeft) { + ref byte aboveLeft = ref Unsafe.Subtract(ref aboveRow[0], 1); if (topPixelCount > 0 && leftPixelCount > 0) { - aboveRow[-1] = aboveNeighbor[-1]; + aboveLeft = Unsafe.Subtract(ref aboveNeighbor[0], 1); } else if (topPixelCount > 0) { - aboveRow[-1] = aboveNeighbor[0]; + aboveLeft = aboveNeighbor[0]; } else if (leftPixelCount > 0) { - aboveRow[-1] = leftNeighbor[0]; + aboveLeft = leftNeighbor[0]; } else { - aboveRow[-1] = 128; + aboveLeft = 128; } - leftColumn[-1] = aboveRow[-1]; + Unsafe.Subtract(ref leftColumn[0], 1) = aboveLeft; } if (useFilterIntra) @@ -857,7 +858,7 @@ internal class Av1PredictionDecoder input[count + 2] = buffer[count - 1]; // interpolate half-sample edge positions - buffer[-2] = input[0]; + Unsafe.Subtract(ref buffer[0], 2) = input[0]; for (int i = 0; i < count; i++) { int s = -input[i] + (9 * input[i + 1]) + (9 * input[i + 2]) - input[i + 3]; diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs index f1623a26d..c4c0d97d9 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictorFactory.cs @@ -102,7 +102,8 @@ internal class Av1PredictorFactory } } - internal static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, Av1FilterIntraMode filterIntraMode) => throw new NotImplementedException(); + internal static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span aboveRow, Span leftColumn, Av1FilterIntraMode filterIntraMode) + => Av1FilterIntraPredictor.Predict(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode); internal static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span aboveRow, Span leftColumn) { diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs index 2b21a35fd..186b47981 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs @@ -3,6 +3,7 @@ using SixLabors.ImageSharp.Formats.Heif.Av1; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; +using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; @@ -368,6 +369,50 @@ public class Av1PredictorTests Assert.Equal(expected, actual); } + [Theory] + [MemberData(nameof(GetFilterIntraPredictions))] + public void FilterIntraMatchesLibaomScalarVector(int mode, byte[] expected) + { + byte[] destination = new byte[16]; + byte[] aboveData = [17, 30, 70, 110, 150]; + Span above = aboveData.AsSpan(1); + byte[] left = [40, 80, 120, 160]; + + Av1PredictorFactory.FilterIntraPredictor( + destination, + 4, + Av1TransformSize.Size4x4, + above, + left, + (Av1FilterIntraMode)mode); + + Assert.Equal(expected, destination); + } + + [Theory] + [MemberData(nameof(GetFilterIntraTransformSizes))] + public void FilterIntraSupportsEveryPermittedTransformSize(int transformSizeIndex) + { + Av1TransformSize transformSize = (Av1TransformSize)transformSizeIndex; + int width = transformSize.GetWidth(); + int height = transformSize.GetHeight(); + byte[] destination = new byte[width * height]; + byte[] aboveData = new byte[width + 1]; + byte[] left = new byte[height]; + Array.Fill(aboveData, (byte)73); + Array.Fill(left, (byte)73); + + Av1PredictorFactory.FilterIntraPredictor( + destination, + (nuint)width, + transformSize, + aboveData.AsSpan(1), + left, + Av1FilterIntraMode.DC); + + Assert.All(destination, value => Assert.Equal(73, value)); + } + private static void AssertValue(byte expected, byte actual) { Assert.NotEqual(0, actual); @@ -399,6 +444,45 @@ public class Av1PredictorTests return combinations; } + public static TheoryData GetFilterIntraPredictions() => new() + { + { + (int)Av1FilterIntraMode.DC, + [42, 65, 89, 123, 72, 77, 91, 110, 105, 100, 104, 112, 142, 128, 123, 124] + }, + { + (int)Av1FilterIntraMode.Vertical, + [44, 79, 116, 153, 69, 94, 126, 158, 94, 109, 136, 163, 119, 124, 146, 168] + }, + { + (int)Av1FilterIntraMode.Horizontal, + [47, 67, 87, 107, 83, 93, 103, 113, 122, 127, 132, 137, 161, 163, 166, 168] + }, + { + (int)Av1FilterIntraMode.Directional157, + [38, 55, 81, 111, 64, 62, 73, 92, 97, 83, 81, 86, 134, 113, 103, 100] + }, + { + (int)Av1FilterIntraMode.Paeth, + [49, 81, 114, 148, 82, 105, 132, 159, 117, 132, 153, 174, 152, 159, 177, 190] + }, + }; + + public static TheoryData GetFilterIntraTransformSizes() + { + TheoryData transformSizes = []; + for (int i = 0; i < (int)Av1TransformSize.AllSizes; i++) + { + Av1TransformSize transformSize = (Av1TransformSize)i; + if (transformSize.GetWidth() <= 32 && transformSize.GetHeight() <= 32) + { + transformSizes.Add(i); + } + } + + return transformSizes; + } + private static string GetExpectedDigext(Av1TransformSize size, Av1PredictionMode mode) => size switch { Av1TransformSize.Size4x4 => Digests4x4[(int)mode],