Browse Source

Fix AV1 still decode lifecycle

pull/2633/head
James Jackson-South 2 months ago
parent
commit
98ab882e77
  1. 7
      HEIF_IMPLEMENTATION_PLAN.md
  2. 42
      src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs
  3. 11
      src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs
  4. 15
      tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs

7
HEIF_IMPLEMENTATION_PLAN.md

@ -95,10 +95,9 @@ This assessment is based on the current source after the upstream ImageSharp mer
### AV1 decoder ### AV1 decoder
- `Av1Decoder.ReadTile()` checks `tileReader` in the branch where it is known to be null, so the first tile cannot reach reconstruction. - The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one frame, and it disposes the reconstruction planes after pixel conversion. Reference-frame, `show_existing_frame`, and multi-frame ownership remain incomplete.
- Decoder state is allocated or replaced at multiple points, making parsed tile state, reconstructed frame state, and ownership unclear.
- The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. - The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements.
- Loop restoration, filter intra prediction, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches. - Loop restoration, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches.
- The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active prediction and block reconstruction path is still limited to 8-bit samples and must be connected to the existing high-bit-depth inverse-transform core. - The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active prediction and block reconstruction path is still limited to 8-bit samples and must be connected to the existing high-bit-depth inverse-transform core.
- `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. - `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete.
- The inverse-transform path allocates arrays in a per-transform hot path. - The inverse-transform path allocates arrays in a per-transform hot path.
@ -122,7 +121,7 @@ This assessment is based on the current source after the upstream ImageSharp mer
- The repository contains HEIC, HIF, and AVIF assets, but only the legacy JPEG HIF path reaches a full reference-image comparison. - The repository contains HEIC, HIF, and AVIF assets, but only the legacy JPEG HIF path reaches a full reference-image comparison.
- HEVC fixtures are identified but not decoded, and there are no HEVC algorithm tests. - HEVC fixtures are identified but not decoded, and there are no HEVC algorithm tests.
- The strongest AV1 integration test only verifies that the first tile produces non-zero luma data. - The strongest AV1 integration test now drives the single-still decoder through tile parsing, reconstruction, and pixel conversion, but only verifies non-zero output rather than independent reference pixels.
- Several full-image, inverse-transform, entropy, and frame-header cases are disabled or commented out. - Several full-image, inverse-transform, entropy, and frame-header cases are disabled or commented out.
- Existing bitstream, predictor, transform, and entropy unit tests are useful foundations, but many compare two in-tree implementations with the same assumptions. - Existing bitstream, predictor, transform, and entropy unit tests are useful foundations, but many compare two in-tree implementations with the same assumptions.
- There is no decode matrix covering bit depth, subsampling, range, matrix coefficients, alpha, grids, transformations, metadata, truncated data, or resource limits. - There is no decode matrix covering bit depth, subsampling, range, matrix coefficients, alpha, grids, transformations, metadata, truncated data, or resource limits.

42
src/ImageSharp/Formats/Heif/Av1/Av1Decoder.cs

@ -14,7 +14,6 @@ internal class Av1Decoder : IAv1TileReader
private readonly ObuReader obuReader; private readonly ObuReader obuReader;
private readonly Configuration configuration; private readonly Configuration configuration;
private Av1TileReader? tileReader; private Av1TileReader? tileReader;
private Av1FrameDecoder? frameDecoder;
public Av1Decoder(Configuration configuration) public Av1Decoder(Configuration configuration)
{ {
@ -28,8 +27,6 @@ internal class Av1Decoder : IAv1TileReader
public Av1FrameInfo? FrameInfo { get; private set; } public Av1FrameInfo? FrameInfo { get; private set; }
public Av1FrameBuffer<byte>? FrameBuffer { get; private set; }
public Image<TPixel> Decode<TPixel>(Span<byte> buffer) public Image<TPixel> Decode<TPixel>(Span<byte> buffer)
where TPixel : unmanaged, IPixel<TPixel> where TPixel : unmanaged, IPixel<TPixel>
{ {
@ -40,14 +37,33 @@ internal class Av1Decoder : IAv1TileReader
Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader));
this.FrameInfo = this.tileReader.FrameInfo; this.FrameInfo = this.tileReader.FrameInfo;
this.FrameBuffer = new(this.configuration, this.SequenceHeader, this.SequenceHeader.ColorConfig.GetColorFormat(), false); using Av1FrameBuffer<byte> frameBuffer = new(
this.frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, this.FrameBuffer); this.configuration,
this.frameDecoder.DecodeFrame(); this.SequenceHeader,
this.SequenceHeader.ColorConfig.GetColorFormat(),
false);
Av1FrameDecoder frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, frameBuffer);
frameDecoder.DecodeFrame();
Image<TPixel>? resultImage = null;
try
{
resultImage = new Image<TPixel>(
this.configuration,
this.FrameHeader.FrameSize.FrameWidth,
this.FrameHeader.FrameSize.FrameHeight,
null);
Image<TPixel> resultImage = new(this.FrameHeader.FrameSize.FrameWidth, this.FrameHeader.FrameSize.FrameHeight); ImageFrame<TPixel> resultFrame = resultImage.Frames.RootFrame;
ImageFrame<TPixel> resultFrame = resultImage.Frames.RootFrame; Av1YuvConverter.ConvertToRgb(this.configuration, frameBuffer, resultFrame);
Av1YuvConverter.ConvertToRgb(this.configuration, this.FrameBuffer, resultFrame); return resultImage;
return resultImage; }
catch
{
resultImage?.Dispose();
throw;
}
} }
public void ReadTile(Span<byte> tileData, int tileNum) public void ReadTile(Span<byte> tileData, int tileNum)
@ -56,13 +72,9 @@ internal class Av1Decoder : IAv1TileReader
{ {
this.SequenceHeader = this.obuReader.SequenceHeader; this.SequenceHeader = this.obuReader.SequenceHeader;
this.FrameHeader = this.obuReader.FrameHeader; this.FrameHeader = this.obuReader.FrameHeader;
Guard.NotNull(this.tileReader, nameof(this.tileReader));
Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader)); Guard.NotNull(this.SequenceHeader, nameof(this.SequenceHeader));
Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader)); Guard.NotNull(this.FrameHeader, nameof(this.FrameHeader));
this.FrameInfo = new(this.SequenceHeader); this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader, this.FrameHeader);
this.FrameBuffer = new(this.configuration, this.SequenceHeader, this.SequenceHeader.ColorConfig.GetColorFormat(), false);
this.frameDecoder = new(this.SequenceHeader, this.FrameHeader, this.FrameInfo, this.FrameBuffer);
this.tileReader = new Av1TileReader(this.configuration, this.SequenceHeader, this.FrameHeader, this.frameDecoder);
} }
this.tileReader.ReadTile(tileData, tileNum); this.tileReader.ReadTile(tileData, tileNum);

11
src/ImageSharp/Formats/Heif/Av1/Tiling/Av1TileReader.cs

@ -33,14 +33,13 @@ internal class Av1TileReader : IAv1TileReader
private readonly int[] firstTransformOffset = new int[2]; private readonly int[] firstTransformOffset = new int[2];
private readonly int[] coefficientIndex = []; private readonly int[] coefficientIndex = [];
private readonly Configuration configuration; private readonly Configuration configuration;
private readonly IAv1FrameDecoder frameDecoder; private readonly IAv1FrameDecoder? frameDecoder;
public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, IAv1FrameDecoder frameDecoder) public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader)
{ {
this.FrameHeader = frameHeader; this.FrameHeader = frameHeader;
this.configuration = configuration; this.configuration = configuration;
this.SequenceHeader = sequenceHeader; this.SequenceHeader = sequenceHeader;
this.frameDecoder = frameDecoder;
// init_main_frame_ctxt // init_main_frame_ctxt
this.FrameInfo = new(this.SequenceHeader); this.FrameInfo = new(this.SequenceHeader);
@ -66,6 +65,10 @@ internal class Av1TileReader : IAv1TileReader
this.coefficientIndex = new int[Av1Constants.MaxPlanes]; this.coefficientIndex = new int[Av1Constants.MaxPlanes];
} }
public Av1TileReader(Configuration configuration, ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, IAv1FrameDecoder frameDecoder)
: this(configuration, sequenceHeader, frameHeader)
=> this.frameDecoder = frameDecoder;
public ObuFrameHeader FrameHeader { get; } public ObuFrameHeader FrameHeader { get; }
public ObuSequenceHeader SequenceHeader { get; } public ObuSequenceHeader SequenceHeader { get; }
@ -126,7 +129,7 @@ internal class Av1TileReader : IAv1TileReader
this.ParsePartition(ref reader, modeInfoPosition, superBlockSize, superblockInfo, tileInfo); this.ParsePartition(ref reader, modeInfoPosition, superBlockSize, superblockInfo, tileInfo);
// decoding of the superblock // decoding of the superblock
this.frameDecoder.DecodeSuperblock(modeInfoPosition, superblockInfo, tileInfo); this.frameDecoder?.DecodeSuperblock(modeInfoPosition, superblockInfo, tileInfo);
} }
} }
} }

15
tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs

@ -6,12 +6,27 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit;
using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Memory;
using SixLabors.ImageSharp.PixelFormats;
namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1; namespace SixLabors.ImageSharp.Tests.Formats.Heif.Av1;
[Trait("Format", "Avif")] [Trait("Format", "Avif")]
public class Av1TilingTests public class Av1TilingTests
{ {
[Fact]
public void DecoderReadsFirstTile()
{
string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, TestImages.Heif.Orange4x4);
byte[] content = File.ReadAllBytes(filePath);
Av1Decoder decoder = new(Configuration.Default);
using Image<Rgba32> image = decoder.Decode<Rgba32>(content.AsSpan(0x010E, 0x001D));
Assert.Equal(4, image.Width);
Assert.Equal(4, image.Height);
Assert.True(image.Frames.RootFrame.PixelBuffer.DangerousGetSingleSpan().ContainsAnyExcept(default(Rgba32)));
}
[Theory] [Theory]
[InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d, 21, 1)] [InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d, 21, 1)]
public void DecodePixelsFirstTile(string filename, int dataOffset, int dataSize, int tileOffset, int superblockCount) public void DecodePixelsFirstTile(string filename, int dataOffset, int dataSize, int tileOffset, int superblockCount)

Loading…
Cancel
Save