tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Document HEIF frame pipeline

pull/2633/head
James Jackson-South 1 week ago
parent
50ae1237fd
commit
c7a92ecfea
3 changed files with 92 additions and 63 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 79
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs
  2. 66
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
  3. 10
      src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs

79
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameDecoder.cs
View File

@ -8,16 +8,53 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Reconstructs the coded blocks of one AV1 still-image frame into planar sample buffers.
/// </summary>
internal class Av1FrameDecoder : IAv1FrameDecoder
{
/// <summary>
/// The sequence-level superblock and color configuration.
/// </summary>
private readonly ObuSequenceHeader sequenceHeader;
/// <summary>
/// The frame-level tile, quantization, and reconstruction configuration.
/// </summary>
private readonly ObuFrameHeader frameHeader;
/// <summary>
/// The parsed superblock and block-mode information for the frame.
/// </summary>
private readonly Av1FrameInfo frameInfo;
/// <summary>
/// The destination planar sample buffers for reconstructed pixels.
/// </summary>
private readonly Av1FrameBuffer<byte> frameBuffer;
/// <summary>
/// The coefficient inverse-quantization stage shared across superblocks.
/// </summary>
private readonly Av1InverseQuantizer inverseQuantizer;
/// <summary>
/// The frame's base per-segment and per-plane dequantization values.
/// </summary>
private readonly Av1DeQuantizationContext deQuants;
/// <summary>
/// The block reconstruction stage that applies prediction and inverse transforms.
/// </summary>
private readonly Av1BlockDecoder blockDecoder;
/// <summary>
/// Initializes a new instance of the <see cref="Av1FrameDecoder"/> class.
/// </summary>
/// <param name="sequenceHeader">The parsed AV1 sequence header.</param>
/// <param name="frameHeader">The parsed AV1 frame header.</param>
/// <param name="frameInfo">The parsed superblock and block-mode information.</param>
/// <param name="frameBuffer">The destination planar sample buffers.</param>
public Av1FrameDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, Av1FrameInfo frameInfo, Av1FrameBuffer<byte> frameBuffer)
{
this.sequenceHeader = sequenceHeader;
@ -29,8 +66,12 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
this.blockDecoder = new(this.sequenceHeader, this.frameHeader, this.frameBuffer);
}
/// <summary>
/// Reconstructs every coded tile of the frame; in-loop post-processing stages remain disabled until implemented.
/// </summary>
public void DecodeFrame()
{
// Tile columns are the outer traversal because each call walks that column's tile rows and their superblocks.
for (int column = 0; column < this.frameHeader.TilesInfo.TileColumnCount; column++)
{
this.DecodeFrameTiles(column);
@ -39,6 +80,9 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
bool doLoopFilterFlag = false;
bool doLoopRestoration = false;
bool doUpscale = false;
// These flags remain false until the corresponding normative stages have complete scalar implementations
// and independent still-image vectors; silently running partial filters would corrupt reconstructed pixels.
if (doLoopFilterFlag)
{
this.DecodeLoopFilterForFrame();
@ -61,14 +105,18 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
}
/// <summary>
/// SVT: decode_tile
/// Reconstructs every tile row in one tile column.
/// </summary>
/// <param name="tileColumn">The zero-based tile-column index.</param>
/// <remarks>SVT-AV1: <c>decode_tile</c>.</remarks>
private void DecodeFrameTiles(int tileColumn)
{
int tileRowCount = this.frameHeader.TilesInfo.TileRowCount;
int tileCount = tileRowCount * this.frameHeader.TilesInfo.TileColumnCount;
for (int row = 0; row < tileRowCount; row++)
{
// Tile row starts are signaled in 4x4 mode-info units. Convert to pixels and then to superblock rows
// so the frame-level superblock store and the tile-local syntax address the same region.
int superblockRowTileStart = this.frameHeader.TilesInfo.TileRowStartModeInfo[row] << Av1Constants.ModeInfoSizeLog2 >>
this.sequenceHeader.SuperblockSizeLog2;
int superblockRow = row + superblockRowTileStart;
@ -82,8 +130,13 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
}
/// <summary>
/// SVT: decode_tile_row
/// Reconstructs the superblocks in one tile row from left to right.
/// </summary>
/// <param name="tileRow">The zero-based tile-row index.</param>
/// <param name="tileColumn">The zero-based tile-column index.</param>
/// <param name="modeInfoRow">The frame-relative row in 4x4 mode-info units.</param>
/// <param name="superblockRow">The frame-relative superblock row.</param>
/// <remarks>SVT-AV1: <c>decode_tile_row</c>.</remarks>
private void DecodeTileRow(int tileRow, int tileColumn, int modeInfoRow, int superblockRow)
{
int superblockModeInfoSizeLog2 = this.sequenceHeader.SuperblockSizeLog2 - Av1Constants.ModeInfoSizeLog2;
@ -96,6 +149,7 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
for (int modeInfoColumn = tileInfo.TileColumnStartModeInfo[tileColumn]; modeInfoColumn < tileInfo.TileColumnStartModeInfo[tileColumn + 1];
modeInfoColumn += this.sequenceHeader.SuperblockModeInfoSize)
{
// Convert the signaled 4x4 mode-info column to the frame-level superblock index used by Av1FrameInfo.
int superblockColumn = modeInfoColumn << Av1Constants.ModeInfoSizeLog2 >> this.sequenceHeader.SuperblockSizeLog2;
Av1SuperblockInfo superblockInfo = this.frameInfo.GetSuperblock(new Point(superblockColumn, superblockRow));
@ -106,8 +160,12 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
}
/// <summary>
/// SVT: svt_aom_decode_super_block
/// Reconstructs one superblock after applying its block state and delta-Q context.
/// </summary>
/// <param name="modeInfoPosition">The superblock's top-left position in 4x4 mode-info units.</param>
/// <param name="superblockInfo">The decoded syntax and block modes for the superblock.</param>
/// <param name="tileInfo">The tile that contains the superblock.</param>
/// <remarks>SVT-AV1: <c>svt_aom_decode_super_block</c>.</remarks>
public void DecodeSuperblock(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo)
{
this.blockDecoder.UpdateSuperblock(superblockInfo);
@ -116,8 +174,12 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
}
/// <summary>
/// SVT: decode_partition
/// Reconstructs each decoded block in a superblock partition.
/// </summary>
/// <param name="modeInfoPosition">The superblock's frame-relative origin in 4x4 mode-info units.</param>
/// <param name="superblockInfo">The superblock whose block modes are traversed.</param>
/// <param name="tileInfo">The tile boundary information used by intra prediction.</param>
/// <remarks>SVT-AV1: <c>decode_partition</c>.</remarks>
private void DecodePartition(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo)
{
foreach (Av1BlockModeInfo modeInfo in superblockInfo.GetModeInfos())
@ -125,18 +187,23 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
Point subPosition = modeInfo.PositionInSuperblock;
Av1BlockSize subSize = modeInfo.BlockSize;
Point globalPosition = new(modeInfoPosition.X, modeInfoPosition.Y);
// Block positions are stored relative to the superblock; prediction and reconstruction require frame-relative mode-info coordinates.
globalPosition.Offset(subPosition);
this.blockDecoder.DecodeBlock(modeInfo, globalPosition, subSize, superblockInfo, tileInfo);
}
}
/// <summary>
/// Traverses frame superblocks in the order required by the not-yet-implemented deblocking stage.
/// </summary>
private void DecodeLoopFilterForFrame()
{
int superblockSizeLog2 = this.sequenceHeader.SuperblockSizeLog2;
int pictureWidthInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameWidth, this.sequenceHeader.SuperblockSizeLog2);
int pictureHeightInSuperblocks = Av1Math.DivideLog2Ceiling(this.frameHeader.FrameSize.FrameHeight, this.sequenceHeader.SuperblockSizeLog2);
// Loop over a frame : tregger dec_loop_filter_sb for each SB
// Deblocking uses raster traversal so each block can consume already reconstructed top and left edges.
for (int superblockIndexY = 0; superblockIndexY < pictureHeightInSuperblocks; ++superblockIndexY)
{
for (int superblockIndexX = 0; superblockIndexX < pictureWidthInSuperblocks; ++superblockIndexX)
@ -148,7 +215,7 @@ internal class Av1FrameDecoder : IAv1FrameDecoder
Point superblockPoint = new(superblockOriginX, superblockOriginY);
Av1SuperblockInfo superblockInfo = this.frameInfo.GetSuperblock(superblockPoint);
// LF function for a SB
// Superblock filtering remains disabled until its complete plane and edge-strength implementation is available.
/*
DecodeLoopFilterForSuperblock(
superblockInfo,

66
src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs
View File

@ -5,70 +5,32 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Defines the work-in-progress AV1 still-image frame-encoding boundary.
/// </summary>
internal class Av1FrameEncoder
{
/// <summary>
/// The source plane samples supplied for frame encoding.
/// </summary>
private readonly Av1FrameBuffer<byte> frameBuffer;
/// <summary>
/// Initializes a new instance of the <see cref="Av1FrameEncoder"/> class.
/// </summary>
/// <param name="frameBuffer">The source frame samples to encode.</param>
public Av1FrameEncoder(Av1FrameBuffer<byte> frameBuffer)
{
this.frameBuffer = frameBuffer;
}
/// <summary>
/// SVT: svt_av1_enc_init
/// Represents the not-yet-implemented entry point for encoding one AV1 still-image frame.
/// </summary>
/// <remarks>SVT-AV1: <c>svt_av1_enc_init</c>.</remarks>
public static void Encode()
{
/************************************
* Thread Handles
************************************/
// Resource Coordination
// Single thread calling svt_aom_resource_coordination_kernel with context enc_handle_ptr->resource_coordination_context_ptr
// Multiple threads calling svt_aom_picture_analysis_kernel, with context enc_handle_ptr->picture_analysis_context_ptr_array
// Picture Decision
// Single thread calling svt_aom_picture_decision_kernel with context enc_handle_ptr->picture_decision_context_ptr
// Motion Estimation
// Multiple threads calling svt_aom_motion_estimation_kernel with context enc_handle_ptr->motion_estimation_context_ptr_array
// Initial Rate Control
// Single thread calling svt_aom_initial_rate_control_kernel with context enc_handle_ptr->initial_rate_control_context_ptr
// Source Based Oprations
// <Multiple threads calling svt_aom_source_based_operations_kernel with context enc_handle_ptr->source_based_operations_context_ptr_array
// TPL dispenser
// Multiple threads calling svt_aom_tpl_disp_kernel with context enc_handle_ptr->tpl_disp_context_ptr_array
// Picture Manager
// Single thread calling svt_aom_picture_manager_kernel with context enc_handle_ptr->picture_manager_context_ptr
// Rate Control
// Single thread calling svt_aom_rate_control_kernel with context enc_handle_ptr->rate_control_context_ptr
// Mode Decision Configuration Process
// Multiple threads calling svt_aom_mode_decision_configuration_kernel with context enc_handle_ptr->mode_decision_configuration_context_ptr_array
// EncDec Process
// Multiple threads calling svt_aom_mode_decision_kernel enc_handle_ptr->enc_dec_context_ptr_array
// Dlf Process
// Multiple threads calling svt_aom_dlf_kernel with context enc_handle_ptr->dlf_context_ptr_array
// Cdef Process
// Multiple threads calling svt_aom_cdef_kernel enc_handle_ptr->cdef_context_ptr_array
// Rest Process
// Multiple threads calling svt_aom_rest_kernel enc_handle_ptr->rest_context_ptr_array
// Entropy Coding Process
// Multiple threads calling svt_aom_entropy_coding_kernel enc_handle_ptr->entropy_coding_context_ptr_array
// Packetization
// Single thread calling svt_aom_packetization_kernel with context enc_handle_ptr->packetization_context_ptr
// svt_print_memory_usage();
// Still-image encoding needs the normative analysis, transform, quantization, entropy, and packetization stages,
// but it does not require SVT-AV1's application-level worker graph or video-sequence process orchestration.
}
}

10
src/ImageSharp/Formats/Heif/Av1/Pipeline/IAv1FrameDecoder.cs
View File

@ -6,15 +6,15 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline;
/// <summary>
/// Interface for decoder of a single frame.
/// Defines reconstruction of decoded AV1 superblocks within a single still-image frame.
/// </summary>
internal interface IAv1FrameDecoder
{
/// <summary>
/// Decode a single superblock.
/// Reconstructs one decoded superblock into the current frame buffer.
/// </summary>
/// <param name="modeInfoPosition">The top left position of the superblock, in mode info units.</param>
/// <param name="superblockInfo">The superblock to decode</param>
/// <param name="tileInfo">The tile in whcih the superblock is positioned.</param>
/// <param name="modeInfoPosition">The superblock's top-left position in 4x4 mode-info units.</param>
/// <param name="superblockInfo">The decoded syntax and block modes for the superblock.</param>
/// <param name="tileInfo">The tile that contains the superblock.</param>
void DecodeSuperblock(Point modeInfoPosition, Av1SuperblockInfo superblockInfo, Av1TileInfo tileInfo);
}

Write Preview
Loading…
Cancel
Save