diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 56a7cc7e9..581bb522e 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -99,7 +99,7 @@ This assessment is based on the current source after the upstream ImageSharp mer - The single-still `Av1Decoder` path now parses tile state before allocating and reconstructing one frame, and it disposes the reconstruction planes after pixel conversion. Reference-frame, `show_existing_frame`, and multi-frame ownership remain incomplete. - The reconstruction pipeline disables loop filtering, CDEF, super-resolution, loop restoration, and padding with constant flags. These are normative stages when signaled, not optional quality improvements. - Loop restoration, palette paths, `show_existing_frame`, reference/CDF state, and other syntax paths contain `NotImplementedException` or equivalent unsupported branches. -- The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active prediction and block reconstruction path is still limited to 8-bit samples and must be connected to the existing high-bit-depth inverse-transform core. +- The frame buffer now establishes two-byte native sample storage, logical plane rows, and sample-unit block strides for 10/12-bit frames. The active intra-prediction, inverse-transform, and block-reconstruction path selects native 16-bit samples for 10/12-bit frames and has focused pipeline wiring coverage. Independently encoded high-bit-depth AVIF conformance files are still required; chroma-from-luma storage and the other normative reconstruction stages listed below remain incomplete. - `Av1YuvConverter` now consumes the signaled range, supported H.273 matrix coefficients, subsampling, and chroma sample position for 8, 10, and 12-bit output and uses one allocator-backed RGB row. Constant-luminance and chromaticity-derived matrices, ICtCp, and encoder-side subsampling remain incomplete. - The inverse-transform path allocates arrays in a per-transform hot path. - No usable end-to-end AV1 SIMD path was found. The most visible 4x4 forward-transform SIMD call is commented out, while the production prediction, transform, filter, and output paths are predominantly scalar. diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index 70803a4c6..e9269c7b2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -1,7 +1,9 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Numerics; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -15,13 +17,11 @@ internal class Av1PredictionDecoder private readonly ObuSequenceHeader sequenceHeader; private readonly ObuFrameHeader frameHeader; - private readonly bool is16BitPipeline; - public Av1PredictionDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader, bool is16BitPipeline) + public Av1PredictionDecoder(ObuSequenceHeader sequenceHeader, ObuFrameHeader frameHeader) { this.sequenceHeader = sequenceHeader; this.frameHeader = frameHeader; - this.is16BitPipeline = is16BitPipeline; } /// @@ -37,15 +37,60 @@ internal class Av1PredictionDecoder Av1BitDepth bitDepth, int blockModeInfoColumnOffset, int blockModeInfoRowOffset) + => this.DecodeCore( + partitionInfo, + plane, + transformSize, + tileInfo, + pixelBuffer, + pixelStride, + bitDepth, + blockModeInfoColumnOffset, + blockModeInfoRowOffset); + + /// + /// AV1: 7.11.2 Reconstruct. + /// + public void Decode( + Av1PartitionInfo partitionInfo, + Av1Plane plane, + Av1TransformSize transformSize, + Av1TileInfo tileInfo, + Span pixelBuffer, + int pixelStride, + Av1BitDepth bitDepth, + int blockModeInfoColumnOffset, + int blockModeInfoRowOffset) + => this.DecodeCore( + partitionInfo, + plane, + transformSize, + tileInfo, + pixelBuffer, + pixelStride, + bitDepth, + blockModeInfoColumnOffset, + blockModeInfoRowOffset); + + private void DecodeCore( + Av1PartitionInfo partitionInfo, + Av1Plane plane, + Av1TransformSize transformSize, + Av1TileInfo tileInfo, + Span pixelBuffer, + int pixelStride, + Av1BitDepth bitDepth, + int blockModeInfoColumnOffset, + int blockModeInfoRowOffset) + where T : unmanaged, IBinaryInteger { int stride = pixelStride; // Deviation from SVT: Buffer starts at PREVIOUS row. - Span topNeighbor = pixelBuffer; - Span leftNeighbor = pixelBuffer[(stride - 1)..]; - Span startOfPixels = pixelBuffer[stride..]; + Span topNeighbor = pixelBuffer; + Span leftNeighbor = pixelBuffer[(stride - 1)..]; + Span startOfPixels = pixelBuffer[stride..]; - bool is16BitPipeline = this.is16BitPipeline; Av1PredictionMode mode = (plane == Av1Plane.Y) ? partitionInfo.ModeInfo.YMode : partitionInfo.ModeInfo.UvMode; if (plane != Av1Plane.Y && partitionInfo.ModeInfo.UvMode == Av1PredictionMode.UvChromaFromLuma) @@ -92,7 +137,8 @@ internal class Av1PredictionDecoder bitDepth); } - private void PredictChromaFromLumaBlock(Av1PartitionInfo partitionInfo, Av1ChromaFromLumaContext? chromaFromLumaContext, Span pixelBuffer, int stride, Av1TransformSize transformSize, Av1Plane plane) + private void PredictChromaFromLumaBlock(Av1PartitionInfo partitionInfo, Av1ChromaFromLumaContext? chromaFromLumaContext, Span pixelBuffer, int stride, Av1TransformSize transformSize, Av1Plane plane) + where T : unmanaged, IBinaryInteger { Av1BlockModeInfo modeInfo = partitionInfo.ModeInfo; bool isChromaFromLumaAllowedFlag = IsChromaFromLumaAllowedWithFrameHeader(partitionInfo, this.sequenceHeader.ColorConfig, this.frameHeader); @@ -112,22 +158,6 @@ internal class Av1PredictionDecoder // assert((transformSize.GetHeight() - 1) * CFL_BUF_LINE + transformSize.GetWidth() <= CFL_BUF_SQUARE); Av1BitDepth bitDepth = this.sequenceHeader.ColorConfig.BitDepth; - if ((bitDepth != Av1BitDepth.EightBit) || this.is16BitPipeline) - { - /* 16 bit pipeline - svt_cfl_predict_hbd( - chromaFromLumaContext->recon_buf_q3, - (uint16_t*)dst, - dst_stride, - (uint16_t*)dst, - dst_stride, - alpha_q3, - cc->bit_depth, - tx_size_wide[tx_size], - tx_size_high[tx_size]); - return;*/ - } - ChromaFromLumaPredict( chromaFromLumaContext.Q3Buffer!.DangerousGetSingleSpan(), pixelBuffer, @@ -177,7 +207,8 @@ internal class Av1PredictionDecoder return Av1Math.RoundPowerOf2Signed(scaledLumaQ6, 6); } - private static void ChromaFromLumaPredict(Span predictedBufferQ3, Span predictedBuffer, int predictedStride, Span destinationBuffer, int destinationStride, int alphaQ3, Av1BitDepth bitDepth, int width, int height) + private static void ChromaFromLumaPredict(Span predictedBufferQ3, Span predictedBuffer, int predictedStride, Span destinationBuffer, int destinationStride, int alphaQ3, Av1BitDepth bitDepth, int width, int height) + where T : unmanaged, IBinaryInteger { // TODO: Make SIMD variant of this method. int maxPixelValue = (1 << bitDepth.GetBitCount()) - 1; @@ -186,7 +217,8 @@ internal class Av1PredictionDecoder for (int i = 0; i < width; i++) { int alphaQ0 = GetScaledLumaQ0(alphaQ3, predictedBufferQ3[i]); - destinationBuffer[i] = (byte)Av1Math.Clamp(alphaQ0 + predictedBuffer[i], 0, maxPixelValue); + int predicted = int.CreateChecked(predictedBuffer[i]); + destinationBuffer[i] = T.CreateChecked(Av1Math.Clamp(alphaQ0 + predicted, 0, maxPixelValue)); } destinationBuffer = destinationBuffer[destinationStride..]; @@ -195,20 +227,21 @@ internal class Av1PredictionDecoder } } - private void PredictIntraBlock( + private void PredictIntraBlock( Av1PartitionInfo partitionInfo, Av1Plane plane, Av1TransformSize transformSize, Av1TileInfo tileInfo, - Span pixelBuffer, + Span pixelBuffer, int pixelBufferStride, - Span topNeighbor, - Span leftNeighbor, + Span topNeighbor, + Span leftNeighbor, int referenceStride, Av1PredictionMode mode, int blockModeInfoColumnOffset, int blockModeInfoRowOffset, Av1BitDepth bitDepth) + where T : unmanaged, IBinaryInteger { // TODO:are_parameters_computed variable for CFL so that cal part for V plane we can skip, // once we compute for U plane, this parameter is block level parameter. @@ -286,49 +319,25 @@ internal class Av1PredictionDecoder bool disableEdgeFilter = !this.sequenceHeader.EnableIntraEdgeFilter; - // Calling all other intra predictors except CFL & pallate... - if (bitDepth == Av1BitDepth.EightBit && !this.is16BitPipeline) - { - this.DecodeBuildIntraPredictors( - partitionInfo, - topNeighbor, - leftNeighbor, - (nuint)referenceStride, - pixelBuffer, - (nuint)pixelBufferStride, - mode, - angleDelta, - filterIntraMode, - transformSize, - disableEdgeFilter, - haveTop ? Math.Min(transformWidth, xr + transformWidth) : 0, - haveTopRight ? Math.Min(transformWidth, xr) : 0, - haveLeft ? Math.Min(transformHeight, yd + transformHeight) : 0, - haveBottomLeft ? Math.Min(transformHeight, yd) : 0, - plane); - } - else - { - /* 16bit - decode_build_intra_predictors_high(xd, - (uint16_t*) top_neigh_array, //As per SVT Enc - (uint16_t*) left_neigh_array, - ref_stride,// As per SVT Enc - (uint16_t*) pv_pred_buf, - pred_stride, - mode, - angle_delta, - filter_intra_mode, - tx_size, - disable_edge_filter, - have_top? AOMMIN(transformWidth, xr + transformWidth) : 0, - have_top_right? AOMMIN(transformWidth, xr) : 0, - have_left? AOMMIN(transformHeight, yd + transformHeight) : 0, - have_bottom_left? AOMMIN(transformHeight, yd) : 0, - plane, - bit_depth); - */ - } + // Calling all other intra predictors except CFL and palette. + this.DecodeBuildIntraPredictors( + partitionInfo, + topNeighbor, + leftNeighbor, + (nuint)referenceStride, + pixelBuffer, + (nuint)pixelBufferStride, + mode, + angleDelta, + filterIntraMode, + transformSize, + disableEdgeFilter, + haveTop ? Math.Min(transformWidth, xr + transformWidth) : 0, + haveTopRight ? Math.Min(transformWidth, xr) : 0, + haveLeft ? Math.Min(transformHeight, yd + transformHeight) : 0, + haveBottomLeft ? Math.Min(transformHeight, yd) : 0, + plane, + bitDepth.GetBitCount()); } private static Av1BlockSize ScaleChromaBlockSize(Av1BlockSize blockSize, bool subX, bool subY) @@ -559,12 +568,12 @@ internal class Av1PredictionDecoder } } - private void DecodeBuildIntraPredictors( + private void DecodeBuildIntraPredictors( Av1PartitionInfo partitionInfo, - Span aboveNeighbor, - Span leftNeighbor, + Span aboveNeighbor, + Span leftNeighbor, nuint referenceStride, - Span destination, + Span destination, nuint destinationStride, Av1PredictionMode mode, int angleDelta, @@ -575,12 +584,17 @@ internal class Av1PredictionDecoder int topRightPixelCount, int leftPixelCount, int bottomLeftPixelCount, - Av1Plane plane) + Av1Plane plane, + int bitDepth) + where T : unmanaged, IBinaryInteger { - Span aboveData = stackalloc byte[(Av1Constants.MaxTransformSize * 2) + 32]; - Span leftData = stackalloc byte[(Av1Constants.MaxTransformSize * 2) + 32]; - Span aboveRow = aboveData[16..]; - Span leftColumn = leftData[16..]; + int baseValue = 128 << (bitDepth - 8); + Span aboveData = stackalloc T[(Av1Constants.MaxTransformSize * 2) + 32]; + Span leftData = stackalloc T[(Av1Constants.MaxTransformSize * 2) + 32]; + aboveData.Fill(T.CreateChecked(baseValue - 1)); + leftData.Fill(T.CreateChecked(baseValue + 1)); + Span aboveRow = aboveData[16..]; + Span leftColumn = leftData[16..]; int transformWidth = transformSize.GetWidth(); int transformHeight = transformSize.GetHeight(); bool isDirectionalMode = mode.IsDirectional(); @@ -628,21 +642,19 @@ internal class Av1PredictionDecoder if ((!needAbove && leftPixelCount == 0) || (!needLeft && topPixelCount == 0)) { - byte val; + T value; if (needLeft) { - val = (byte)((topPixelCount > 0) ? aboveNeighbor[0] : 129); + value = topPixelCount > 0 ? aboveNeighbor[0] : T.CreateChecked(baseValue + 1); } else { - val = (byte)((leftPixelCount > 0) ? leftNeighbor[0] : 127); + value = leftPixelCount > 0 ? leftNeighbor[0] : T.CreateChecked(baseValue - 1); } - ref byte destinationRef = ref destination[0]; for (int i = 0; i < transformHeight; ++i) { - Unsafe.InitBlock(ref destinationRef, val, (uint)transformWidth); - destinationRef = ref Unsafe.Add(ref destinationRef, destinationStride); + destination.Slice(i * (int)destinationStride, transformWidth).Fill(value); } return; @@ -662,7 +674,7 @@ internal class Av1PredictionDecoder needBottom = angle > 180; } - uint numLeftPixelsNeeded = (uint)(transformHeight + (needBottom ? transformWidth : 0)); + int numLeftPixelsNeeded = transformHeight + (needBottom ? transformWidth : 0); int i = 0; if (leftPixelCount > 0) { @@ -682,18 +694,18 @@ internal class Av1PredictionDecoder if (i < numLeftPixelsNeeded) { - Unsafe.InitBlock(ref leftColumn[i], leftColumn[i - 1], numLeftPixelsNeeded - (uint)i); + leftColumn.Slice(i, numLeftPixelsNeeded - i).Fill(leftColumn[i - 1]); } } else { if (topPixelCount > 0) { - Unsafe.InitBlock(ref leftColumn[0], aboveNeighbor[0], numLeftPixelsNeeded); + leftColumn[..numLeftPixelsNeeded].Fill(aboveNeighbor[0]); } else { - Unsafe.InitBlock(ref leftColumn[0], 129, numLeftPixelsNeeded); + leftColumn[..numLeftPixelsNeeded].Fill(T.CreateChecked(baseValue + 1)); } } } @@ -712,39 +724,39 @@ internal class Av1PredictionDecoder needRight = angle < 90; } - uint numTopPixelsNeeded = (uint)(transformWidth + (needRight ? transformHeight : 0)); + int numTopPixelsNeeded = transformWidth + (needRight ? transformHeight : 0); if (topPixelCount > 0) { - Unsafe.CopyBlock(ref aboveRow[0], ref aboveNeighbor[0], (uint)topPixelCount); + aboveNeighbor[..topPixelCount].CopyTo(aboveRow); int i = topPixelCount; if (needRight && topPixelCount > 0) { Guard.IsTrue(topPixelCount == transformWidth, nameof(topPixelCount), string.Empty); - Unsafe.CopyBlock(ref aboveRow[transformWidth], ref aboveNeighbor[transformWidth], (uint)topPixelCount); - i += topPixelCount; + aboveNeighbor.Slice(transformWidth, topRightPixelCount).CopyTo(aboveRow[transformWidth..]); + i += topRightPixelCount; } if (i < numTopPixelsNeeded) { - Unsafe.InitBlock(ref aboveRow[i], aboveRow[i - 1], numTopPixelsNeeded - (uint)i); + aboveRow.Slice(i, numTopPixelsNeeded - i).Fill(aboveRow[i - 1]); } } else { if (leftPixelCount > 0) { - Unsafe.InitBlock(ref aboveRow[0], leftNeighbor[0], numTopPixelsNeeded); + aboveRow[..numTopPixelsNeeded].Fill(leftNeighbor[0]); } else { - Unsafe.InitBlock(ref aboveRow[0], 127, numTopPixelsNeeded); + aboveRow[..numTopPixelsNeeded].Fill(T.CreateChecked(baseValue - 1)); } } } if (needAboveLeft) { - ref byte aboveLeft = ref Unsafe.Subtract(ref aboveRow[0], 1); + ref T aboveLeft = ref Unsafe.Subtract(ref aboveRow[0], 1); if (topPixelCount > 0 && leftPixelCount > 0) { aboveLeft = Unsafe.Subtract(ref aboveNeighbor[0], 1); @@ -759,7 +771,7 @@ internal class Av1PredictionDecoder } else { - aboveLeft = 128; + aboveLeft = T.CreateChecked(baseValue); } Unsafe.Subtract(ref leftColumn[0], 1) = aboveLeft; @@ -767,7 +779,7 @@ internal class Av1PredictionDecoder if (useFilterIntra) { - Av1PredictorFactory.FilterIntraPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode); + FilterIntraPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, filterIntraMode, bitDepth); return; } @@ -810,7 +822,7 @@ internal class Av1PredictionDecoder { int pixelCount = transformWidth + (needRight ? transformHeight : 0); - UpsampleIntraEdge(aboveRow, pixelCount); + UpsampleIntraEdge(aboveRow, pixelCount, bitDepth); } upsampleLeft = UseIntraEdgeUpsample(transformHeight, transformWidth, angle - 180, filterType); @@ -818,34 +830,144 @@ internal class Av1PredictionDecoder { int pixelCount = transformHeight + (needBottom ? transformWidth : 0); - UpsampleIntraEdge(leftColumn, pixelCount); + UpsampleIntraEdge(leftColumn, pixelCount, bitDepth); } } - Av1PredictorFactory.DirectionalPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle); + DirectionalPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle, bitDepth); return; } // predict if (mode == Av1PredictionMode.DC) { - Av1PredictorFactory.DcPredictor(leftPixelCount > 0, topPixelCount > 0, transformSize, destination, destinationStride, aboveRow, leftColumn); + DcPredictor(leftPixelCount > 0, topPixelCount > 0, transformSize, destination, destinationStride, aboveRow, leftColumn, bitDepth); } else { - Av1PredictorFactory.GeneralPredictor(mode, transformSize, destination, destinationStride, aboveRow, leftColumn); + GeneralPredictor(mode, transformSize, destination, destinationStride, aboveRow, leftColumn); + } + } + + private static void DcPredictor(bool hasLeft, bool hasAbove, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span above, Span left, int bitDepth) + where T : unmanaged + { + if (typeof(T) == typeof(byte)) + { + Av1PredictorFactory.DcPredictor( + hasLeft, + hasAbove, + transformSize, + MemoryMarshal.Cast(destination), + destinationStride, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left)); + } + else + { + Av1PredictorFactory.DcPredictor( + hasLeft, + hasAbove, + transformSize, + MemoryMarshal.Cast(destination), + destinationStride, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + bitDepth); + } + } + + private static void GeneralPredictor(Av1PredictionMode mode, Av1TransformSize transformSize, Span destination, nuint destinationStride, Span above, Span left) + where T : unmanaged + { + if (typeof(T) == typeof(byte)) + { + Av1PredictorFactory.GeneralPredictor( + mode, + transformSize, + MemoryMarshal.Cast(destination), + destinationStride, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left)); + } + else + { + Av1PredictorFactory.GeneralPredictor( + mode, + transformSize, + MemoryMarshal.Cast(destination), + destinationStride, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left)); + } + } + + private static void DirectionalPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, bool upsampleAbove, bool upsampleLeft, int angle, int bitDepth) + where T : unmanaged + { + if (typeof(T) == typeof(byte)) + { + Av1PredictorFactory.DirectionalPredictor( + MemoryMarshal.Cast(destination), + destinationStride, + transformSize, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + upsampleAbove, + upsampleLeft, + angle); + } + else + { + Av1PredictorFactory.DirectionalPredictor( + MemoryMarshal.Cast(destination), + destinationStride, + transformSize, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + upsampleAbove, + upsampleLeft, + angle, + bitDepth); + } + } + + private static void FilterIntraPredictor(Span destination, nuint destinationStride, Av1TransformSize transformSize, Span above, Span left, Av1FilterIntraMode mode, int bitDepth) + where T : unmanaged + { + if (typeof(T) == typeof(byte)) + { + Av1PredictorFactory.FilterIntraPredictor( + MemoryMarshal.Cast(destination), + destinationStride, + transformSize, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + mode); + } + else + { + Av1PredictorFactory.FilterIntraPredictor( + MemoryMarshal.Cast(destination), + destinationStride, + transformSize, + MemoryMarshal.Cast(above), + MemoryMarshal.Cast(left), + mode, + bitDepth); } } - private static void UpsampleIntraEdge(Span buffer, int count) + private static void UpsampleIntraEdge(Span buffer, int count, int bitDepth) + where T : unmanaged, IBinaryInteger { // TODO: Consider creating SIMD version // interpolate half-sample positions Guard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); - Span input = stackalloc byte[MaxUpsampleSize + 3]; - byte beforeBuffer = Unsafe.Subtract(ref buffer[0], 1); + Span input = stackalloc T[MaxUpsampleSize + 3]; + T beforeBuffer = Unsafe.Subtract(ref buffer[0], 1); // copy p[-1..(sz-1)] and extend first and last samples input[0] = beforeBuffer; @@ -861,9 +983,9 @@ internal class Av1PredictionDecoder Unsafe.Subtract(ref buffer[0], 2) = input[0]; for (int i = 0; i < count; i++) { - int s = -input[i] + (9 * input[i + 1]) + (9 * input[i + 2]) - input[i + 3]; - s = Av1Math.Clamp((s + 8) >> 4, 0, 255); - buffer[(2 * i) - 1] = (byte)s; + int s = -int.CreateChecked(input[i]) + (9 * int.CreateChecked(input[i + 1])) + (9 * int.CreateChecked(input[i + 2])) - int.CreateChecked(input[i + 3]); + s = Av1Math.Clamp((s + 8) >> 4, 0, (1 << bitDepth) - 1); + buffer[(2 * i) - 1] = T.CreateChecked(s); buffer[2 * i] = input[i + 2]; } } @@ -886,7 +1008,8 @@ internal class Av1PredictionDecoder /// /// SVT: svt_av1_filter_intra_edge_c /// - private static void FilterIntraEdge(ref byte buffer, int count, int strength) + private static void FilterIntraEdge(ref T buffer, int count, int strength) + where T : unmanaged, IBinaryInteger { // TODO: Consider creating SIMD version if (strength == 0) @@ -898,9 +1021,9 @@ internal class Av1PredictionDecoder [0, 4, 8, 4, 0], [0, 5, 6, 5, 0], [2, 4, 4, 4, 2] ]; int filt = strength - 1; - Span edge = stackalloc byte[129]; + Span edge = stackalloc T[129]; - Unsafe.CopyBlock(ref edge[0], ref buffer, (uint)count); + MemoryMarshal.CreateSpan(ref buffer, count).CopyTo(edge); for (int i = 1; i < count; i++) { int s = 0; @@ -909,11 +1032,11 @@ internal class Av1PredictionDecoder int k = i - 2 + j; k = (k < 0) ? 0 : k; k = (k > count - 1) ? count - 1 : k; - s += edge[k] * kernel[filt][j]; + s += int.CreateChecked(edge[k]) * kernel[filt][j]; } s = (s + 8) >> 4; - Unsafe.Add(ref buffer, i) = (byte)s; + Unsafe.Add(ref buffer, i) = T.CreateChecked(s); } } @@ -1035,18 +1158,19 @@ internal class Av1PredictionDecoder return strength; } - private static void FilterIntraEdgeCorner(Span above, Span left) + private static void FilterIntraEdgeCorner(Span above, Span left) + where T : unmanaged, IBinaryInteger { int[] kernel = [5, 6, 5]; - ref byte aboveRef = ref above[0]; - ref byte leftRef = ref left[0]; - ref byte abovePreviousRef = ref Unsafe.Subtract(ref aboveRef, 1); - ref byte leftPreviousRef = ref Unsafe.Subtract(ref leftRef, 1); - int s = (leftRef * kernel[0]) + (abovePreviousRef * kernel[1]) + (aboveRef * kernel[2]); + ref T aboveRef = ref above[0]; + ref T leftRef = ref left[0]; + ref T abovePreviousRef = ref Unsafe.Subtract(ref aboveRef, 1); + ref T leftPreviousRef = ref Unsafe.Subtract(ref leftRef, 1); + int s = (int.CreateChecked(leftRef) * kernel[0]) + (int.CreateChecked(abovePreviousRef) * kernel[1]) + (int.CreateChecked(aboveRef) * kernel[2]); s = (s + 8) >> 4; - abovePreviousRef = (byte)s; - leftPreviousRef = (byte)s; + abovePreviousRef = T.CreateChecked(s); + leftPreviousRef = T.CreateChecked(s); } private static bool GetFilterType(Av1PartitionInfo partitionInfo, Av1Plane plane) diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs index 230f70a0d..54bb7f4b2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1BlockDecoder.cs @@ -67,10 +67,9 @@ internal class Av1BlockDecoder int chromaTransformUnitCount = isLosslessBlock ? (maxBlocksWide * maxBlocksHigh) >> ((colorConfig.SubSamplingX ? 1 : 0) + (colorConfig.SubSamplingY ? 1 : 0)) : modeInfo.TransformUnitsCount[(int)Av1Plane.U]; - bool highBitDepth = false; - bool is16BitsPipeline = false; + bool highBitDepth = this.frameBuffer.BytesPerSample == 2; int loopFilterStride = this.frameHeader.ModeInfoStride; - Av1PredictionDecoder predictionDecoder = new(this.sequenceHeader, this.frameHeader, false); + Av1PredictionDecoder predictionDecoder = new(this.sequenceHeader, this.frameHeader); Av1InverseQuantizer inverseQuantizer = new(this.sequenceHeader, this.frameHeader); for (int plane = 0; plane < colorConfig.PlaneCount; plane++) @@ -108,17 +107,36 @@ internal class Av1BlockDecoder Point pixelPosition = new( (modeInfoPosition.X >> subX) << Av1Constants.ModeInfoSizeLog2, (modeInfoPosition.Y >> subY) << Av1Constants.ModeInfoSizeLog2); - Span blockReconstructionBuffer = this.frameBuffer.DeriveBlockPointer((Av1Plane)plane, pixelPosition, subX, subY, out int reconstructionStride); + Span blockReconstructionBuffer = default; + Span highBitDepthBlockReconstructionBuffer = default; + int reconstructionStride; + if (highBitDepth) + { + highBitDepthBlockReconstructionBuffer = this.frameBuffer.DeriveBlockPointer16((Av1Plane)plane, pixelPosition, subX, subY, out reconstructionStride); + } + else + { + blockReconstructionBuffer = this.frameBuffer.DeriveBlockPointer((Av1Plane)plane, pixelPosition, subX, subY, out reconstructionStride); + } + for (int tu = 0; tu < transformUnitCount; tu++) { - Span transformBlockReconstructionBuffer; + Span transformBlockReconstructionBuffer = default; + Span highBitDepthTransformBlockReconstructionBuffer = default; int transformBlockOffset; transformSize = transformInfo[0].Size; Span coefficients = superblockInfo.GetCoefficients((Av1Plane)plane)[this.currentCoefficientIndex[plane]..]; transformBlockOffset = ((transformInfo[0].OffsetY * reconstructionStride) + transformInfo[0].OffsetX) << Av1Constants.ModeInfoSizeLog2; - transformBlockReconstructionBuffer = blockReconstructionBuffer.Slice(transformBlockOffset << (highBitDepth ? 1 : 0)); + if (highBitDepth) + { + highBitDepthTransformBlockReconstructionBuffer = highBitDepthBlockReconstructionBuffer[transformBlockOffset..]; + } + else + { + transformBlockReconstructionBuffer = blockReconstructionBuffer[transformBlockOffset..]; + } if (this.isLoopFilterEnabled) { @@ -141,16 +159,32 @@ internal class Av1BlockDecoder // if (!inter_block) { // SVT: svt_av1_predict_intra - predictionDecoder.Decode( - partitionInfo, - (Av1Plane)plane, - transformSize, - tileInfo, - transformBlockReconstructionBuffer, - reconstructionStride, - this.frameBuffer.BitDepth, - transformInfo[0].OffsetX, - transformInfo[0].OffsetY); + if (highBitDepth) + { + predictionDecoder.Decode( + partitionInfo, + (Av1Plane)plane, + transformSize, + tileInfo, + highBitDepthTransformBlockReconstructionBuffer, + reconstructionStride, + this.frameBuffer.BitDepth, + transformInfo[0].OffsetX, + transformInfo[0].OffsetY); + } + else + { + predictionDecoder.Decode( + partitionInfo, + (Av1Plane)plane, + transformSize, + tileInfo, + transformBlockReconstructionBuffer, + reconstructionStride, + this.frameBuffer.BitDepth, + transformInfo[0].OffsetX, + transformInfo[0].OffsetY); + } } int numberOfCoefficients = 0; @@ -169,7 +203,21 @@ internal class Av1BlockDecoder { this.currentCoefficientIndex[plane] += numberOfCoefficients + 1; - if (this.frameBuffer.BitDepth == Av1BitDepth.EightBit && !is16BitsPipeline) + if (highBitDepth) + { + // AV1 high-bit-depth reconstruction stores unsigned samples in the existing signed 16-bit transform representation. + Av1InverseTransformer.ReconstructHighBitDepth( + quantizationCoefficients, + highBitDepthTransformBlockReconstructionBuffer, + reconstructionStride, + transformSize, + transformType, + plane, + numberOfCoefficients, + isLossless, + this.frameBuffer.BitDepth); + } + else { // SVT: svt_aom_inv_transform_recon8bit Av1InverseTransformer.Reconstruct8Bit( @@ -182,10 +230,6 @@ internal class Av1BlockDecoder numberOfCoefficients, isLossless); } - else - { - throw new NotImplementedException("No support for 16 bit pipeline yet."); - } } } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs index fbcb41efe..bc85d6ebd 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TilingTests.cs @@ -54,6 +54,37 @@ public class Av1TilingTests Assert.True(frameBuffer.BufferY.DangerousGetSingleSpan().ContainsAnyExcept(0)); } + [Theory] + [InlineData((int)Av1BitDepth.TenBit, 1023)] + [InlineData((int)Av1BitDepth.TwelveBit, 4095)] + public void DecodePixelsFirstTileThroughHighBitDepthPipeline(int bitDepthIndex, ushort maximum) + { + string filePath = Path.Combine(TestEnvironment.InputImagesDirectoryFullPath, TestImages.Heif.Orange4x4); + byte[] content = File.ReadAllBytes(filePath); + const int dataOffset = 0x010E; + const int dataSize = 0x001D; + const int tileOffset = 21; + Span headerSpan = content.AsSpan(dataOffset, dataSize); + Span tileSpan = content.AsSpan(tileOffset, dataSize - tileOffset); + Av1BitStreamReader bitStreamReader = new(headerSpan); + IAv1TileReader stub = new Av1TileDecoderStub(); + ObuReader obuReader = new(); + obuReader.ReadAll(ref bitStreamReader, dataSize, () => stub); + + // Reuse known-good tile syntax after parsing so this test isolates native high-bit prediction and reconstruction wiring. + obuReader.SequenceHeader.ColorConfig.BitDepth = (Av1BitDepth)bitDepthIndex; + using Av1FrameBuffer frameBuffer = new(Configuration.Default, obuReader.SequenceHeader, Av1ColorFormat.Yuv444, false); + Av1FrameInfo frameInfo = new(obuReader.SequenceHeader); + Av1FrameDecoder frameDecoder = new(obuReader.SequenceHeader, obuReader.FrameHeader, frameInfo, frameBuffer); + Av1TileReader tileReader = new(Configuration.Default, obuReader.SequenceHeader, obuReader.FrameHeader, frameDecoder); + + tileReader.ReadTile(tileSpan, 0); + + Span yRow = frameBuffer.GetHighBitDepthRowSpan(Av1Plane.Y, 0, 0, 0); + Assert.True(yRow[..4].ContainsAnyExcept(0)); + Assert.All(yRow[..4].ToArray(), value => Assert.InRange(value, (ushort)0, maximum)); + } + [Theory] [InlineData(TestImages.Heif.XnConvert, 0x010E, 0x03CC, 18, 16)] [InlineData(TestImages.Heif.Orange4x4, 0x010E, 0x001d, 21, 1)]