diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 746dca51f..21758d3cc 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -245,6 +245,7 @@ This snapshot pins or classifies the available references and failures; it does | `HevcCabacContext` and `HevcCabacDecoder` | HEVC sections 9.3.3.2 through 9.3.3.4 context initialization, arithmetic decoding, bypass decoding, and termination | HM `source/Lib/TLibCommon/ContextModel.cpp`, `source/Lib/TLibCommon/TComCABACTables.cpp`, and `source/Lib/TLibDecoder/TDecBinCoderCABAC.cpp` at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` `common/ihevc_cabac_tables.c`, `common/ihevc_cabac_tables.h`, `decoder/ihevcd_cabac.c`, and `decoder/ihevcd_cabac.h` at `c83a76b084498d55f252f48b2e3786804cdf24b7` | Implement the bit-exact scalar probability-state transitions, range normalization, bounded byte refills, bypass modes, termination, and alignment validation for one slice tile or wavefront substream. CABAC decisions are serial and adaptive, so this primitive is the scalar oracle rather than a SIMD target; later prediction, transform, filtering, and color-conversion stages must reuse ImageSharp's compatible SIMD kernels. The decoder owns no container boxes, access units, reference pictures, tracks, timing, playback, or general-purpose bitstream surface. | | `HevcCabacContexts` | HEVC section 9.3.2.2 initialization of context variables for an intra-coded slice and Range Extensions context additions | HM `source/Lib/TLibCommon/ContextTables.h` and `source/Lib/TLibDecoder/TDecSbac.cpp` function `resetEntropy` at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` `common/ihevc_cabac_tables.c` and `decoder/ihevcd_cabac.c` at `c83a76b084498d55f252f48b2e3786804cdf24b7` | Initialize only the context groups reachable from the independently coded intra-picture syntax, including transform skip, chroma quantization adjustment, and cross-component prediction. Keep one contiguous mutable context set per entropy substream so tile and wavefront synchronization can copy the normative adaptive state without retaining reference-picture or playback state. Omit skip, merge, motion, reference-index, and other inter-picture-only groups. | | `HevcCabacSyntaxReader` | HEVC sections 9.3.3 through 9.3.4 CABAC binarization and context selection for intra coding units, transform trees, residual coefficients, sample-adaptive offset, and Range Extensions tools | HM `source/Lib/TLibDecoder/TDecSbac.cpp`, `source/Lib/TLibCommon/TComChromaFormat.h`, and `source/Lib/TLibCommon/TComTrQuant.cpp` at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` `decoder/ihevcd_cabac.c`, `decoder/ihevcd_parse_slice.c`, and `decoder/ihevcd_parse_residual.c` at `c83a76b084498d55f252f48b2e3786804cdf24b7` | Decode only syntax values reachable while reconstructing the selected independently coded still picture: intra partition and mode selectors, transform splitting and coded-block flags, quantization adjustments, transform skip, coefficient significance and levels, cross-component prediction, SAO, bypass alignment, and termination. Keep neighbor-dependent context selection in the owning coding-tree/reconstruction layer and expose no inter prediction, reference-picture, access-unit, track, timing, or playback syntax. | +| `HevcCoefficientScanOrder`, `HevcCoefficientCodingParameters`, and `HevcCoefficientDecoder` | HEVC sections 9.3.4.2.4 through 9.3.4.2.8 last-position, significance-map, level, sign-data-hiding, and Rice-adaptation processes; Range Extensions transform-skip context and bypass-alignment rules | HM `source/Lib/TLibCommon/TComRom.cpp` grouped scan generation, `source/Lib/TLibCommon/TComChromaFormat.cpp` transform-unit entropy parameters, `source/Lib/TLibCommon/TComTrQuant.cpp` significance-context derivation, and `source/Lib/TLibDecoder/TDecSbac.cpp` function `parseCoeffNxN` at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` `decoder/ihevcd_parse_residual.c` at `c83a76b084498d55f252f48b2e3786804cdf24b7`; ImageSharp `MemoryAllocator` and fixed inline buffers | Generate diagonal, horizontal, and vertical grouped scans into entropy-substream-owned scratch; derive directional scan selection, last-significant positions, right/lower group contexts, significant coefficient flags, greater-than-one/two levels, escape remainders, sign hiding, and the four persistent Rice states. Allocate one pooled scratch block per entropy substream and perform no allocation or dynamic stack growth per transform block. This serial adaptive stage intentionally remains scalar and feeds the existing SIMD-first inverse quantization, inverse transform, transform-skip, and residual-reconstruction kernels. | | `HevcPlane` and `HevcPictureBuffer` | HEVC sections 6.2 and 6.3 source and decoded picture sample-array dimensions for monochrome, 4:2:0, 4:2:2, 4:4:4, and separate-color-plane coding | HM `source/Lib/TLibCommon/TComPicYuv.cpp` at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` decoded-picture buffer layout paths at `c83a76b084498d55f252f48b2e3786804cdf24b7`; ImageSharp `MemoryAllocator` and `Buffer2D` | Allocate one allocator-owned native `ushort` plane set for the selected still picture so the same reconstruction path preserves every supported 8-through-16-bit sample without byte-backed reinterpretation. Derive chroma dimensions by ceiling division for each sampling layout and treat separate color planes as full-resolution independently coded arrays. The buffer contains one picture only and introduces no decoded-picture buffer, reference lifetime, frame queue, or playback state. | | `HevcCodingTreeState` | HEVC sections 6.4 coding-tree block and coding-block availability plus section 9.3 split-flag context derivation | HM `source/Lib/TLibCommon/TComDataCU.cpp` function `getCtxSplitFlag` and `source/Lib/TLibDecoder/TDecCu.cpp` coding-tree traversal at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` `decoder/ihevcd_parse_slice.c` coding-tree paths at `c83a76b084498d55f252f48b2e3786804cdf24b7`; ImageSharp `MemoryAllocator` and `Buffer2D` | Store only leaf depth, effective luma QP, transquant-bypass, and PCM state at minimum-coding-block resolution for the selected picture. Derive split contexts from caller-approved left and above availability so slice and tile boundaries remain owned by traversal rather than hidden in a general block graph. Clip edge writes to the coded still-picture state and add no prediction units, motion fields, references, decoded-picture queue, or sequence lifetime. | | `HevcIntraPredictionState` | HEVC sections 8.4.2 and 9.3 intra luma/chroma prediction-mode derivation and binarization | HM `source/Lib/TLibCommon/TComDataCU.cpp` functions `getIntraDirPredictor` and `getAllowedChromaDir`, plus `source/Lib/TLibDecoder/TDecSbac.cpp` functions `parseIntraDirLumaAng` and `parseIntraDirChroma`, at `9c1f298659ab0cee9dc13d23d0304221575410b9`; Android `libhevc` intra-mode parsing and neighbor derivation paths at `c83a76b084498d55f252f48b2e3786804cdf24b7`; ImageSharp `MemoryAllocator` and `Buffer2D` | Preserve the normative two-pass luma flag/suffix order, spatial most-probable-mode derivation, omitted-mode reinsertion, four-way minimum-CU partition order, explicit chroma candidate substitution, and derived-chroma mode at 4x4 luma resolution. Accept slice/tile availability from traversal, fill only the selected still-picture map, and add no inter prediction unit, motion, reference, or sequence state. | @@ -289,7 +290,7 @@ This assessment is based on the current source after the upstream ImageSharp mer ### HEVC decoder and encoder - `Heif4CharCode` recognizes `hvc1` image items, `HevcCodecConfiguration` validates and associates the bounded `hvcC` property, and Identify reports its HEVC precision and monochrome shape, but `HeifCompressionFactory` has no HEVC item decoder. -- The HEVC path now validates NAL headers and RBSP escaping, parses and links `hvcC` VPS/SPS/PPS structures through Range Extensions, and parses the selected image item's length-delimited base-layer IDR slice headers and entropy payload boundaries. Its scalar CABAC primitive implements the exact adaptive state transitions, arithmetic range normalization, bypass modes, bounded refills, termination, and stop-bit validation from the pinned HM reference. The one-allocation context owner initializes every intra-picture and Range Extensions context that still-image reconstruction can select while omitting inter-picture-only groups. The codec-local syntax reader decodes the intra coding-unit, transform, coefficient, SAO, and Range Extensions binarizations. Allocator-owned native `ushort` picture planes cover monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit reconstruction without a decoded-picture queue. Coding-tree traversal and neighbor-derived context selection are not yet wired. The path is deliberately not registered as an item decoder until coding-tree reconstruction, still-image intra prediction, inverse quantization/transform, deblocking, sample-adaptive offset, color conversion, and pixel output are implemented. +- The HEVC path now validates NAL headers and RBSP escaping, parses and links `hvcC` VPS/SPS/PPS structures through Range Extensions, and parses the selected image item's length-delimited base-layer IDR slice headers and entropy payload boundaries. Its scalar CABAC primitive implements the exact adaptive state transitions, arithmetic range normalization, bypass modes, bounded refills, termination, and stop-bit validation from the pinned HM reference. The one-allocation context owner initializes every intra-picture and Range Extensions context that still-image reconstruction can select while omitting inter-picture-only groups. The codec-local syntax reader decodes the intra coding-unit, transform, coefficient, SAO, and Range Extensions binarizations. Coefficient decoding now covers grouped directional scans, last-significant and significance contexts, greater-than-one/two and escape levels, sign hiding, bypass alignment, and persistent Rice adaptation from one pooled entropy-substream scratch owner. Allocator-owned native `ushort` picture planes cover monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit reconstruction without a decoded-picture queue. Coding-tree and transform-unit traversal are not yet wired. The path is deliberately not registered as an item decoder until coding-tree reconstruction, still-image intra prediction, inverse quantization/transform, deblocking, sample-adaptive offset, color conversion, and pixel output are implemented. - There is no HEVC encoder. The current HEIC-branded encoder writes a legacy JPEG payload and therefore cannot provide HEIC output. - Existing HEVC tests prove container identification only; they do not decode or compare HEIC pixels. @@ -477,6 +478,8 @@ Implement and verify in dependency order: - [x] Implement allocation-free SIMD-first transform-skip normalization, complete-block coefficient rotation, transquant-bypass copying, implicit intra-direction selection, and horizontal/vertical inverse residual DPCM with a scalar fallback and signed residual clipping. - [ ] Decode explicit inter residual-DPCM modes and connect bypass, transform skip, residual DPCM, prediction addition, and lossless reconstruction through transform-unit traversal. - [ ] Connect coefficient decoding, inverse quantization, transform selection, reusable scratch, and add/clip to transform-unit traversal. + - [x] Implement grouped diagonal/horizontal/vertical scans, last-significant and significance context derivation, coefficient levels and signs, sign-data hiding, Range Extensions bypass alignment, four persistent Rice states, and one pooled scratch owner with no per-transform allocation or `stackalloc`. + - [ ] Invoke coefficient decoding and the existing reconstruction kernels from transform-unit traversal, then verify complete transform blocks from independently encoded HEIC fixtures. - [ ] Deblocking and sample-adaptive offset for every signaled luma/chroma and bit-depth path. - [ ] Tiles, wavefront entry points, dependent slices, and all other parallelization syntax permitted by the exposed still-image profiles. - [ ] Supplemental enhancement information that changes image presentation or metadata exposed by ImageSharp. @@ -575,6 +578,7 @@ Tasks: - [x] Add a permanent frame-wide HEVC inverse-transform benchmark. On .NET 10, dense-coefficient 32x32 twelve-bit inverse DCT, transposition, and add/clip measured 3.643 milliseconds per padded 1920x1088 frame, compared with 45.23 milliseconds with hardware intrinsics disabled: 12.4 times faster with zero managed allocations. - [x] Add a permanent frame-wide HEVC inverse-quantization benchmark. On .NET 10, dense 32x32 twelve-bit flat and scaling-list paths measured 69.98 and 238.4 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 1.469 and 1.609 milliseconds: 21.0 and 6.7 times faster with zero managed allocations. Pre-expanding the scaling matrices once reduced the SIMD scaling-list path from 4.974 milliseconds to 238.4 microseconds. - [x] Add a permanent frame-wide HEVC residual-reconstruction benchmark. On .NET 10, dense 32x32 twelve-bit transform skip, horizontal RDPCM, and vertical RDPCM measured 79.64, 275.61, and 110.81 microseconds per padded 1920x1088 frame, compared with forced-scalar timings of 571.5 microseconds, 1.512 milliseconds, and 1.257 milliseconds: 7.2, 5.5, and 11.3 times faster with zero managed allocations. + - [x] Add a permanent frame-wide HEVC grouped coefficient-scan benchmark. On .NET 10, generating the complete diagonal 32x32 scan for all 2,040 transform blocks in a padded 1920x1088 frame measured 1.432 milliseconds with zero managed allocations. CABAC decisions remain serial and require an independently encoded complete-slice workload after transform-unit traversal is connected. - [ ] Implement vector paths only for confirmed hot loops, using existing `Vector128`, `Vector256`, and `Vector512` helper and dispatch patterns where supported. - [ ] Prioritize shared color conversion and pixel packing, chroma upsampling, inverse-transform add-and-clip, intra predictors, HEVC deblock/SAO, AV1 loop filter/CDEF/restoration, and contiguous grid copies. - [ ] Benchmark the complete decode color pipeline on representative 8/10/12-bit AVIF and HEIC images with and without embedded ICC profiles. Report absolute end-to-end timings and allocations in addition to the isolated YUV/CICP and ICC stage costs. diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContexts.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContexts.cs index 5dfede1af..096ba6756 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContexts.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacContexts.cs @@ -114,9 +114,27 @@ internal sealed class HevcCabacContexts private const int ContextCount = 178; /// - /// The HEVC intra-slice initialization values in the same order as the owned context ranges. + /// The contiguous adaptive context storage owned by the entropy substream. + /// + private readonly HevcCabacContext[] contexts; + + /// + /// Initializes a new instance of the class for an intra-coded slice. + /// + /// The slice luma quantization parameter. + public HevcCabacContexts(int quantizationParameter) + { + this.contexts = new HevcCabacContext[ContextCount]; + for (int index = 0; index < this.contexts.Length; index++) + { + this.contexts[index] = new HevcCabacContext(quantizationParameter, IntraInitializationValues[index]); + } + } + + /// + /// Gets the HEVC intra-slice initialization values in the same order as the owned context ranges. /// - private static readonly byte[] IntraInitializationValues = + private static ReadOnlySpan IntraInitializationValues => [ // cu_transquant_bypass_flag @@ -178,24 +196,6 @@ internal sealed class HevcCabacContexts 154, 154, 154, 154, 154, 154, 154, 154, 154, 154 ]; - /// - /// The contiguous adaptive context storage owned by the entropy substream. - /// - private readonly HevcCabacContext[] contexts; - - /// - /// Initializes a new instance of the class for an intra-coded slice. - /// - /// The slice luma quantization parameter. - public HevcCabacContexts(int quantizationParameter) - { - this.contexts = new HevcCabacContext[ContextCount]; - for (int index = 0; index < this.contexts.Length; index++) - { - this.contexts[index] = new HevcCabacContext(quantizationParameter, IntraInitializationValues[index]); - } - } - /// /// Gets the coding-unit transquant-bypass context. /// diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacDecoder.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacDecoder.cs index 2fa8d9992..5b21e0a6f 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcCabacDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCabacDecoder.cs @@ -8,44 +8,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// internal ref struct HevcCabacDecoder { - /// - /// The least-probable-symbol subrange for each probability state and current range class. - /// - private static readonly byte[] LeastProbableSymbolRanges = - [ - 128, 176, 208, 240, 128, 167, 197, 227, 128, 158, 187, 216, 123, 150, 178, 205, - 116, 142, 169, 195, 111, 135, 160, 185, 105, 128, 152, 175, 100, 122, 144, 166, - 95, 116, 137, 158, 90, 110, 130, 150, 85, 104, 123, 142, 81, 99, 117, 135, - 77, 94, 111, 128, 73, 89, 105, 122, 69, 85, 100, 116, 66, 80, 95, 110, - 62, 76, 90, 104, 59, 72, 86, 99, 56, 69, 81, 94, 53, 65, 77, 89, - 51, 62, 73, 85, 48, 59, 69, 80, 46, 56, 66, 76, 43, 53, 63, 72, - 41, 50, 59, 69, 39, 48, 56, 65, 37, 45, 54, 62, 35, 43, 51, 59, - 33, 41, 48, 56, 32, 39, 46, 53, 30, 37, 43, 50, 29, 35, 41, 48, - 27, 33, 39, 45, 26, 31, 37, 43, 24, 30, 35, 41, 23, 28, 33, 39, - 22, 27, 32, 37, 21, 26, 30, 35, 20, 24, 29, 33, 19, 23, 27, 31, - 18, 22, 26, 30, 17, 21, 25, 28, 16, 20, 23, 27, 15, 19, 22, 25, - 14, 18, 21, 24, 14, 17, 20, 23, 13, 16, 19, 22, 12, 15, 18, 21, - 12, 14, 17, 20, 11, 14, 16, 19, 11, 13, 15, 18, 10, 12, 15, 17, - 10, 12, 14, 16, 9, 11, 13, 15, 9, 11, 12, 14, 8, 10, 12, 14, - 8, 9, 11, 13, 7, 9, 11, 12, 7, 9, 10, 12, 7, 8, 10, 11, - 6, 8, 9, 11, 6, 7, 9, 10, 6, 7, 8, 9, 2, 2, 2, 2 - ]; - - /// - /// The normalization shift for each quantized least-probable-symbol range. - /// - private static readonly byte[] LeastProbableSymbolNormalizationShifts = - [ - 6, 5, 4, 4, - 3, 3, 3, 3, - 2, 2, 2, 2, - 2, 2, 2, 2, - 1, 1, 1, 1, - 1, 1, 1, 1, - 1, 1, 1, 1, - 1, 1, 1, 1 - ]; - /// /// The complete bounded entropy-substream bytes. /// @@ -90,6 +52,44 @@ internal ref struct HevcCabacDecoder this.bitsNeeded = -8; } + /// + /// Gets the least-probable-symbol subrange for each probability state and current range class. + /// + private static ReadOnlySpan LeastProbableSymbolRanges => + [ + 128, 176, 208, 240, 128, 167, 197, 227, 128, 158, 187, 216, 123, 150, 178, 205, + 116, 142, 169, 195, 111, 135, 160, 185, 105, 128, 152, 175, 100, 122, 144, 166, + 95, 116, 137, 158, 90, 110, 130, 150, 85, 104, 123, 142, 81, 99, 117, 135, + 77, 94, 111, 128, 73, 89, 105, 122, 69, 85, 100, 116, 66, 80, 95, 110, + 62, 76, 90, 104, 59, 72, 86, 99, 56, 69, 81, 94, 53, 65, 77, 89, + 51, 62, 73, 85, 48, 59, 69, 80, 46, 56, 66, 76, 43, 53, 63, 72, + 41, 50, 59, 69, 39, 48, 56, 65, 37, 45, 54, 62, 35, 43, 51, 59, + 33, 41, 48, 56, 32, 39, 46, 53, 30, 37, 43, 50, 29, 35, 41, 48, + 27, 33, 39, 45, 26, 31, 37, 43, 24, 30, 35, 41, 23, 28, 33, 39, + 22, 27, 32, 37, 21, 26, 30, 35, 20, 24, 29, 33, 19, 23, 27, 31, + 18, 22, 26, 30, 17, 21, 25, 28, 16, 20, 23, 27, 15, 19, 22, 25, + 14, 18, 21, 24, 14, 17, 20, 23, 13, 16, 19, 22, 12, 15, 18, 21, + 12, 14, 17, 20, 11, 14, 16, 19, 11, 13, 15, 18, 10, 12, 15, 17, + 10, 12, 14, 16, 9, 11, 13, 15, 9, 11, 12, 14, 8, 10, 12, 14, + 8, 9, 11, 13, 7, 9, 11, 12, 7, 9, 10, 12, 7, 8, 10, 11, + 6, 8, 9, 11, 6, 7, 9, 10, 6, 7, 8, 9, 2, 2, 2, 2, + ]; + + /// + /// Gets the normalization shift for each quantized least-probable-symbol range. + /// + private static ReadOnlySpan LeastProbableSymbolNormalizationShifts => + [ + 6, 5, 4, 4, + 3, 3, 3, 3, + 2, 2, 2, 2, + 2, 2, 2, 2, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + 1, 1, 1, 1, + ]; + /// /// Gets the number of whole entropy-substream bytes loaded into the arithmetic decoder. /// diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientCodingParameters.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientCodingParameters.cs new file mode 100644 index 000000000..a92f60135 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientCodingParameters.cs @@ -0,0 +1,343 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Contains the immutable entropy-coding parameters for one HEVC transform block. +/// +internal readonly struct HevcCoefficientCodingParameters +{ + /// + /// Initializes a new instance of the struct. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// The coefficient scan selected for the block. + /// Whether transform skip or transquant bypass selects the single significance context. + /// Whether the first coefficient sign in an eligible group is inferred. + /// Whether Rice parameters adapt across transform blocks. + /// Whether coefficient bypass data is byte aligned when escape data is present. + /// Whether coefficient remainders use the bounded extended-precision prefix. + /// The component transform dynamic range excluding its sign bit. + /// The luma/chroma and transformed/non-transformed Rice statistics selector. + public HevcCoefficientCodingParameters( + int width, + int height, + HevcPlane plane, + HevcCoefficientScanType scanType, + bool useSingleSignificanceContext, + bool signDataHidingEnabled, + bool persistentRiceAdaptationEnabled, + bool cabacBypassAlignmentEnabled, + bool extendedPrecisionProcessingEnabled, + int maximumLog2TransformDynamicRange, + int riceStatisticsIndex) + { + this.Width = width; + this.Height = height; + this.Plane = plane; + this.ScanType = scanType; + this.FirstSignificanceMapContext = GetFirstSignificanceMapContext(width, height, plane != HevcPlane.Y, scanType, useSingleSignificanceContext); + this.SignDataHidingEnabled = signDataHidingEnabled; + this.PersistentRiceAdaptationEnabled = persistentRiceAdaptationEnabled; + this.CabacBypassAlignmentEnabled = cabacBypassAlignmentEnabled; + this.ExtendedPrecisionProcessingEnabled = extendedPrecisionProcessingEnabled; + this.MaximumLog2TransformDynamicRange = maximumLog2TransformDynamicRange; + this.RiceStatisticsIndex = riceStatisticsIndex; + } + + /// + /// Gets the transform-block width. + /// + public int Width { get; } + + /// + /// Gets the transform-block height. + /// + public int Height { get; } + + /// + /// Gets the reconstructed component. + /// + public HevcPlane Plane { get; } + + /// + /// Gets the coefficient scan selected for the block. + /// + public HevcCoefficientScanType ScanType { get; } + + /// + /// Gets the first significant-coefficient context within the component context set. + /// + public int FirstSignificanceMapContext { get; } + + /// + /// Gets a value indicating whether an eligible first coefficient sign is inferred from the group parity. + /// + public bool SignDataHidingEnabled { get; } + + /// + /// Gets a value indicating whether Rice parameters adapt across transform blocks. + /// + public bool PersistentRiceAdaptationEnabled { get; } + + /// + /// Gets a value indicating whether coefficient bypass data is byte aligned when escape data is present. + /// + public bool CabacBypassAlignmentEnabled { get; } + + /// + /// Gets a value indicating whether coefficient remainders use the bounded extended-precision prefix. + /// + public bool ExtendedPrecisionProcessingEnabled { get; } + + /// + /// Gets the component transform dynamic range excluding its sign bit. + /// + public int MaximumLog2TransformDynamicRange { get; } + + /// + /// Gets the luma/chroma and transformed/non-transformed Rice statistics selector. + /// + public int RiceStatisticsIndex { get; } + + /// + /// Gets the raster-position context mapping for a 4 by 4 transform block. + /// + private static ReadOnlySpan SignificanceContexts4x4 => + [ + 0, 1, 4, 5, + 2, 3, 4, 5, + 6, 6, 8, 8, + 7, 7, 8, 8, + ]; + + /// + /// Creates the coefficient parameters selected by the active sequence, picture, and transform-unit state. + /// + /// The active picture parameters. + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// Whether the containing coding unit uses intra prediction. + /// The effective intra prediction mode, or a value ignored for inter prediction. + /// Whether the transform block bypasses the inverse transform. + /// Whether the coding unit bypasses inverse quantization and inverse transform. + /// The residual differential-pulse-code-modulation mode selected for the block. + /// The coefficient entropy-coding parameters for the transform block. + public static HevcCoefficientCodingParameters Create( + HevcPictureParameterSet pictureParameterSet, + int width, + int height, + HevcPlane plane, + bool isIntra, + int intraPredictionMode, + bool transformSkip, + bool transquantBypass, + HevcResidualDpcmMode residualDpcmMode) + { + HevcSequenceParameterSet sequenceParameterSet = pictureParameterSet.SequenceParameterSet; + bool isChroma = plane != HevcPlane.Y; + bool nonTransformed = transformSkip || transquantBypass; + HevcCoefficientScanType scanType = SelectScanType( + width, + height, + plane, + isIntra, + intraPredictionMode, + sequenceParameterSet.ChromaFormat, + sequenceParameterSet.SeparateColorPlaneFlag); + + return new HevcCoefficientCodingParameters( + width, + height, + plane, + scanType, + sequenceParameterSet.TransformSkipContextEnabled && nonTransformed, + pictureParameterSet.SignDataHidingEnabled && !transquantBypass && residualDpcmMode == HevcResidualDpcmMode.None, + sequenceParameterSet.PersistentRiceAdaptationEnabled, + sequenceParameterSet.CabacBypassAlignmentEnabled, + sequenceParameterSet.ExtendedPrecisionProcessingEnabled, + sequenceParameterSet.GetMaxTransformDynamicRange(plane), + (isChroma ? 2 : 0) + (nonTransformed ? 1 : 0)); + } + + /// + /// Selects the scan direction from transform geometry and the effective intra prediction direction. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// Whether the containing coding unit uses intra prediction. + /// The effective intra prediction mode. + /// The sequence chroma-format identifier. + /// Whether each 4:4:4 component is coded as an independent color plane. + /// The selected coefficient scan. + public static HevcCoefficientScanType SelectScanType( + int width, + int height, + HevcPlane plane, + bool isIntra, + int intraPredictionMode, + byte chromaFormat, + bool separateColorPlane) + { + if (!isIntra) + { + return HevcCoefficientScanType.Diagonal; + } + + bool isSubsampledChroma = plane != HevcPlane.Y && !separateColorPlane; + int subsamplingX = isSubsampledChroma && chromaFormat is 1 or 2 ? 1 : 0; + int subsamplingY = isSubsampledChroma && chromaFormat == 1 ? 1 : 0; + if (width > (8 >> subsamplingX) || height > (8 >> subsamplingY)) + { + return HevcCoefficientScanType.Diagonal; + } + + int mode = plane != HevcPlane.Y && chromaFormat == 2 && !separateColorPlane + ? HevcIntraPredictionMode.RemapChroma422(intraPredictionMode) + : intraPredictionMode; + + // Modes close to vertical place correlated residuals along rows, while modes close to horizontal use the + // transposed column scan. All other modes retain the diagonal scan. + if (Math.Abs(mode - HevcIntraPredictionMode.Vertical) <= 4) + { + return HevcCoefficientScanType.Horizontal; + } + + return Math.Abs(mode - HevcIntraPredictionMode.Horizontal) <= 4 + ? HevcCoefficientScanType.Vertical + : HevcCoefficientScanType.Diagonal; + } + + /// + /// Derives the coded-sub-block significance context from already decoded right and lower groups. + /// + /// The raster-ordered significant-group flags. + /// The current group horizontal coordinate. + /// The current group vertical coordinate. + /// Zero when neither neighbor is significant; otherwise, one. + public int GetSignificantGroupContext(ReadOnlySpan groupFlags, int groupX, int groupY) + { + int widthInGroups = this.Width / 4; + int heightInGroups = this.Height / 4; + bool rightSignificant = groupX < widthInGroups - 1 && groupFlags[(groupY * widthInGroups) + groupX + 1] != 0; + bool lowerSignificant = groupY < heightInGroups - 1 && groupFlags[((groupY + 1) * widthInGroups) + groupX] != 0; + return rightSignificant || lowerSignificant ? 1 : 0; + } + + /// + /// Derives the two-bit right-and-lower significance pattern for coefficient contexts. + /// + /// The raster-ordered significant-group flags. + /// The current group horizontal coordinate. + /// The current group vertical coordinate. + /// The right flag in bit zero and the lower flag in bit one. + public int GetSignificancePattern(ReadOnlySpan groupFlags, int groupX, int groupY) + { + int widthInGroups = this.Width / 4; + int heightInGroups = this.Height / 4; + int right = groupX < widthInGroups - 1 && groupFlags[(groupY * widthInGroups) + groupX + 1] != 0 ? 1 : 0; + int lower = groupY < heightInGroups - 1 && groupFlags[((groupY + 1) * widthInGroups) + groupX] != 0 ? 1 : 0; + return right + (lower << 1); + } + + /// + /// Derives the significant-coefficient context from its position and neighboring coefficient groups. + /// + /// The coefficient raster position. + /// The right-and-lower significant-group pattern. + /// The context index within the component significance-map context set. + public int GetSignificantCoefficientContext(int rasterPosition, int significancePattern) + { + bool isChroma = this.Plane != HevcPlane.Y; + if (this.FirstSignificanceMapContext == (isChroma ? 15 : 27)) + { + return this.FirstSignificanceMapContext; + } + + int y = rasterPosition / this.Width; + int x = rasterPosition - (y * this.Width); + if (x + y == 0) + { + return 0; + } + + if (this.Width == 4 && this.Height == 4) + { + return SignificanceContexts4x4[(y * 4) + x]; + } + + int context; + switch (significancePattern) + { + case 0: + int positionInGroup = (x & 3) + (y & 3); + context = positionInGroup >= 3 ? 0 : positionInGroup >= 1 ? 1 : 2; + break; + case 1: + int yInGroup = y & 3; + context = yInGroup >= 2 ? 0 : yInGroup >= 1 ? 1 : 2; + break; + case 2: + int xInGroup = x & 3; + context = xInGroup >= 2 ? 0 : xInGroup >= 1 ? 1 : 2; + break; + default: + context = 2; + break; + } + + bool isBeyondFirstGroup = (x >> 2) + (y >> 2) > 0; + return this.FirstSignificanceMapContext + (isBeyondFirstGroup && !isChroma ? 3 : 0) + context; + } + + /// + /// Selects the greater-than-one and greater-than-two context set for one coefficient group. + /// + /// The coefficient-group scan index. + /// Whether the preceding group ended after finding a coefficient greater than one. + /// The zero-based context set within the component context range. + public int GetLevelContextSet(int subset, bool foundGreaterThanOne) + { + int nonFirstSubsetOffset = this.Plane == HevcPlane.Y && subset > 0 ? 2 : 0; + return nonFirstSubsetOffset + (foundGreaterThanOne ? 1 : 0); + } + + /// + /// Derives the first significant-coefficient context within one component context set. + /// + /// The transform-block width. + /// The transform-block height. + /// Whether the transform block belongs to a chroma channel. + /// The selected coefficient scan. + /// Whether Range Extensions selects the single-context mode. + /// The first significant-coefficient context index. + private static int GetFirstSignificanceMapContext( + int width, + int height, + bool isChroma, + HevcCoefficientScanType scanType, + bool useSingleSignificanceContext) + { + if (useSingleSignificanceContext) + { + return isChroma ? 15 : 27; + } + + if (width == 4 && height == 4) + { + return 0; + } + + if (width == 8 && height == 8) + { + return isChroma ? 9 : scanType == HevcCoefficientScanType.Diagonal ? 9 : 15; + } + + return isChroma ? 12 : 21; + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs new file mode 100644 index 000000000..264f1c602 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs @@ -0,0 +1,389 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Buffers; +using System.Numerics; +using SixLabors.ImageSharp.Memory; + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Decodes HEVC transform coefficients while retaining entropy-substream Rice state and reusable scratch storage. +/// +internal sealed class HevcCoefficientDecoder : IDisposable +{ + /// + /// The maximum coefficient count in a 32 by 32 transform block. + /// + private const int MaximumCoefficientCount = 32 * 32; + + /// + /// The maximum number of 4 by 4 coefficient groups in a transform block. + /// + private const int MaximumCoefficientGroupCount = MaximumCoefficientCount / 16; + + /// + /// The maximum number of significant coefficients in one coefficient group. + /// + private const int CoefficientsPerGroup = 16; + + /// + /// The maximum number of greater-than-one flags coded in one coefficient group. + /// + private const int GreaterThanOneFlagCount = 8; + + /// + /// The minimum scan-position separation that enables sign-data hiding. + /// + private const int SignDataHidingThreshold = 4; + + /// + /// The divisor that converts a persistent adaptation statistic to its Rice parameter. + /// + private const int RiceAdaptationDivisor = 4; + + /// + /// The first scratch index occupied by coefficient-group significance flags. + /// + private const int CoefficientGroupFlagsOffset = MaximumCoefficientCount; + + /// + /// The first scratch index occupied by significant coefficient raster positions. + /// + private const int CoefficientPositionsOffset = CoefficientGroupFlagsOffset + MaximumCoefficientGroupCount; + + /// + /// The first scratch index occupied by absolute coefficient levels. + /// + private const int AbsoluteLevelsOffset = CoefficientPositionsOffset + CoefficientsPerGroup; + + /// + /// The total number of pooled integers used by coefficient decoding. + /// + private const int ScratchLength = AbsoluteLevelsOffset + CoefficientsPerGroup; + + /// + /// The allocator-owned scan and coefficient-group working storage reused for every transform block. + /// + private readonly IMemoryOwner scratchOwner; + + /// + /// The persistent Rice statistics for transformed and non-transformed luma and chroma blocks. + /// + private InlineArray4 riceAdaptationStatistics; + + /// + /// Initializes a new instance of the class for one entropy substream. + /// + /// The configuration providing pooled codec memory. + public HevcCoefficientDecoder(Configuration configuration) + { + this.scratchOwner = configuration.MemoryAllocator.Allocate(ScratchLength); + this.riceAdaptationStatistics = default; + } + + /// + /// Gets the minimum coordinate represented by each last-significant prefix. + /// + private static ReadOnlySpan MinimumCoordinateInGroup => [0, 1, 2, 3, 4, 6, 8, 12, 16, 24]; + + /// + /// Gets the last-significant prefix selected by each transform coordinate. + /// + private static ReadOnlySpan CoordinateGroupIndex => + [ + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + ]; + + /// + /// Decodes one transform block into raster-ordered signed coefficient levels. + /// + /// The current entropy-substream syntax reader. + /// The destination coefficient block. + /// The transform-block coefficient coding parameters. + /// The number of nonzero coefficients decoded into . + public int Decode(ref HevcCabacSyntaxReader reader, Span coefficients, in HevcCoefficientCodingParameters parameters) + { + int width = parameters.Width; + int height = parameters.Height; + int coefficientCount = width * height; + bool isChroma = parameters.Plane != HevcPlane.Y; + coefficients[..coefficientCount].Clear(); + + ReadLastSignificantPosition(ref reader, in parameters, out int lastX, out int lastY); + int lastRasterPosition = (lastY * width) + lastX; + Span scratch = this.scratchOwner.Memory.Span; + Span scan = scratch[..coefficientCount]; + int lastScanPosition = HevcCoefficientScanOrder.Write(scan, width, height, parameters.ScanType, lastRasterPosition); + int groupCount = coefficientCount / CoefficientsPerGroup; + Span significantGroupFlags = scratch.Slice(CoefficientGroupFlagsOffset, groupCount); + Span positions = scratch.Slice(CoefficientPositionsOffset, CoefficientsPerGroup); + Span absoluteLevels = scratch.Slice(AbsoluteLevelsOffset, CoefficientsPerGroup); + significantGroupFlags.Clear(); + + int widthInGroups = width / 4; + int lastSubset = lastScanPosition / CoefficientsPerGroup; + int significantScanPosition = lastScanPosition; + int c1 = 1; + int totalNonZero = 0; + ref int currentRiceStatistic = ref this.riceAdaptationStatistics[parameters.RiceStatisticsIndex]; + + // Coefficient groups are decoded from the last significant position toward DC. This direction makes the + // already decoded right and lower groups available to the significance-context derivation below. + for (int subset = lastSubset; subset >= 0; subset--) + { + int subsetStart = subset * CoefficientsPerGroup; + int riceParameter = currentRiceStatistic / RiceAdaptationDivisor; + bool updateRiceStatistic = parameters.PersistentRiceAdaptationEnabled; + int nonZeroCount = 0; + int lastNonZeroScanPosition = -1; + int firstNonZeroScanPosition = CoefficientsPerGroup; + bool escapeDataPresent = false; + + if (significantScanPosition == lastScanPosition) + { + lastNonZeroScanPosition = significantScanPosition; + firstNonZeroScanPosition = significantScanPosition; + significantScanPosition--; + positions[0] = lastRasterPosition; + nonZeroCount = 1; + } + + int groupRasterPosition = scan[subsetStart]; + int groupY = (groupRasterPosition / width) / 4; + int groupX = (groupRasterPosition % width) / 4; + int groupIndex = (groupY * widthInGroups) + groupX; + if (subset == lastSubset || subset == 0) + { + significantGroupFlags[groupIndex] = 1; + } + else + { + int groupContext = parameters.GetSignificantGroupContext(significantGroupFlags, groupX, groupY); + significantGroupFlags[groupIndex] = reader.ReadSignificantCoefficientGroup(isChroma, groupContext) ? 1 : 0; + } + + int significancePattern = parameters.GetSignificancePattern(significantGroupFlags, groupX, groupY); + for (; significantScanPosition >= subsetStart; significantScanPosition--) + { + int rasterPosition = scan[significantScanPosition]; + bool isSignificant = false; + if (significantGroupFlags[groupIndex] != 0) + { + if (significantScanPosition > subsetStart || subset == 0 || nonZeroCount != 0) + { + int contextIndex = parameters.GetSignificantCoefficientContext(rasterPosition, significancePattern); + isSignificant = reader.ReadSignificantCoefficient(isChroma, contextIndex); + } + else + { + // A coded significant group must contain at least one coefficient. When every later flag is + // zero, the first scan position is therefore inferred rather than consuming another CABAC bin. + isSignificant = true; + } + } + + if (isSignificant) + { + positions[nonZeroCount++] = rasterPosition; + if (lastNonZeroScanPosition < 0) + { + lastNonZeroScanPosition = significantScanPosition; + } + + firstNonZeroScanPosition = significantScanPosition; + } + } + + if (nonZeroCount == 0) + { + continue; + } + + bool hideSign = lastNonZeroScanPosition - firstNonZeroScanPosition >= SignDataHidingThreshold; + int contextSet = parameters.GetLevelContextSet(subset, c1 == 0); + c1 = 1; + absoluteLevels[..nonZeroCount].Fill(1); + int greaterThanOneCount = Math.Min(nonZeroCount, GreaterThanOneFlagCount); + int firstGreaterThanOneIndex = -1; + + for (int index = 0; index < greaterThanOneCount; index++) + { + bool greaterThanOne = reader.ReadCoefficientGreaterThanOne(isChroma, (contextSet * 4) + c1); + if (greaterThanOne) + { + c1 = 0; + if (firstGreaterThanOneIndex < 0) + { + firstGreaterThanOneIndex = index; + } + else + { + escapeDataPresent = true; + } + } + else if (c1 is > 0 and < 3) + { + c1++; + } + + absoluteLevels[index] = greaterThanOne ? 2 : 1; + } + + if (c1 == 0 && firstGreaterThanOneIndex >= 0) + { + bool greaterThanTwo = reader.ReadCoefficientGreaterThanTwo(isChroma, contextSet); + absoluteLevels[firstGreaterThanOneIndex] = greaterThanTwo ? 3 : 2; + escapeDataPresent |= greaterThanTwo; + } + + escapeDataPresent |= nonZeroCount > GreaterThanOneFlagCount; + if (escapeDataPresent && parameters.CabacBypassAlignmentEnabled) + { + reader.AlignBypass(); + } + + int signCount = hideSign && parameters.SignDataHidingEnabled ? nonZeroCount - 1 : nonZeroCount; + uint coefficientSigns = reader.ReadBypassBits(signCount); + int nextSignBit = signCount - 1; + int firstCoefficientAtLeastTwo = 1; + if (escapeDataPresent) + { + for (int index = 0; index < nonZeroCount; index++) + { + int baseLevel = index < GreaterThanOneFlagCount ? 2 + firstCoefficientAtLeastTwo : 1; + if (absoluteLevels[index] == baseLevel) + { + uint remainder = reader.ReadCoefficientRemaining( + riceParameter, + parameters.ExtendedPrecisionProcessingEnabled, + parameters.MaximumLog2TransformDynamicRange); + + ulong decodedLevel = (ulong)remainder + (uint)baseLevel; + if (decodedLevel > int.MaxValue) + { + throw new InvalidImageContentException("The HEVC transform coefficient level is too large."); + } + + absoluteLevels[index] = (int)decodedLevel; + if (decodedLevel > (3UL << riceParameter)) + { + riceParameter = parameters.PersistentRiceAdaptationEnabled ? riceParameter + 1 : Math.Min(riceParameter + 1, 4); + } + + if (updateRiceStatistic) + { + int initialRiceParameter = currentRiceStatistic / RiceAdaptationDivisor; + if (remainder >= (3UL << initialRiceParameter)) + { + currentRiceStatistic++; + } + else if (((ulong)remainder * 2) < (1UL << initialRiceParameter) && currentRiceStatistic > 0) + { + currentRiceStatistic--; + } + + // Only the first escape value in a coefficient group updates persistent state. + updateRiceStatistic = false; + } + } + + if (absoluteLevels[index] >= 2) + { + firstCoefficientAtLeastTwo = 0; + } + } + } + + int absoluteSum = 0; + for (int index = 0; index < nonZeroCount; index++) + { + int rasterPosition = positions[index]; + int level = absoluteLevels[index]; + absoluteSum += level; + if (index == nonZeroCount - 1 && hideSign && parameters.SignDataHidingEnabled) + { + level = (absoluteSum & 1) == 0 ? level : -level; + } + else if (((coefficientSigns >> nextSignBit--) & 1U) != 0) + { + level = -level; + } + + coefficients[rasterPosition] = level; + } + + totalNonZero += nonZeroCount; + } + + return totalNonZero; + } + + /// + /// Releases the allocator-owned coefficient scratch storage. + /// + public void Dispose() => this.scratchOwner.Dispose(); + + /// + /// Decodes the raster coordinates of the final significant coefficient. + /// + /// The current entropy-substream syntax reader. + /// The transform-block coefficient coding parameters. + /// The decoded horizontal coordinate. + /// The decoded vertical coordinate. + private static void ReadLastSignificantPosition( + ref HevcCabacSyntaxReader reader, + in HevcCoefficientCodingParameters parameters, + out int x, + out int y) + { + bool verticalScan = parameters.ScanType == HevcCoefficientScanType.Vertical; + int syntaxWidth = verticalScan ? parameters.Height : parameters.Width; + int syntaxHeight = verticalScan ? parameters.Width : parameters.Height; + bool isChroma = parameters.Plane != HevcPlane.Y; + x = ReadLastSignificantCoordinate(ref reader, isChroma, syntaxWidth, true); + y = ReadLastSignificantCoordinate(ref reader, isChroma, syntaxHeight, false); + if (verticalScan) + { + (x, y) = (y, x); + } + } + + /// + /// Decodes one last-significant coefficient coordinate from its context prefix and bypass suffix. + /// + /// The current entropy-substream syntax reader. + /// Whether the coordinate belongs to a chroma transform block. + /// The transform-block extent along the coded axis. + /// Whether to use the horizontal rather than vertical context set. + /// The decoded zero-based coefficient coordinate. + private static int ReadLastSignificantCoordinate(ref HevcCabacSyntaxReader reader, bool isChroma, int size, bool horizontal) + { + int convertedSize = BitOperations.Log2((uint)size) - 2; + int contextOffset = isChroma ? 0 : (convertedSize * 3) + ((convertedSize + 1) >> 2); + int contextShift = isChroma ? convertedSize : (convertedSize + 3) >> 2; + int maximumPrefix = CoordinateGroupIndex[size - 1]; + int prefix; + for (prefix = 0; prefix < maximumPrefix; prefix++) + { + int contextIndex = contextOffset + (prefix >> contextShift); + bool prefixContinues = horizontal + ? reader.ReadLastSignificantX(isChroma, contextIndex) + : reader.ReadLastSignificantY(isChroma, contextIndex); + + if (!prefixContinues) + { + break; + } + } + + if (prefix <= 3) + { + return prefix; + } + + int suffixLength = (prefix - 2) >> 1; + return MinimumCoordinateInGroup[prefix] + (int)reader.ReadBypassBits(suffixLength); + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanOrder.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanOrder.cs new file mode 100644 index 000000000..75eccfe29 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanOrder.cs @@ -0,0 +1,150 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Writes the grouped coefficient scan used by HEVC residual entropy coding. +/// +internal static class HevcCoefficientScanOrder +{ + /// + /// The width and height of one coefficient group. + /// + private const int CoefficientGroupSize = 4; + + /// + /// Writes the grouped scan for one transform block into caller-owned storage. + /// + /// The destination receiving raster coefficient indices in scan order. + /// The transform-block width. + /// The transform-block height. + /// The scan direction selected for the transform block. + /// The raster index of the last significant coefficient. + /// The scan position of . + public static int Write(Span destination, int width, int height, HevcCoefficientScanType scanType, int lastRasterPosition) + { + int widthInGroups = width / CoefficientGroupSize; + int heightInGroups = height / CoefficientGroupSize; + int groupCount = widthInGroups * heightInGroups; + int lastScanPosition = -1; + ScanGenerator groupScan = new(widthInGroups, heightInGroups, scanType); + + // H.265 scans the 4x4 groups first, then applies the same direction inside each group. Keeping this grouped + // layout contiguous lets coefficient decoding walk every 16-entry subset without lookup-table allocations. + for (int groupIndex = 0; groupIndex < groupCount; groupIndex++) + { + int groupOffsetX = groupScan.X * CoefficientGroupSize; + int groupOffsetY = groupScan.Y * CoefficientGroupSize; + int groupScanOffset = groupIndex * CoefficientGroupSize * CoefficientGroupSize; + ScanGenerator coefficientScan = new(CoefficientGroupSize, CoefficientGroupSize, scanType); + + for (int coefficientIndex = 0; coefficientIndex < CoefficientGroupSize * CoefficientGroupSize; coefficientIndex++) + { + int rasterPosition = ((groupOffsetY + coefficientScan.Y) * width) + groupOffsetX + coefficientScan.X; + int scanPosition = groupScanOffset + coefficientIndex; + destination[scanPosition] = rasterPosition; + if (rasterPosition == lastRasterPosition) + { + lastScanPosition = scanPosition; + } + + coefficientScan.MoveNext(); + } + + groupScan.MoveNext(); + } + + return lastScanPosition; + } + + /// + /// Advances through one rectangular scan without retaining a heap-backed lookup table. + /// + private struct ScanGenerator + { + /// + /// The scan width. + /// + private readonly int width; + + /// + /// The scan height. + /// + private readonly int height; + + /// + /// The selected scan direction. + /// + private readonly HevcCoefficientScanType scanType; + + /// + /// Initializes a new instance of the struct. + /// + /// The scan width. + /// The scan height. + /// The scan direction. + public ScanGenerator(int width, int height, HevcCoefficientScanType scanType) + { + this.width = width; + this.height = height; + this.scanType = scanType; + this.X = 0; + this.Y = 0; + } + + /// + /// Gets the current horizontal coordinate. + /// + public int X { get; private set; } + + /// + /// Gets the current vertical coordinate. + /// + public int Y { get; private set; } + + /// + /// Advances to the next coordinate in the selected scan direction. + /// + public void MoveNext() + { + switch (this.scanType) + { + case HevcCoefficientScanType.Diagonal: + if (this.X == this.width - 1 || this.Y == 0) + { + this.Y += this.X + 1; + this.X = 0; + if (this.Y >= this.height) + { + this.X += this.Y - (this.height - 1); + this.Y = this.height - 1; + } + } + else + { + this.X++; + this.Y--; + } + + break; + case HevcCoefficientScanType.Horizontal: + if (++this.X == this.width) + { + this.X = 0; + this.Y++; + } + + break; + default: + if (++this.Y == this.height) + { + this.Y = 0; + this.X++; + } + + break; + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanType.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanType.cs new file mode 100644 index 000000000..93ceca238 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientScanType.cs @@ -0,0 +1,25 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Identifies the coefficient scan used by one HEVC transform block. +/// +internal enum HevcCoefficientScanType +{ + /// + /// The up-right diagonal scan. + /// + Diagonal, + + /// + /// The row-major horizontal scan. + /// + Horizontal, + + /// + /// The column-major vertical scan. + /// + Vertical, +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionMode.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionMode.cs new file mode 100644 index 000000000..8dc56dad0 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionMode.cs @@ -0,0 +1,35 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +namespace SixLabors.ImageSharp.Formats.Heif.Hevc; + +/// +/// Provides the HEVC intra-mode values and chroma-format mapping shared by entropy decoding and reconstruction. +/// +internal static class HevcIntraPredictionMode +{ + /// + /// The horizontal angular prediction mode. + /// + public const int Horizontal = 10; + + /// + /// The vertical angular prediction mode. + /// + public const int Vertical = 26; + + /// + /// Gets the 4:2:2 chroma intra-angle remapping defined by H.265 Table 8-4. + /// + private static ReadOnlySpan Chroma422AngleMap => + [ + 0, 1, 2, 2, 2, 2, 3, 5, 7, 8, 10, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 23, 24, 24, 25, 25, 26, 27, 27, 28, 28, 29, 29, 30, 31, + ]; + + /// + /// Maps a coded chroma intra mode to the angular mode used by a 4:2:2 chroma block. + /// + /// The effective coded chroma intra mode. + /// The prediction angle used by the rectangular chroma block. + public static int RemapChroma422(int mode) => Chroma422AngleMap[mode]; +} diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs index 86e5a543b..c7779f2da 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs @@ -111,7 +111,7 @@ internal sealed class HevcIntraPredictionState : IDisposable int predictionBlockLog2 = usesNxNPartitions ? log2Size - 1 : log2Size; int predictionBlockSize = 1 << predictionBlockLog2; int predictionBlockCount = usesNxNPartitions ? 4 : 1; - Span mostProbableFlags = stackalloc byte[4]; + InlineArray4 mostProbableFlags = default; // HEVC codes every prev_intra_luma_pred_flag before any associated mode suffix. Preserve that two-pass // ordering because decoding one complete mode at a time would consume a different CABAC bit sequence. @@ -120,7 +120,9 @@ internal sealed class HevcIntraPredictionState : IDisposable mostProbableFlags[index] = reader.ReadPreviousIntraLumaPredictionFlag() ? (byte)1 : (byte)0; } - Span mostProbableModes = stackalloc byte[3]; + InlineArray4 mostProbableModes = default; + Span mostProbableModeSpan = mostProbableModes[..3]; + for (int index = 0; index < predictionBlockCount; index++) { int offsetX = (index & 1) * predictionBlockSize; @@ -135,22 +137,22 @@ internal sealed class HevcIntraPredictionState : IDisposable predictionY, predictionLeftAvailable, predictionAboveAvailable, - mostProbableModes); + mostProbableModeSpan); int mode; if (mostProbableFlags[index] != 0) { - mode = mostProbableModes[reader.ReadMostProbableIntraLumaPredictionIndex()]; + mode = mostProbableModeSpan[reader.ReadMostProbableIntraLumaPredictionIndex()]; } else { - SortThree(mostProbableModes); + SortThree(mostProbableModeSpan); mode = reader.ReadRemainingIntraLumaPredictionMode(); - for (int candidate = 0; candidate < mostProbableModes.Length; candidate++) + for (int candidate = 0; candidate < mostProbableModeSpan.Length; candidate++) { // The remaining-mode code omits the three probable values, so each candidate at or below the // provisional result advances the decoded mode over that omitted slot. - mode += mode >= mostProbableModes[candidate] ? 1 : 0; + mode += mode >= mostProbableModeSpan[candidate] ? 1 : 0; } } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs index caa99a5b8..b06696440 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs @@ -12,16 +12,6 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc; /// internal static class HevcResidualReconstructor { - /// - /// The horizontal intra-prediction mode defined by H.265. - /// - private const int HorizontalIntraPredictionMode = 10; - - /// - /// The vertical intra-prediction mode defined by H.265. - /// - private const int VerticalIntraPredictionMode = 26; - /// /// The minimum residual sample represented by the decoder reconstruction pipeline. /// @@ -70,14 +60,6 @@ internal static class HevcResidualReconstructor static abstract int Invoke(int value, int shift); } - /// - /// Gets the 4:2:2 chroma intra-angle remapping defined by H.265 Table 8-4. - /// - private static ReadOnlySpan Chroma422IntraAngleMap => - [ - 0, 1, 2, 2, 2, 2, 3, 5, 7, 8, 10, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 23, 24, 24, 25, 25, 26, 27, 27, 28, 28, 29, 29, 30, 31, - ]; - /// /// Copies one transquant-bypass coefficient block into residual sample order. /// @@ -154,11 +136,11 @@ internal static class HevcResidualReconstructor /// The residual differential mode selected by the prediction direction. public static HevcResidualDpcmMode GetImplicitResidualDpcmMode(int intraPredictionMode, bool remapChroma422) { - int predictionMode = remapChroma422 ? Chroma422IntraAngleMap[intraPredictionMode] : intraPredictionMode; + int predictionMode = remapChroma422 ? HevcIntraPredictionMode.RemapChroma422(intraPredictionMode) : intraPredictionMode; return predictionMode switch { - HorizontalIntraPredictionMode => HevcResidualDpcmMode.Horizontal, - VerticalIntraPredictionMode => HevcResidualDpcmMode.Vertical, + HevcIntraPredictionMode.Horizontal => HevcResidualDpcmMode.Horizontal, + HevcIntraPredictionMode.Vertical => HevcResidualDpcmMode.Vertical, _ => HevcResidualDpcmMode.None, }; } diff --git a/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcCoefficientScanBenchmarks.cs b/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcCoefficientScanBenchmarks.cs new file mode 100644 index 000000000..1c33134f8 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/Codecs/Heif/HevcCoefficientScanBenchmarks.cs @@ -0,0 +1,58 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Formats.Heif.Hevc; + +namespace SixLabors.ImageSharp.Benchmarks.Codecs.Heif; + +/// +/// Measures full-frame HEVC grouped coefficient-scan generation into reusable transform scratch. +/// +[MemoryDiagnoser(displayGenColumns: false)] +public class HevcCoefficientScanBenchmarks +{ + /// + /// The coded frame width, which is an exact multiple of the maximum transform-block side. + /// + private const int Width = 1920; + + /// + /// The coded frame height including the final padded coding-tree row for a 1080-line presentation. + /// + private const int Height = 1088; + + /// + /// The maximum transform-block side. + /// + private const int BlockSize = 32; + + /// + /// The reusable maximum-size grouped scan destination. + /// + private readonly int[] scan = new int[BlockSize * BlockSize]; + + /// + /// Measures diagonal scan generation for every maximum-size transform block in one coded full-HD frame. + /// + /// The final last-significant scan position, keeping the generated scan observable. + [Benchmark] + public int WriteDiagonalFrame() + { + int lastScanPosition = 0; + for (int y = 0; y < Height; y += BlockSize) + { + for (int x = 0; x < Width; x += BlockSize) + { + lastScanPosition = HevcCoefficientScanOrder.Write( + this.scan, + BlockSize, + BlockSize, + HevcCoefficientScanType.Diagonal, + (BlockSize * BlockSize) - 1); + } + } + + return lastScanPosition; + } +} diff --git a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcCoefficientDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcCoefficientDecoderTests.cs new file mode 100644 index 000000000..299e84dfe --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcCoefficientDecoderTests.cs @@ -0,0 +1,287 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Hevc; +using SixLabors.ImageSharp.Tests.Memory; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Hevc; + +/// +/// Verifies HEVC coefficient scan selection, grouped ordering, and CABAC level reconstruction. +/// +public class HevcCoefficientDecoderTests +{ + /// + /// Verifies the three normative four-by-four coefficient scans. + /// + /// The scan direction under test. + /// The expected raster indices in scan order. + [Theory] + [MemberData(nameof(GetFourByFourScans))] + public void WritesFourByFourScan(int scanType, int[] expected) + { + int[] actual = new int[16]; + + int lastScanPosition = HevcCoefficientScanOrder.Write(actual, 4, 4, (HevcCoefficientScanType)scanType, expected[^1]); + + Assert.Equal(expected, actual); + Assert.Equal(15, lastScanPosition); + } + + /// + /// Verifies that grouped scans visit each coefficient exactly once for every supported transform geometry. + /// + /// The transform-block width. + /// The transform-block height. + /// The scan direction under test. + [Theory] + [InlineData(4, 4, HevcCoefficientScanType.Diagonal)] + [InlineData(8, 8, HevcCoefficientScanType.Diagonal)] + [InlineData(8, 4, HevcCoefficientScanType.Horizontal)] + [InlineData(4, 8, HevcCoefficientScanType.Vertical)] + [InlineData(16, 32, HevcCoefficientScanType.Diagonal)] + [InlineData(32, 16, HevcCoefficientScanType.Horizontal)] + [InlineData(32, 32, HevcCoefficientScanType.Vertical)] + public void GroupedScanVisitsEveryCoefficient(int width, int height, int scanType) + { + int coefficientCount = width * height; + int[] scan = new int[coefficientCount]; + bool[] visited = new bool[coefficientCount]; + + int lastScanPosition = HevcCoefficientScanOrder.Write(scan, width, height, (HevcCoefficientScanType)scanType, coefficientCount - 1); + + Assert.InRange(lastScanPosition, 0, coefficientCount - 1); + foreach (int rasterPosition in scan) + { + Assert.InRange(rasterPosition, 0, coefficientCount - 1); + Assert.False(visited[rasterPosition]); + visited[rasterPosition] = true; + } + + Assert.All(visited, Assert.True); + } + + /// + /// Verifies that an eight-by-eight diagonal scan groups coefficients in the normative group order. + /// + [Fact] + public void DiagonalEightByEightScanUsesGroupedOrder() + { + int[] scan = new int[64]; + + HevcCoefficientScanOrder.Write(scan, 8, 8, HevcCoefficientScanType.Diagonal, 63); + + Assert.Equal(0, scan[0]); + Assert.Equal(32, scan[16]); + Assert.Equal(4, scan[32]); + Assert.Equal(36, scan[48]); + } + + /// + /// Verifies transform geometry and intra direction select horizontal, vertical, or diagonal scans. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// Whether the containing coding unit uses intra prediction. + /// The effective intra mode. + /// The sequence chroma format. + /// The expected coefficient scan. + [Theory] + [InlineData(8, 8, HevcPlane.Y, false, 26, 3, HevcCoefficientScanType.Diagonal)] + [InlineData(8, 8, HevcPlane.Y, true, 26, 3, HevcCoefficientScanType.Horizontal)] + [InlineData(8, 8, HevcPlane.Y, true, 10, 3, HevcCoefficientScanType.Vertical)] + [InlineData(8, 8, HevcPlane.Y, true, 18, 3, HevcCoefficientScanType.Diagonal)] + [InlineData(16, 16, HevcPlane.Y, true, 26, 3, HevcCoefficientScanType.Diagonal)] + [InlineData(8, 8, HevcPlane.Cb, true, 26, 1, HevcCoefficientScanType.Diagonal)] + [InlineData(4, 4, HevcPlane.Cb, true, 26, 1, HevcCoefficientScanType.Horizontal)] + public void SelectsScanType( + int width, + int height, + int plane, + bool isIntra, + int mode, + byte chromaFormat, + int expected) + { + HevcCoefficientScanType actual = HevcCoefficientCodingParameters.SelectScanType( + width, + height, + (HevcPlane)plane, + isIntra, + mode, + chromaFormat, + false); + + Assert.Equal((HevcCoefficientScanType)expected, actual); + } + + /// + /// Verifies the significance-map context bases selected by transform size, scan, channel, and Range Extensions. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// The selected coefficient scan. + /// Whether the Range Extensions single-context mode applies. + /// The expected first significance-map context. + [Theory] + [InlineData(4, 4, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false, 0)] + [InlineData(8, 8, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false, 9)] + [InlineData(8, 8, HevcPlane.Y, HevcCoefficientScanType.Horizontal, false, 15)] + [InlineData(16, 16, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false, 21)] + [InlineData(8, 8, HevcPlane.Cb, HevcCoefficientScanType.Vertical, false, 9)] + [InlineData(16, 16, HevcPlane.Cr, HevcCoefficientScanType.Diagonal, false, 12)] + [InlineData(4, 4, HevcPlane.Y, HevcCoefficientScanType.Diagonal, true, 27)] + [InlineData(4, 4, HevcPlane.Cb, HevcCoefficientScanType.Diagonal, true, 15)] + public void SelectsFirstSignificanceContext( + int width, + int height, + int plane, + int scanType, + bool singleContext, + int expected) + { + HevcCoefficientCodingParameters parameters = CreateParameters( + width, + height, + (HevcPlane)plane, + (HevcCoefficientScanType)scanType, + singleContext); + + Assert.Equal(expected, parameters.FirstSignificanceMapContext); + } + + /// + /// Verifies the complete four-by-four raster-position context mapping. + /// + [Fact] + public void FourByFourSignificanceContextsMatchNormativeMap() + { + int[] expected = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8]; + HevcCoefficientCodingParameters parameters = CreateParameters(4, 4, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false); + + for (int rasterPosition = 0; rasterPosition < expected.Length; rasterPosition++) + { + Assert.Equal(expected[rasterPosition], parameters.GetSignificantCoefficientContext(rasterPosition, 0)); + } + } + + /// + /// Verifies significant-group and coefficient contexts use already decoded right and lower groups. + /// + [Fact] + public void SignificanceContextsUseRightAndLowerGroups() + { + int[] groupFlags = [0, 1, 1, 0]; + HevcCoefficientCodingParameters luma = CreateParameters(8, 8, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false); + HevcCoefficientCodingParameters chroma = CreateParameters(8, 8, HevcPlane.Cb, HevcCoefficientScanType.Diagonal, false); + + Assert.Equal(1, luma.GetSignificantGroupContext(groupFlags, 0, 0)); + Assert.Equal(3, luma.GetSignificancePattern(groupFlags, 0, 0)); + Assert.Equal(0, luma.GetSignificantGroupContext(groupFlags, 1, 0)); + Assert.Equal(0, luma.GetSignificancePattern(groupFlags, 1, 0)); + Assert.Equal(0, luma.GetSignificantCoefficientContext(0, 3)); + Assert.Equal(11, luma.GetSignificantCoefficientContext(1, 3)); + Assert.Equal(14, luma.GetSignificantCoefficientContext(4, 0)); + Assert.Equal(11, chroma.GetSignificantCoefficientContext(4, 0)); + } + + /// + /// Verifies luma and chroma level-context sets track subset position and preceding greater-than-one state. + /// + [Fact] + public void SelectsLevelContextSets() + { + HevcCoefficientCodingParameters luma = CreateParameters(8, 8, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false); + HevcCoefficientCodingParameters chroma = CreateParameters(8, 8, HevcPlane.Cb, HevcCoefficientScanType.Diagonal, false); + + Assert.Equal(0, luma.GetLevelContextSet(0, false)); + Assert.Equal(1, luma.GetLevelContextSet(0, true)); + Assert.Equal(2, luma.GetLevelContextSet(1, false)); + Assert.Equal(3, luma.GetLevelContextSet(1, true)); + Assert.Equal(0, chroma.GetLevelContextSet(0, false)); + Assert.Equal(1, chroma.GetLevelContextSet(3, true)); + } + + /// + /// Verifies a fixed CABAC substream reconstructs one positive DC coefficient without allocating per block. + /// + [Fact] + public void DecodesPositiveDcCoefficientWithReusableScratch() + { + TestMemoryAllocator allocator = new(); + allocator.EnableNonThreadSafeLogging(); + Configuration configuration = new() { MemoryAllocator = allocator }; + using HevcCoefficientDecoder decoder = new(configuration); + int[] coefficients = new int[16]; + HevcCoefficientCodingParameters parameters = CreateParameters(4, 4, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false); + + HevcCabacSyntaxReader firstReader = new([0xEE, 0x48], 22); + int firstNonZeroCount = decoder.Decode(ref firstReader, coefficients, in parameters); + HevcCabacSyntaxReader secondReader = new([0xEE, 0x48], 22); + int secondNonZeroCount = decoder.Decode(ref secondReader, coefficients, in parameters); + + Assert.Equal(1, firstNonZeroCount); + Assert.Equal(1, secondNonZeroCount); + Assert.Equal(1, coefficients[0]); + Assert.All(coefficients[1..], value => Assert.Equal(0, value)); + Assert.Single(allocator.AllocationLog); + } + + /// + /// Verifies the bypass-coded sign is applied to a fixed single-coefficient CABAC substream. + /// + [Fact] + public void DecodesNegativeDcCoefficient() + { + using HevcCoefficientDecoder decoder = new(Configuration.Default); + HevcCabacSyntaxReader reader = new([0xF4, 0x24], 22); + int[] coefficients = new int[16]; + HevcCoefficientCodingParameters parameters = CreateParameters(4, 4, HevcPlane.Y, HevcCoefficientScanType.Diagonal, false); + + int nonZeroCount = decoder.Decode(ref reader, coefficients, in parameters); + + Assert.Equal(1, nonZeroCount); + Assert.Equal(-1, coefficients[0]); + Assert.All(coefficients[1..], value => Assert.Equal(0, value)); + } + + /// + /// Gets the exact raster order for each four-by-four scan direction. + /// + /// The scan direction and expected raster positions. + public static TheoryData GetFourByFourScans() => + new() + { + { + (int)HevcCoefficientScanType.Diagonal, + [0, 4, 1, 8, 5, 2, 12, 9, 6, 3, 13, 10, 7, 14, 11, 15] + }, + { + (int)HevcCoefficientScanType.Horizontal, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + }, + { + (int)HevcCoefficientScanType.Vertical, + [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15] + }, + }; + + /// + /// Creates explicit coefficient parameters for scan and entropy tests without requiring a parsed parameter set. + /// + /// The transform-block width. + /// The transform-block height. + /// The reconstructed component. + /// The coefficient scan. + /// Whether the Range Extensions single significance context applies. + /// The coefficient coding parameters. + private static HevcCoefficientCodingParameters CreateParameters( + int width, + int height, + HevcPlane plane, + HevcCoefficientScanType scanType, + bool singleContext) + => new(width, height, plane, scanType, singleContext, false, false, false, false, 15, plane == HevcPlane.Y ? 0 : 2); +}