From 8e21e3933ae8712d706134cc926d7f75ee902862 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Wed, 26 Aug 2026 07:32:36 +1000 Subject: [PATCH] Verify HEVC 8-bit still image decoding --- .gitattributes | 3 + HEIF_IMPLEMENTATION_PLAN.md | 20 ++- .../Heif/Hevc/HevcCoefficientDecoder.cs | 26 +++- .../Formats/Heif/Hevc/HevcDeblockingFilter.cs | 60 ++++++--- .../Formats/Heif/Hevc/HevcDeblockingState.cs | 8 +- .../Heif/Hevc/HevcIntraPredictionState.cs | 77 ++++++++--- ...cIntraPredictor.OperatorImplementations.cs | 6 +- .../Hevc/HevcIntraPredictor.References.cs | 14 +- .../Hevc/HevcInverseTransformer.Operations.cs | 2 +- .../Hevc/HevcPictureDecoder.TransformTree.cs | 2 + .../Heif/Hevc/HevcPictureDecoder.Traversal.cs | 46 ++++--- .../Heif/Hevc/HevcReconstructionState.cs | 9 +- .../HevcSampleAdaptiveOffsetParameters.cs | 4 + .../Formats/Heif/HeifDecoderTests.cs | 31 +++++ .../Heif/Hevc/HevcInverseTransformerTests.cs | 15 +++ .../Heif/Hevc/HevcPictureDecoderTests.cs | 122 ++++++++++++++++++ tests/ImageSharp.Tests/TestImages.cs | 13 ++ ...evcStillImage_Rgba32_dwsample-heic-640.png | 3 + .../DecodeHevcStillImage_Rgba32_image1.png | 3 + .../DecodeHevcStillImage_Rgba32_image2.png | 3 + .../DecodeHevcStillImage_Rgba32_image3.png | 3 + .../DecodeHevcStillImage_Rgba32_image4.png | 3 + .../Input/Heif/Hevc/dwsample-tile-hm.yuv | 3 + .../Images/Input/Heif/Hevc/dwsample-tile.hvc1 | 3 + .../Images/Input/Heif/Hevc/dwsample-tile.hvcc | 3 + .../Input/Heif/Hevc/image1-tile1-hm.yuv | 3 + .../Images/Input/Heif/Hevc/image1-tile1.hvc1 | 3 + .../Images/Input/Heif/Hevc/image1-tile1.hvcc | 3 + .../Input/Heif/Hevc/image1-tile2-hm.yuv | 3 + .../Images/Input/Heif/Hevc/image1-tile2.hvc1 | 3 + tests/Images/Input/Heif/Hevc/image2-tile.hvcc | 3 + .../Input/Heif/Hevc/image2-tile1-hm.yuv | 3 + .../Images/Input/Heif/Hevc/image2-tile1.hvc1 | 3 + 33 files changed, 421 insertions(+), 85 deletions(-) create mode 100644 tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs create mode 100644 tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_dwsample-heic-640.png create mode 100644 tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image1.png create mode 100644 tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image2.png create mode 100644 tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image3.png create mode 100644 tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image4.png create mode 100644 tests/Images/Input/Heif/Hevc/dwsample-tile-hm.yuv create mode 100644 tests/Images/Input/Heif/Hevc/dwsample-tile.hvc1 create mode 100644 tests/Images/Input/Heif/Hevc/dwsample-tile.hvcc create mode 100644 tests/Images/Input/Heif/Hevc/image1-tile1-hm.yuv create mode 100644 tests/Images/Input/Heif/Hevc/image1-tile1.hvc1 create mode 100644 tests/Images/Input/Heif/Hevc/image1-tile1.hvcc create mode 100644 tests/Images/Input/Heif/Hevc/image1-tile2-hm.yuv create mode 100644 tests/Images/Input/Heif/Hevc/image1-tile2.hvc1 create mode 100644 tests/Images/Input/Heif/Hevc/image2-tile.hvcc create mode 100644 tests/Images/Input/Heif/Hevc/image2-tile1-hm.yuv create mode 100644 tests/Images/Input/Heif/Hevc/image2-tile1.hvc1 diff --git a/.gitattributes b/.gitattributes index a0643702b3..a212331e59 100644 --- a/.gitattributes +++ b/.gitattributes @@ -138,6 +138,9 @@ *.heic filter=lfs diff=lfs merge=lfs -text *.hif filter=lfs diff=lfs merge=lfs -text *.avif filter=lfs diff=lfs merge=lfs -text +*.hvc1 filter=lfs diff=lfs merge=lfs -text +*.hvcc filter=lfs diff=lfs merge=lfs -text +*.yuv filter=lfs diff=lfs merge=lfs -text ############################################################################### # Handle ICC files by git lfs ############################################################################### diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 4ae8cab263..b8c9fecd37 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -47,12 +47,15 @@ Checkboxes may be marked complete only when the implementation and the verificat - [x] Implement allocation-free SIMD-first HEVC inverse quantization for flat and custom scaling lists, 4/8/16/32 transforms, transform-skip scaling rules, extended precision, signed normalization shifts, and transform-range clipping. Store compact and transform-sized scaling matrices in one contiguous parameter-set allocation so the hot path can widen consecutive weights without coordinate division. - [x] Derive each reference-unit availability flag from coding-tree, slice, tile, constrained-intra, and picture-boundary state, then connect prediction to transform-unit traversal and the bounded item decoder lifecycle. - [x] Register the bounded `hvc1` still-image decoder, reconstruct the selected PPS/SPS picture, crop the conformance window directly in the pooled row converter, and pass the native planes through the shared SIMD-first H.273 pipeline. + - [x] Compare native HEVC planes byte-for-byte with HM 18 output for four item payloads extracted from independently produced HEIC files: two 8-bit 4:2:0 grid tiles, a cropped 8-bit 4:2:0 grid tile, and an 8-bit 4:4:4 wavefront tile. + - [x] Compare complete HEIC presentation output with libheif 1.23.1 and libde265 1.1.1 for single images, multi-tile grids, an auxiliary alpha image, an ICC-bearing image, and a 4:4:4 wavefront image. Exact native-plane tests remain the codec oracle because libheif's selected fused 4:2:0 RGB path uses nearest-neighbor chroma while ImageSharp uses its higher-quality signaled-position interpolation. + - [ ] Add permanent HEVC 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 fixtures covering every exposed profile and Range Extensions tool, with exact native-plane references from HM or another independent decoder. - [x] Implement the shared SIMD-first YUV/CICP paths in both directions for AV1 and HEVC bit depths, chroma formats, ranges, matrix coefficients, transfer functions, color primaries, and chroma positions, with scalar fallback only after the available vector widths are exhausted. - [ ] Complete the independently generated AV1 and HEVC conformance matrix for every supported color-signaling combination before closing the Phase 5 color exit gate. - [x] Apply ICC conversion only after the SIMD YUV/CICP stage, alpha composition, grid assembly, and presentation transforms have produced the presented RGB image; retain ImageSharp's shared ICC converter and optimize reusable bulk kernels rather than creating a HEIF-specific color-management implementation. - [ ] Verify ICC preservation, conversion, compaction, and metadata skipping for grids, alpha-composited images, every presented sequence frame, and the completed HEVC path. - [x] Independently encode AVIF primary, grid, alpha-auxiliary, and two-frame sequence vectors with libavif 1.4.2/libaom 3.14.1; verify exact profile preservation, non-sRGB conversion, canonical-sRGB compaction, non-sRGB compact preservation, metadata skipping, every presented frame, and unchanged composed alpha. - - [ ] Repeat the same matrix for HEVC primary, grid, alpha-auxiliary, and sequence paths after HEVC reconstruction is complete. + - [ ] Repeat the complete ICC preservation, conversion, compaction, and skipping matrix for HEVC primary, grid, alpha-auxiliary, and sequence paths. The current libheif comparison establishes presentation interoperability for one ICC-bearing still image but does not close these separate color-management contracts. - [ ] Benchmark representative 8/10/12-bit end-to-end AVIF and HEIC color pipelines with and without ICC conversion, recording absolute timings and allocations before marking Phase 5 complete. - [x] Add permanent frame-wide `Av1ColorConversionBenchmarks` coverage for sequential 1920x1080 YUV 4:2:0 conversion in both directions at 8, 10, and 12 bits, including managed-allocation reporting. - [x] Record the .NET 10 short-run AV1 baseline on 2026-08-25. SIMD decode measured 4.025/2.803/2.742 ms and encode measured 3.750/2.555/2.450 ms at 8/10/12 bits with 488-744 B allocated per frame. Forced-scalar decode measured 31.15/29.47/28.40 ms and encode measured 19.19/15.77/15.14 ms, making the normal SIMD pipeline 7.7-10.5 times faster for decode and 5.1-6.2 times faster for encode on this machine. @@ -279,22 +282,22 @@ This assessment is based on the current source after the upstream ImageSharp mer - Item IDs are resolved as keys rather than list indices; metadata children are indexed and parsed in dependency order rather than physical order. - Item property associations now preserve physical `ipco` indices, apply one-based 7-bit or 15-bit indices and essential flags, associate by item ID, and reject arbitrary unknown essential properties. - Item locations now support bounded file-relative and `idat`-relative storage, multiple ordered extents, versioned item IDs, 0/4/8-byte registered field sizes, and 64-bit offsets. Referenced-item construction method two and external data references remain explicitly unsupported. -- Grid derived-image decoding now parses both registered descriptor widths, resolves the ordered `dimg` cells, validates tile count, coding format, dimensions, canvas coverage, and edge overlap, then composes the output through ImageSharp row buffers. Unsupported grid tile codecs can select only a decodable thumbnail of the same primary grid. Independent AV1 and JPEG grid fixtures are still required, and HEVC grids remain blocked on the HEVC decoder. -- Alpha auxiliary decoding now recognizes `auxC`, `auxl`, and `prem`, supports a direct auxiliary image/grid and the per-color-grid-tile form, preserves normalized alpha through `L16`/`Rgba64`, box-resamples differing plane sizes, and reports alpha presence. HEVC alpha remains blocked on the HEVC image-item decoder, while independent AVIF alpha fixtures are still required against the incomplete AV1 reconstruction pipeline. +- Grid derived-image decoding now parses both registered descriptor widths, resolves the ordered `dimg` cells, validates tile count, coding format, dimensions, canvas coverage, and edge overlap, then composes the output through ImageSharp row buffers. Unsupported grid tile codecs can select only a decodable thumbnail of the same primary grid. HEVC grids are exercised by independent libheif output and exact HM tile-plane references; independent AV1 and JPEG grid fixtures are still required. +- Alpha auxiliary decoding now recognizes `auxC`, `auxl`, and `prem`, supports a direct auxiliary image/grid and the per-color-grid-tile form, preserves normalized alpha through `L16`/`Rgba64`, box-resamples differing plane sizes, and reports alpha presence. HEVC auxiliary-alpha presentation is exercised against independent libheif output, while independent AVIF alpha fixtures are still required against the incomplete AV1 reconstruction pipeline. - Clean aperture, image rotation, and image mirror properties now validate their registered payloads, exact integer crop geometry, and essential associations; affect Identify dimensions; and reuse ImageSharp's optimized crop/rotate/flip processors after auxiliary alpha composition in the MIAF-defined order. Independent transform vectors must still verify every crop/rotation/mirror/alpha combination. - Decode now resolves `cdsc`-associated Exif and `application/rdf+xml` XMP items for the primary still image, validates the declared Exif TIFF-header offset, and attaches the payloads through ImageSharp's existing profile types before presentation transforms. Independent AVIF, HEIC, and HIF metadata fixtures and Identify-time profile reporting remain required. - ICC and `nclx` CICP color properties are now associated with the presented color item instead of global parser state, validated, and exposed on Decode and Identify through the existing ImageSharp profiles. AV1 decode requires specified sequence-header fields and the range flag to match `nclx`, uses the property only for fields the bitstream leaves unspecified, and exposes the effective sequence-header CICP description when no property is present. Independent libavif fixtures verify exact ICC byte preservation, observable non-sRGB conversion, canonical-sRGB compaction, non-sRGB compact preservation, and metadata skipping for primary, grid, alpha-composited, and every presented sequence frame; the alpha vector also verifies that ICC conversion does not alter composed alpha. Identify-time bitstream fallback, independent CICP vectors, and HEVC integration remain required. - Pixel aspect ratio now preserves the complete unsigned spacing pair, affects Decode and Identify through ImageSharp's existing unitless resolution metadata, and follows quarter-turn presentation rotation. Independent grid, rotation, and maximum-spacing fixtures remain required. -- AV1 codec configuration is now retained per `av01` image item rather than in decoder-global state. Decode requires the property, validates its fixed record against the item's sequence header and any associated `pixi` channel depths, and requires matching configurations across grid tiles. Identify now reports the configuration's 8/10/12-bit precision and monochrome shape. The optional `configOBUs` sequence is bounded and validated, including its mandatory size fields, first-position/at-most-one sequence-header rule, the image item's exactly-one sequence-header rule, and exact comparison of a repeated configuration header with the item header. Content light-level and mastering-display color-volume information are decoded from the bounded `clli` and `mdcv` image properties and matching AV1 HDR metadata OBUs, with the representations' distinct fixed-point precision, grid/property precedence, and `SkipMetadata` behavior preserved. The still-image `cclv`, `amve`, `reve`, and `ndwt` properties preserve content-volume, ambient-viewing, mastering-environment, and diffuse-white descriptions without importing their video-SEI or sample-entry state models. Independent fixtures and encoder preservation remain required. HEVC `hvcC` is now associated with each `hvc1` image item, structurally validated through its VPS/SPS/PPS arrays, and used to report encoded precision and monochrome shape; item-payload validation is implemented for the bounded decoder path, while pixel reconstruction remains incomplete. +- AV1 codec configuration is now retained per `av01` image item rather than in decoder-global state. Decode requires the property, validates its fixed record against the item's sequence header and any associated `pixi` channel depths, and requires matching configurations across grid tiles. Identify now reports the configuration's 8/10/12-bit precision and monochrome shape. The optional `configOBUs` sequence is bounded and validated, including its mandatory size fields, first-position/at-most-one sequence-header rule, the image item's exactly-one sequence-header rule, and exact comparison of a repeated configuration header with the item header. Content light-level and mastering-display color-volume information are decoded from the bounded `clli` and `mdcv` image properties and matching AV1 HDR metadata OBUs, with the representations' distinct fixed-point precision, grid/property precedence, and `SkipMetadata` behavior preserved. The still-image `cclv`, `amve`, `reve`, and `ndwt` properties preserve content-volume, ambient-viewing, mastering-environment, and diffuse-white descriptions without importing their video-SEI or sample-entry state models. Independent fixtures and encoder preservation remain required. HEVC `hvcC` is associated with each `hvc1` image item, structurally validated through its VPS/SPS/PPS arrays, and used to report encoded precision and monochrome shape. The bounded decoder reconstructs exact native planes for the verified 8-bit 4:2:0 and 4:4:4 item payloads; the remaining profile, bit-depth, chroma-layout, and Range Extensions matrix is not yet established. - Several image-item properties and relationships remain missing or parsed without fully affecting output. - Identify and decode now use the same bounded metadata parser and both validate the complete leading file type box. The parsed state is still mutable and Identify does not yet report the complete bit depth, color, profile, or transform model. ### HEVC decoder and encoder - `Heif4CharCode` recognizes `hvc1` image items, `HevcCodecConfiguration` validates and associates the bounded `hvcC` property, Identify reports its HEVC precision and monochrome shape, and `HeifCompressionFactory` registers the bounded HEVC still-image decoder. -- The HEVC path validates NAL headers and RBSP escaping, links `hvcC` VPS/SPS/PPS structures through Range Extensions, parses one length-delimited base-layer IDR picture, and reconstructs its complete intra-coded coding tree. It includes CABAC, transform traversal, coefficient decoding, intra prediction, inverse quantization and transform, PCM, deblocking, sample-adaptive offset, and allocator-owned native `ushort` planes for monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit samples. The item adapter converts the SPS conformance window directly through the shared pooled SIMD-first H.273 pipeline and preserves the effective container-or-VUI CICP description. The internal HEVC suite passes, but independent pixel comparisons for the repository's HEIC fixtures and the complete color/precision/layout matrix remain open, so conformance is not yet established. +- The HEVC path validates NAL headers and RBSP escaping, links `hvcC` VPS/SPS/PPS structures through Range Extensions, parses one length-delimited base-layer IDR picture, and reconstructs its complete intra-coded coding tree. It includes CABAC, transform traversal, coefficient decoding, intra prediction, inverse quantization and transform, PCM, deblocking, sample-adaptive offset, and allocator-owned native `ushort` planes for monochrome, 4:2:0, 4:2:2, 4:4:4, separate-color-plane, and 8-through-16-bit samples. The item adapter converts the SPS conformance window directly through the shared pooled SIMD-first H.273 pipeline and preserves the effective container-or-VUI CICP description. Four real item payloads now match HM 18 native planes exactly, covering 8-bit 4:2:0 grids, cropping, 8-bit 4:4:4, and wavefront entry points. Complete images covering grids, auxiliary alpha, ICC, and 4:4:4 presentation also match independent libheif output within the documented chroma-reconstruction tolerance. High bit depths, monochrome, 4:2:2, remaining Range Extensions tools, and the complete color matrix still require independent fixtures before HEVC conformance is established. - There is no HEVC encoder. The current HEIC-branded encoder writes a legacy JPEG payload and therefore cannot provide HEIC output. -- Existing HEVC tests prove container identification only; they do not decode or compare HEIC pixels. +- Permanent HEVC conformance tests compare extracted item planes exactly with HM output and complete presented images with pinned libheif/libde265 reference PNGs. The fixtures do not yet cover the complete exposed profile matrix. ### AV1 decoder @@ -469,7 +472,7 @@ Implement and verify in dependency order: - [ ] Intra prediction for every luma and chroma mode, including strong intra smoothing and constrained prediction rules. - [x] Implement and verify the allocation-free SIMD-first predictor primitive for all 35 modes, 4/8/16/32 blocks, and 8/10/12-bit samples. - [x] Implement reconstructed-plane reference collection, substitution, asymmetric chroma-unit handling, and reference-filter selection from caller-owned availability and scratch spans. - - [ ] Build each availability flag from reconstructed-plane and coding state, and connect the predictor to transform-unit traversal. + - [x] Build each availability flag from reconstructed-plane and coding state, and connect the predictor to transform-unit traversal for the independently verified 8-bit 4:2:0 and 4:4:4 item paths. - [ ] Scaling lists, inverse quantization, transform skip, every required inverse transform, range-extension precision, and lossless reconstruction. - [x] Implement allocation-free SIMD-first inverse DCT for every 4/8/16/32 width and height combination, the four-by-four intra-luma inverse DST, normative intermediate and residual clipping, and saturated prediction addition for 8/10/12-bit samples. Verify normal and forced-scalar execution against fixed results and a dense scalar oracle. - [x] Implement the SIMD-first inverse-quantization primitive for flat and scaling-list paths across every transform size, component matrix, prediction mode, transform-skip scaling rule, signed normalization direction, and range-extension precision path. @@ -485,6 +488,9 @@ Implement and verify in dependency order: - [ ] Deblocking and sample-adaptive offset for every signaled luma/chroma and bit-depth path. - [ ] Tiles, wavefront entry points, dependent slices, and all other parallelization syntax permitted by the exposed still-image profiles. - [ ] Supplemental enhancement information that changes image presentation or metadata exposed by ImageSharp. + - [x] Establish exact HM 18 native-plane conformance for four real 8-bit item payloads covering 4:2:0, 4:4:4, cropping, multiple grid positions, and wavefront entry points. + - [x] Establish independent libheif/libde265 presentation comparisons for single images, grids, auxiliary alpha, ICC, and 4:4:4 output without weakening the exact native-plane codec oracle. + - [ ] Establish exact native-plane conformance for 8/10/12-bit monochrome, 4:2:0, 4:2:2, and 4:4:4 across every exposed HEVC profile and required Range Extensions tool. Each subsystem keeps a specification-shaped scalar oracle derived from the HEVC specification and the pinned implementation reference, while production hot paths are designed SIMD-first with scalar fallback. No valid syntax in the exposed HEIC profiles may terminate in an unsupported branch or silently omit a normative reconstruction stage. diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs index 657f1ed480..09342f01d3 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcCoefficientDecoder.cs @@ -359,8 +359,13 @@ internal sealed class HevcCoefficientDecoder : IDisposable int syntaxWidth = verticalScan ? parameters.Height : parameters.Width; int syntaxHeight = verticalScan ? parameters.Width : parameters.Height; bool isChroma = parameters.Plane != HevcPlane.Y; - x = ReadLastSignificantCoordinate(ref reader, isChroma, syntaxWidth, true); - y = ReadLastSignificantCoordinate(ref reader, isChroma, syntaxHeight, false); + int xPrefix = ReadLastSignificantPrefix(ref reader, isChroma, syntaxWidth, true); + int yPrefix = ReadLastSignificantPrefix(ref reader, isChroma, syntaxHeight, false); + + // The HEVC syntax carries both context-coded prefixes before either bypass-coded suffix. Decoding a suffix + // immediately after its own prefix changes the arithmetic bit order whenever both coordinates need suffixes. + x = ReadLastSignificantSuffix(ref reader, xPrefix); + y = ReadLastSignificantSuffix(ref reader, yPrefix); if (verticalScan) { (x, y) = (y, x); @@ -368,14 +373,14 @@ internal sealed class HevcCoefficientDecoder : IDisposable } /// - /// Decodes one last-significant coefficient coordinate from its context prefix and bypass suffix. + /// Decodes one context-coded last-significant coefficient coordinate prefix. /// /// The current entropy-substream syntax reader. /// Whether the coordinate belongs to a chroma transform block. /// The transform-block extent along the coded axis. /// Whether to use the horizontal rather than vertical context set. - /// The decoded zero-based coefficient coordinate. - private static int ReadLastSignificantCoordinate(ref HevcCabacSyntaxReader reader, bool isChroma, int size, bool horizontal) + /// The decoded coordinate prefix. + private static int ReadLastSignificantPrefix(ref HevcCabacSyntaxReader reader, bool isChroma, int size, bool horizontal) { int convertedSize = BitOperations.Log2((uint)size) - 2; int contextOffset = isChroma ? 0 : (convertedSize * 3) + ((convertedSize + 1) >> 2); @@ -395,6 +400,17 @@ internal sealed class HevcCoefficientDecoder : IDisposable } } + return prefix; + } + + /// + /// Expands one last-significant coordinate prefix with its bypass-coded suffix. + /// + /// The current entropy-substream syntax reader. + /// The context-coded coordinate prefix. + /// The decoded zero-based coefficient coordinate. + private static int ReadLastSignificantSuffix(ref HevcCabacSyntaxReader reader, int prefix) + { if (prefix <= 3) { return prefix; diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs index da21782593..3c115d4e3e 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs @@ -17,15 +17,16 @@ internal static class HevcDeblockingFilter private interface IEdgeOperator { /// - /// Loads four samples at one signed distance across the edge. + /// Loads samples at one signed distance across the edge. /// /// The reconstructed picture. /// The component plane. /// The first Q-side sample X coordinate. /// The first Q-side sample Y coordinate. /// The signed sample distance across the edge. - /// Four widened samples ordered along the edge. - public static abstract Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance); + /// The number of valid low lanes to load. + /// The widened samples ordered along the edge. + public static abstract Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count); /// /// Stores four samples at one signed distance across the edge. @@ -250,14 +251,14 @@ internal static class HevcDeblockingFilter return; } - Vector128 p3 = TOperator.LoadVector(picture, plane, x, y, -4); - Vector128 p2 = TOperator.LoadVector(picture, plane, x, y, -3); - Vector128 p1 = TOperator.LoadVector(picture, plane, x, y, -2); - Vector128 p0 = TOperator.LoadVector(picture, plane, x, y, -1); - Vector128 q0 = TOperator.LoadVector(picture, plane, x, y, 0); - Vector128 q1 = TOperator.LoadVector(picture, plane, x, y, 1); - Vector128 q2 = TOperator.LoadVector(picture, plane, x, y, 2); - Vector128 q3 = TOperator.LoadVector(picture, plane, x, y, 3); + Vector128 p3 = TOperator.LoadVector(picture, plane, x, y, -4, 4); + Vector128 p2 = TOperator.LoadVector(picture, plane, x, y, -3, 4); + Vector128 p1 = TOperator.LoadVector(picture, plane, x, y, -2, 4); + Vector128 p0 = TOperator.LoadVector(picture, plane, x, y, -1, 4); + Vector128 q0 = TOperator.LoadVector(picture, plane, x, y, 0, 4); + Vector128 q1 = TOperator.LoadVector(picture, plane, x, y, 1, 4); + Vector128 q2 = TOperator.LoadVector(picture, plane, x, y, 2, 4); + Vector128 q3 = TOperator.LoadVector(picture, plane, x, y, 3, 4); // Each Int32 lane is one row or column along the edge. The threshold decision is shared by all four lanes, // while the filter arithmetic stays lane-local and exactly matches the scalar equations below. @@ -366,10 +367,10 @@ internal static class HevcDeblockingFilter return; } - Vector128 p1Vector = TOperator.LoadVector(picture, plane, x, y, -2); - Vector128 p0Vector = TOperator.LoadVector(picture, plane, x, y, -1); - Vector128 q0Vector = TOperator.LoadVector(picture, plane, x, y, 0); - Vector128 q1Vector = TOperator.LoadVector(picture, plane, x, y, 1); + Vector128 p1Vector = TOperator.LoadVector(picture, plane, x, y, -2, count); + Vector128 p0Vector = TOperator.LoadVector(picture, plane, x, y, -1, count); + Vector128 q0Vector = TOperator.LoadVector(picture, plane, x, y, 0, count); + Vector128 q1Vector = TOperator.LoadVector(picture, plane, x, y, 1, count); Vector128 deltaVector = (((q0Vector - p0Vector) * 4) + p1Vector - q1Vector + Vector128.Create(4)) >> 3; deltaVector = Vector128.Clamp(deltaVector, Vector128.Create(-tc), Vector128.Create(tc)); Vector128 minimum = Vector128.Zero; @@ -529,12 +530,25 @@ internal static class HevcDeblockingFilter { /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance) - => Vector128.Create( + public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) + { + if (count == 4) + { + return Vector128.Create( + (int)picture.GetRowSpan(plane, y)[x + distance], + picture.GetRowSpan(plane, y + 1)[x + distance], + picture.GetRowSpan(plane, y + 2)[x + distance], + picture.GetRowSpan(plane, y + 3)[x + distance]); + } + + // Subsampled chroma edges contain two samples. Zeroing the unused lanes keeps the vector path within + // the plane while allowing the shared kernel to operate on both valid samples in one instruction stream. + return Vector128.Create( (int)picture.GetRowSpan(plane, y)[x + distance], picture.GetRowSpan(plane, y + 1)[x + distance], - picture.GetRowSpan(plane, y + 2)[x + distance], - picture.GetRowSpan(plane, y + 3)[x + distance]); + 0, + 0); + } /// [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -571,9 +585,15 @@ internal static class HevcDeblockingFilter { /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance) + public static Vector128 LoadVector(HevcPictureBuffer picture, HevcPlane plane, int x, int y, int distance, int count) { ref ushort source = ref picture.GetRowSpan(plane, y + distance)[x]; + if (count == 2) + { + // The packed load used by full-width segments would read two samples beyond a subsampled edge. + return Vector128.Create((int)source, Unsafe.Add(ref source, 1), 0, 0); + } + Vector64 packed = Unsafe.As>(ref source); return Vector128.WidenLower(Vector128.Create(packed, Vector64.Zero)).AsInt32(); } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingState.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingState.cs index 681b723070..cf66a6ab5a 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingState.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingState.cs @@ -39,11 +39,13 @@ internal sealed class HevcDeblockingState : IDisposable { int width = DivideCeilingByPowerOfTwo(sequenceParameterSet.Width, UnitLog2); int height = DivideCeilingByPowerOfTwo(sequenceParameterSet.Height, UnitLog2); + + // MarkBlock combines sparse edge flags with existing values, so zero initialization is part of the state contract. this.boundaries = [ - configuration.MemoryAllocator.Allocate2D(width, height), - configuration.MemoryAllocator.Allocate2D(width, height), - configuration.MemoryAllocator.Allocate2D(width, height), + configuration.MemoryAllocator.Allocate2D(width, height, AllocationOptions.Clean), + configuration.MemoryAllocator.Allocate2D(width, height, AllocationOptions.Clean), + configuration.MemoryAllocator.Allocate2D(width, height, AllocationOptions.Clean), ]; } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs index c7779f2da7..ec2ccae2f7 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictionState.cs @@ -55,6 +55,21 @@ internal sealed class HevcIntraPredictionState : IDisposable /// private readonly Buffer2D chromaModes; + /// + /// The resolved chroma intra mode at minimum-prediction-block resolution in luma coordinates. + /// + private readonly Buffer2D effectiveChromaModes; + + /// + /// Whether derived chroma prediction selects the colocated luma prediction block. + /// + private readonly bool derivedChromaUsesColocatedLuma; + + /// + /// The mask selecting a luma coordinate within its coding-tree block. + /// + private readonly int codingTreeBlockMask; + /// /// Initializes a new instance of the class. /// @@ -77,6 +92,13 @@ internal sealed class HevcIntraPredictionState : IDisposable this.chromaModes = configuration.MemoryAllocator.Allocate2D( this.WidthInMinPredictionBlocks, this.HeightInMinPredictionBlocks); + + this.effectiveChromaModes = configuration.MemoryAllocator.Allocate2D( + this.WidthInMinPredictionBlocks, + this.HeightInMinPredictionBlocks); + + this.derivedChromaUsesColocatedLuma = sequenceParameterSet.ChromaFormat == 3; + this.codingTreeBlockMask = (1 << sequenceParameterSet.CodingTreeBlockLog2) - 1; } /// @@ -130,7 +152,10 @@ internal sealed class HevcIntraPredictionState : IDisposable int predictionX = x + offsetX; int predictionY = y + offsetY; bool predictionLeftAvailable = offsetX != 0 || leftAvailable; - bool predictionAboveAvailable = offsetY != 0 || aboveAvailable; + + // Luma MPM derivation treats an above prediction unit across a CTB boundary as unavailable. This is + // narrower than sample reconstruction availability and keeps the candidate order synchronized with CABAC. + bool predictionAboveAvailable = (predictionY & this.codingTreeBlockMask) != 0 && (offsetY != 0 || aboveAvailable); this.GetMostProbableLumaModes( predictionX, @@ -161,31 +186,45 @@ internal sealed class HevcIntraPredictionState : IDisposable } /// - /// Decodes and records the chroma intra mode of one leaf coding unit. + /// Decodes and records the chroma intra modes of one leaf coding unit. /// /// The current entropy-substream syntax reader. /// The coding-unit left coordinate in luma samples. /// The coding-unit top coordinate in luma samples. /// The base-two logarithm of the square coding-unit size. - public void DecodeChromaMode(ref HevcCabacSyntaxReader reader, int x, int y, int log2Size) + /// Whether the coding unit contains four luma prediction units. + public void DecodeChromaModes(ref HevcCabacSyntaxReader reader, int x, int y, int log2Size, bool usesNxNPartitions) { - int selector = reader.ReadChromaPredictionModeIndex(); - byte mode; - if (selector < 0) - { - mode = DerivedChromaMode; - } - else + bool usesFourChromaPredictionUnits = this.derivedChromaUsesColocatedLuma && usesNxNPartitions; + int predictionBlockLog2 = usesFourChromaPredictionUnits ? log2Size - 1 : log2Size; + int predictionBlockSize = 1 << predictionBlockLog2; + int predictionBlockCount = usesFourChromaPredictionUnits ? 4 : 1; + for (int index = 0; index < predictionBlockCount; index++) { - ReadOnlySpan candidates = [PlanarMode, VerticalMode, HorizontalMode, DcMode]; - mode = candidates[selector]; - if (mode == this.GetLumaMode(x, y)) + int predictionX = x + ((index & 1) * predictionBlockSize); + int predictionY = y + ((index >> 1) * predictionBlockSize); + int selector = reader.ReadChromaPredictionModeIndex(); + byte mode; + if (selector < 0) { - mode = ChromaReplacementMode; + mode = DerivedChromaMode; + } + else + { + ReadOnlySpan candidates = [PlanarMode, VerticalMode, HorizontalMode, DcMode]; + mode = candidates[selector]; + if (mode == this.GetLumaMode(predictionX, predictionY)) + { + mode = ChromaReplacementMode; + } } - } - this.SetMode(this.chromaModes, x, y, log2Size, mode); + // Combined 4:4:4 follows the four luma prediction partitions of an NxN coding unit. Subsampled formats + // carry one chroma mode for the coding unit and derive it from the top-left luma partition when requested. + this.SetMode(this.chromaModes, predictionX, predictionY, predictionBlockLog2, mode); + byte effectiveMode = mode == DerivedChromaMode ? this.GetLumaMode(predictionX, predictionY) : mode; + this.SetMode(this.effectiveChromaModes, predictionX, predictionY, predictionBlockLog2, effectiveMode); + } } /// @@ -213,10 +252,7 @@ internal sealed class HevcIntraPredictionState : IDisposable /// The luma sample Y coordinate. /// The explicit chroma mode, or the colocated luma mode when chroma uses derived mode. public byte GetEffectiveChromaMode(int x, int y) - { - byte mode = this.GetChromaMode(x, y); - return mode == DerivedChromaMode ? this.GetLumaMode(x, y) : mode; - } + => this.effectiveChromaModes.DangerousGetRowSpan(y >> MinPredictionBlockLog2)[x >> MinPredictionBlockLog2]; /// /// Releases the owned intra-mode maps. @@ -225,6 +261,7 @@ internal sealed class HevcIntraPredictionState : IDisposable { this.lumaModes.Dispose(); this.chromaModes.Dispose(); + this.effectiveChromaModes.Dispose(); } /// diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs index 1dfb66a28b..05185bed9a 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.OperatorImplementations.cs @@ -32,8 +32,10 @@ internal static partial class HevcIntraPredictor { ref ushort topBase = ref MemoryMarshal.GetReference(top); ref ushort destinationBase = ref MemoryMarshal.GetReference(destination); - uint bottomLeft = left[size]; - uint topRight = top[size]; + + // Index zero is the shared corner, so the planar endpoint at coordinate N is stored at N + 1. + uint bottomLeft = left[size + 1]; + uint topRight = top[size + 1]; int shift = BitOperations.Log2((uint)size) + 1; uint rounding = (uint)size; diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.References.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.References.cs index 2c33acba18..b6e1f985f5 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.References.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.References.cs @@ -133,13 +133,19 @@ internal static partial class HevcIntraPredictor line.Slice(leftSampleCount, unitWidth).Fill(picture.GetRowSpan(plane, y - 1)[x - 1]); } - ReadOnlySpan aboveRow = picture.GetRowSpan(plane, y - 1); int topStart = leftSampleCount + unitWidth; - for (int unit = 0; unit < aboveUnitCount; unit++) + ReadOnlySpan aboveAvailability = availability.Slice(cornerUnit + 1, aboveUnitCount); + if (aboveAvailability.IndexOf(true) >= 0) { - if (availability[cornerUnit + unit + 1]) + // A top-edge block can still have reconstructed left references. Load the preceding row only when + // the availability derivation proves that at least one above or above-right unit exists. + ReadOnlySpan aboveRow = picture.GetRowSpan(plane, y - 1); + for (int unit = 0; unit < aboveUnitCount; unit++) { - aboveRow.Slice(x + (unit * unitWidth), unitWidth).CopyTo(line.Slice(topStart + (unit * unitWidth), unitWidth)); + if (aboveAvailability[unit]) + { + aboveRow.Slice(x + (unit * unitWidth), unitWidth).CopyTo(line.Slice(topStart + (unit * unitWidth), unitWidth)); + } } } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs index ef10e5509e..fb5805cc27 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs @@ -455,7 +455,7 @@ internal static partial class HevcInverseTransformer for (; x <= oneVectorFromEnd; x += Vector128.Count) { ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref Unsafe.Add(ref destinationBase, x))); - Vector128 predicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsUInt16()).AsInt32(); + Vector128 predicted = Vector128.WidenLower(Vector128.CreateScalar(packed).AsUInt16()).AsInt32(); Vector128 reconstructed = Vector128.Clamp( predicted + Vector128.LoadUnsafe(ref residualBase, (nuint)x), Vector128.Zero, diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.TransformTree.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.TransformTree.cs index 88a0811687..eefc75e19e 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.TransformTree.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.TransformTree.cs @@ -149,6 +149,7 @@ internal sealed partial class HevcPictureDecoder bool canPredictAcrossComponents = this.pictureParameterSet.CrossComponentPredictionEnabled && primaryFlags.Any && chromaMode == 36 + && chromaBlueGeometry.Process && chromaBlueGeometry.Width == chromaBlueGeometry.Height; if (canPredictAcrossComponents) @@ -409,6 +410,7 @@ internal sealed partial class HevcPictureDecoder useLumaSyntax); this.coefficientDecoder.Decode(ref reader, quantized, in codingParameters); + bool rotate = HevcResidualReconstructor.IsNonTransformedResidualRotated( this.sequenceParameterSet.TransformSkipRotationEnabled, true, diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.Traversal.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.Traversal.cs index 05828d1fb7..9e294324c4 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.Traversal.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcPictureDecoder.Traversal.cs @@ -104,7 +104,7 @@ internal sealed partial class HevcPictureDecoder this.sampleAdaptiveOffsetState.SetLoopFilterRegion(rasterAddress, regionPlane, loopFilterRegion); this.DecodeSampleAdaptiveOffset(ref reader, independentSlice, rasterAddress, ctbX, ctbY, regionId); - bool endOfSliceSegment = this.DecodeCodingTree( + this.DecodeCodingTree( ref reader, x, y, @@ -113,6 +113,10 @@ internal sealed partial class HevcPictureDecoder regionId, colorPlaneIndex); + // HEVC places end_of_slice_segment_flag after the final coding unit of each complete CTB. Reading it + // inside the recursive leaf traversal consumes coefficient data whenever a CTB contains multiple CUs. + bool endOfSliceSegment = reader.ReadTerminate(); + // Wavefront synchronization copies probability and persistent Rice state after the second CTB of each // row. The next row starts with those contexts but a newly initialized arithmetic register. if (this.pictureParameterSet.EntropyCodingSynchronizationEnabled && columnInTile == 1) @@ -161,8 +165,7 @@ internal sealed partial class HevcPictureDecoder /// The coding-tree depth below the coding-tree-block root. /// The current independent-slice and tile prediction region. /// The selected separate-color plane, or zero for combined coding. - /// when the current leaf terminates the slice segment. - private bool DecodeCodingTree( + private void DecodeCodingTree( ref HevcCabacSyntaxReader reader, int x, int y, @@ -191,14 +194,18 @@ internal sealed partial class HevcPictureDecoder } } - if (depth == this.pictureParameterSet.QuantizationParameterDeltaDepth - && this.pictureParameterSet.CodingUnitQuantizationParameterDeltaEnabled) + bool startsQuantizationGroup = depth == this.pictureParameterSet.QuantizationParameterDeltaDepth + || (!split && depth < this.pictureParameterSet.QuantizationParameterDeltaDepth); + if (startsQuantizationGroup && this.pictureParameterSet.CodingUnitQuantizationParameterDeltaEnabled) { + // A leaf above the configured QG depth owns one complete quantization group. Waiting for the configured + // depth would carry the preceding group's coded-delta state into this coding unit and skip required syntax. this.BeginQuantizationGroup(x, y, regionId, colorPlaneIndex); } - if (depth == this.pictureParameterSet.ChromaQuantizationParameterOffsetDepth - && this.pictureParameterSet.ChromaQuantizationParameterOffsetsCb.Count != 0) + bool startsChromaQuantizationGroup = depth == this.pictureParameterSet.ChromaQuantizationParameterOffsetDepth + || (!split && depth < this.pictureParameterSet.ChromaQuantizationParameterOffsetDepth); + if (startsChromaQuantizationGroup && this.pictureParameterSet.ChromaQuantizationParameterOffsetsCb.Count != 0) { this.currentChromaQuantizationAdjustment = 0; this.chromaQuantizationAdjustmentPending = true; @@ -217,23 +224,20 @@ internal sealed partial class HevcPictureDecoder continue; } - if (this.DecodeCodingTree( + this.DecodeCodingTree( ref reader, childX, childY, childLog2Size, depth + 1, regionId, - colorPlaneIndex)) - { - return true; - } + colorPlaneIndex); } - return false; + return; } - return this.DecodeCodingUnit(ref reader, x, y, log2Size, depth, regionId, colorPlaneIndex); + this.DecodeCodingUnit(ref reader, x, y, log2Size, depth, regionId, colorPlaneIndex); } /// @@ -246,8 +250,7 @@ internal sealed partial class HevcPictureDecoder /// The coding-tree depth. /// The current independent-slice and tile prediction region. /// The selected separate-color plane, or zero for combined coding. - /// when this coding unit terminates the slice segment. - private bool DecodeCodingUnit( + private void DecodeCodingUnit( ref HevcCabacSyntaxReader reader, int x, int y, @@ -281,7 +284,7 @@ internal sealed partial class HevcPictureDecoder predictionState.DecodeLumaModes(ref reader, x, y, log2Size, usesNxNPartitions, leftAvailable, aboveAvailable); if (this.sequenceParameterSet.ChromaFormat != 0 && !this.sequenceParameterSet.SeparateColorPlaneFlag) { - predictionState.DecodeChromaMode(ref reader, x, y, log2Size); + predictionState.DecodeChromaModes(ref reader, x, y, log2Size, usesNxNPartitions); } int minimumTransformLog2 = GetMinimumTransformLog2Size(this.sequenceParameterSet, log2Size, usesNxNPartitions); @@ -319,7 +322,6 @@ internal sealed partial class HevcPictureDecoder pcm); this.lastCodedQuantizationParameter = this.currentQuantizationParameter; - return reader.ReadTerminate(); } /// @@ -332,8 +334,12 @@ internal sealed partial class HevcPictureDecoder private void BeginQuantizationGroup(int x, int y, int regionId, int colorPlaneIndex) { HevcPlane plane = this.sequenceParameterSet.SeparateColorPlaneFlag ? (HevcPlane)colorPlaneIndex : HevcPlane.Y; - bool leftAvailable = this.reconstructionState.IsReconstructed(plane, x - 1, y, regionId); - bool aboveAvailable = this.reconstructionState.IsReconstructed(plane, x, y - 1, regionId); + int codingTreeBlockMask = (1 << this.sequenceParameterSet.CodingTreeBlockLog2) - 1; + + // QP prediction neighbours are confined to the current CTB. This differs from intra sample availability, + // which may legitimately use reconstructed samples across the same left or upper CTB boundary. + bool leftAvailable = (x & codingTreeBlockMask) != 0 && this.reconstructionState.IsReconstructed(plane, x - 1, y, regionId); + bool aboveAvailable = (y & codingTreeBlockMask) != 0 && this.reconstructionState.IsReconstructed(plane, x, y - 1, regionId); int fallback = this.lastCodedQuantizationParameter; HevcCodingTreeState codingTreeState = this.codingTreeStates[colorPlaneIndex]; int left = leftAvailable ? codingTreeState.GetQuantizationParameter(x - 1, y) : fallback; diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcReconstructionState.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcReconstructionState.cs index fcba031c0e..6346fb75b6 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcReconstructionState.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcReconstructionState.cs @@ -53,11 +53,14 @@ internal sealed class HevcReconstructionState : IDisposable int heightInUnits = DivideCeilingByPowerOfTwo(this.height, MinPredictionBlockLog2); this.chromaSubsamplingX = !sequenceParameterSet.SeparateColorPlaneFlag && sequenceParameterSet.ChromaFormat is 1 or 2 ? 1 : 0; this.chromaSubsamplingY = !sequenceParameterSet.SeparateColorPlaneFlag && sequenceParameterSet.ChromaFormat == 1 ? 1 : 0; + + // Region identifiers gate every reconstructed-neighbor read. A stale pooled identifier can match the first + // region of a later picture, so these maps must begin at the reserved unavailable value zero. this.regions = [ - configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits), - configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits), - configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits), + configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits, AllocationOptions.Clean), + configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits, AllocationOptions.Clean), + configuration.MemoryAllocator.Allocate2D(widthInUnits, heightInUnits, AllocationOptions.Clean), ]; } diff --git a/src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetParameters.cs b/src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetParameters.cs index bd7e6a6758..41511b4fb0 100644 --- a/src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetParameters.cs +++ b/src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetParameters.cs @@ -282,6 +282,10 @@ internal sealed class HevcSampleAdaptiveOffsetState : IDisposable public HevcSampleAdaptiveOffsetState(Configuration configuration, int codingTreeBlockCount) { this.parameters = configuration.MemoryAllocator.Allocate(codingTreeBlockCount * 3); + + // Slice headers can disable SAO independently for luma and chroma. Initialize every component record to Off + // so an enabled component never causes untouched records from pooled memory to enter the picture-level pass. + this.parameters.Memory.Span.Clear(); this.regions = configuration.MemoryAllocator.Allocate(codingTreeBlockCount * 3); this.loopFilterRegions = configuration.MemoryAllocator.Allocate(codingTreeBlockCount * 3); } diff --git a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs index 4796edce1c..e66fc7cffb 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/HeifDecoderTests.cs @@ -64,6 +64,37 @@ public class HeifDecoderTests Assert.Equal(HeifCompressionMethod.LegacyJpeg, heicMetadata.CompressionMethod); } + /// + /// Verifies complete HEVC still-image decoding for real grid, auxiliary-alpha, 4:2:0, and 4:4:4 HEIC inputs. + /// + /// The real HEIC input and matching output naming context. + /// The independently reported presented width. + /// The independently reported presented height. + /// Whether the presented image carries an ICC profile. + [Theory] + [WithFile(TestImages.Heif.Image1, PixelTypes.Rgba32, 3992, 2992, true)] + [WithFile(TestImages.Heif.Image2, PixelTypes.Rgba32, 3464, 2130, true)] + [WithFile(TestImages.Heif.Image3, PixelTypes.Rgba32, 4242, 2828, true)] + [WithFile(TestImages.Heif.Image4, PixelTypes.Rgba32, 700, 476, true)] + [WithFile(TestImages.Heif.Sample640x427, PixelTypes.Rgba32, 640, 428, false)] + public void DecodeHevcStillImage(TestImageProvider provider, int width, int height, bool hasIccProfile) + where TPixel : unmanaged, IPixel + { + using Image image = provider.GetImage(); + HeifMetadata metadata = image.Metadata.GetHeifMetadata(); + image.DebugSave(provider); + + // The extracted native YUV tiles have byte-exact HM coverage. Pinned libheif 1.23.1 selects its cheaper fused + // nearest-neighbor RGB path by default, so the full-image oracle permits the bounded difference from our + // bilinear reconstruction while still covering grids, alpha composition, color conversion, and presentation. + image.CompareToReferenceOutput(ImageComparer.TolerantPercentage(0.6F), provider); + + Assert.Equal(new Size(width, height), image.Size); + Assert.Equal(HeifCompressionMethod.Hevc, metadata.CompressionMethod); + Assert.Equal(HeifBitDepth.Bit8, metadata.BitDepth); + Assert.Equal(hasIccProfile, image.Metadata.IccProfile is not null); + } + /// /// Verifies that AVIF decoding preserves the exact embedded ICC profile bytes. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcInverseTransformerTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcInverseTransformerTests.cs index f7374e7598..40265650ff 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcInverseTransformerTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcInverseTransformerTests.cs @@ -149,6 +149,21 @@ public class HevcInverseTransformerTests Assert.All(negative, value => Assert.Equal((ushort)0, value)); } + /// + /// Verifies that SIMD residual reconstruction preserves each predicted lane for an extracted HEVC transform block. + /// + [Fact] + public void AddResidualMatchesExtractedHevcBlock() + { + int[] residual = [-6, 0, 2, 4, -9, -25, -15, -2]; + ushort[] destination = [154, 154, 154, 154, 154, 154, 154, 154]; + ushort[] expected = [148, 154, 156, 158, 145, 129, 139, 152]; + + HevcInverseTransformer.AddResidual(residual, destination, destination.Length, destination.Length, 1, 8); + + Assert.True(expected.AsSpan().SequenceEqual(destination)); + } + /// /// Applies both inverse-transform dimensions using direct matrix products and normative rounding points. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs new file mode 100644 index 0000000000..0a03804972 --- /dev/null +++ b/tests/ImageSharp.Tests/Formats/Heif/Hevc/HevcPictureDecoderTests.cs @@ -0,0 +1,122 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using SixLabors.ImageSharp.Formats.Heif.Hevc; + +namespace SixLabors.ImageSharp.Tests.Formats.Heif.Hevc; + +/// +/// Validates complete HEVC still-picture reconstruction against independently decoded samples. +/// +[Trait("Format", "Heif")] +public class HevcPictureDecoderTests +{ + /// + /// Verifies all reconstructed samples from a real HEIC grid tile against the HM reference decoder. + /// + /// The exact HEVC decoder-configuration record associated with the item. + /// The exact HEVC item payload to decode. + /// The corresponding planar samples produced by HM. + [Theory] + [InlineData(TestImages.Heif.Image1TileHvcConfiguration, TestImages.Heif.Image1Tile1Payload, TestImages.Heif.Image1Tile1ReferenceYuv)] + [InlineData(TestImages.Heif.Image1TileHvcConfiguration, TestImages.Heif.Image1Tile2Payload, TestImages.Heif.Image1Tile2ReferenceYuv)] + [InlineData(TestImages.Heif.Image2TileHvcConfiguration, TestImages.Heif.Image2Tile1Payload, TestImages.Heif.Image2Tile1ReferenceYuv)] + [InlineData(TestImages.Heif.DwsampleTileHvcConfiguration, TestImages.Heif.DwsampleTilePayload, TestImages.Heif.DwsampleTileReferenceYuv)] + public void DecodeRealHeicTileMatchesHmReference(string configurationPath, string itemPath, string referencePath) + { + byte[] configurationData = TestFile.Create(configurationPath).Bytes; + byte[] itemData = TestFile.Create(itemPath).Bytes; + byte[] expectedYuv = TestFile.Create(referencePath).Bytes; + HevcCodecConfiguration configuration = new(configurationData); + HevcImageItemBitstream bitstream = new(itemData, configuration); + HevcSequenceParameterSet sequenceParameterSet = bitstream.SliceSegments[0].PictureParameterSet.SequenceParameterSet; + using HevcPictureDecoder decoder = new(Configuration.Default, bitstream.SliceSegments[0].PictureParameterSet); + + decoder.Decode(bitstream); + + int expectedLength = sequenceParameterSet.DisplayWidth * sequenceParameterSet.DisplayHeight; + if (decoder.Picture.ChromaFormat != 0) + { + int chromaWidth = GetDisplaySize(sequenceParameterSet.DisplayWidth, decoder.Picture.GetSubsamplingX(HevcPlane.Cb)); + int chromaHeight = GetDisplaySize(sequenceParameterSet.DisplayHeight, decoder.Picture.GetSubsamplingY(HevcPlane.Cb)); + expectedLength += 2 * chromaWidth * chromaHeight; + } + + Assert.Equal(expectedLength, expectedYuv.Length); + int offset = 0; + AssertPlaneEqual(decoder.Picture, sequenceParameterSet, HevcPlane.Y, expectedYuv, ref offset); + if (decoder.Picture.ChromaFormat != 0) + { + AssertPlaneEqual(decoder.Picture, sequenceParameterSet, HevcPlane.Cb, expectedYuv, ref offset); + AssertPlaneEqual(decoder.Picture, sequenceParameterSet, HevcPlane.Cr, expectedYuv, ref offset); + } + + Assert.Equal(expectedYuv.Length, offset); + } + + /// + /// Compares one decoded component plane with its planar reference samples. + /// + /// The decoded picture containing the component plane. + /// The coded and displayed picture geometry. + /// The component plane to compare. + /// The complete planar YUV reference. + /// The current reference offset, advanced past the compared plane. + private static void AssertPlaneEqual( + HevcPictureBuffer picture, + HevcSequenceParameterSet sequenceParameterSet, + HevcPlane plane, + ReadOnlySpan expected, + ref int offset) + { + int subsamplingX = picture.GetSubsamplingX(plane); + int subsamplingY = picture.GetSubsamplingY(plane); + int sourceX = sequenceParameterSet.ConformanceWindowLeftOffset >> subsamplingX; + int sourceY = sequenceParameterSet.ConformanceWindowTopOffset >> subsamplingY; + int width = GetDisplaySize(sequenceParameterSet.DisplayWidth, subsamplingX); + int height = GetDisplaySize(sequenceParameterSet.DisplayHeight, subsamplingY); + int mismatchCount = 0; + int maximumDifference = 0; + int firstMismatchX = 0; + int firstMismatchY = 0; + ushort firstActual = 0; + ushort firstExpected = 0; + for (int y = 0; y < height; y++) + { + Span actualRow = picture.GetRowSpan(plane, sourceY + y).Slice(sourceX, width); + for (int x = 0; x < width; x++) + { + ushort expectedSample = expected[offset++]; + int difference = Math.Abs(actualRow[x] - expectedSample); + if (difference == 0) + { + continue; + } + + if (mismatchCount == 0) + { + firstMismatchX = x; + firstMismatchY = y; + firstActual = actualRow[x]; + firstExpected = expectedSample; + } + + mismatchCount++; + maximumDifference = Math.Max(maximumDifference, difference); + } + } + + Assert.True( + mismatchCount == 0, + $"{plane} contained {mismatchCount} differing samples. The maximum difference was {maximumDifference}; " + + $"the first mismatch at ({firstMismatchX}, {firstMismatchY}) was {firstActual}, expected {firstExpected}."); + } + + /// + /// Converts a luma display extent to the selected component extent. + /// + /// The displayed luma extent. + /// The component subsampling shift. + /// The displayed component extent. + private static int GetDisplaySize(int lumaSize, int subsampling) => (lumaSize + (1 << subsampling) - 1) >> subsampling; +} diff --git a/tests/ImageSharp.Tests/TestImages.cs b/tests/ImageSharp.Tests/TestImages.cs index f937827297..ac98e8576c 100644 --- a/tests/ImageSharp.Tests/TestImages.cs +++ b/tests/ImageSharp.Tests/TestImages.cs @@ -1279,6 +1279,19 @@ public static class TestImages public static class Heif { + // The tile payloads are exact item extents from the named HEIC fixtures. Their planar references were decoded + // with the HEVC Test Model 18.0 decoder so codec conformance is checked independently of color conversion. + public const string Image1TileHvcConfiguration = "Heif/Hevc/image1-tile1.hvcc"; + public const string Image1Tile1Payload = "Heif/Hevc/image1-tile1.hvc1"; + public const string Image1Tile1ReferenceYuv = "Heif/Hevc/image1-tile1-hm.yuv"; + public const string Image1Tile2Payload = "Heif/Hevc/image1-tile2.hvc1"; + public const string Image1Tile2ReferenceYuv = "Heif/Hevc/image1-tile2-hm.yuv"; + public const string Image2TileHvcConfiguration = "Heif/Hevc/image2-tile.hvcc"; + public const string Image2Tile1Payload = "Heif/Hevc/image2-tile1.hvc1"; + public const string Image2Tile1ReferenceYuv = "Heif/Hevc/image2-tile1-hm.yuv"; + public const string DwsampleTileHvcConfiguration = "Heif/Hevc/dwsample-tile.hvcc"; + public const string DwsampleTilePayload = "Heif/Hevc/dwsample-tile.hvc1"; + public const string DwsampleTileReferenceYuv = "Heif/Hevc/dwsample-tile-hm.yuv"; public const string Image1 = "Heif/image1.heic"; public const string Image2 = "Heif/image2.heic"; public const string Image3 = "Heif/image3.heic"; diff --git a/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_dwsample-heic-640.png b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_dwsample-heic-640.png new file mode 100644 index 0000000000..786e143c6f --- /dev/null +++ b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_dwsample-heic-640.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c580c529e6371d050f04cf8ac3f1cd758eccedb2bcbaca37f78120c14035525 +size 464501 diff --git a/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image1.png b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image1.png new file mode 100644 index 0000000000..9aba8c0b67 --- /dev/null +++ b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eef9b60aae394fb2043ce3f6f51c0ead683de3f613b99ee6761c91afb7913c99 +size 17932847 diff --git a/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image2.png b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image2.png new file mode 100644 index 0000000000..99268f29c8 --- /dev/null +++ b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33d45d4efdbc3e1b4bc2c6dcbd6953a423350b9d7b39be00ed5b54a351dcff59 +size 11579818 diff --git a/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image3.png b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image3.png new file mode 100644 index 0000000000..bfe08d7504 --- /dev/null +++ b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f7bb5e5e7774177f2065bf752523cb0b053e4a18898278d58e1de35b9d9ee34c +size 10528077 diff --git a/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image4.png b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image4.png new file mode 100644 index 0000000000..d988609b8c --- /dev/null +++ b/tests/Images/External/ReferenceOutput/HeifDecoderTests/DecodeHevcStillImage_Rgba32_image4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86832e24985e7ffa41387a5a4993db9fee76b7fe8ebc829c8fe891b02638f8ff +size 195726 diff --git a/tests/Images/Input/Heif/Hevc/dwsample-tile-hm.yuv b/tests/Images/Input/Heif/Hevc/dwsample-tile-hm.yuv new file mode 100644 index 0000000000..e6900655f1 --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/dwsample-tile-hm.yuv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:655e8839c763574402cc02a1671204bc6b0faf22df7168eca3cbd917cc5a8890 +size 821760 diff --git a/tests/Images/Input/Heif/Hevc/dwsample-tile.hvc1 b/tests/Images/Input/Heif/Hevc/dwsample-tile.hvc1 new file mode 100644 index 0000000000..e70be49aaf --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/dwsample-tile.hvc1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c06484dc3196ed6c7f5e46ac7e10aaca1703541261e59107734b281537fe4f8 +size 191419 diff --git a/tests/Images/Input/Heif/Hevc/dwsample-tile.hvcc b/tests/Images/Input/Heif/Hevc/dwsample-tile.hvcc new file mode 100644 index 0000000000..97708f07ea --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/dwsample-tile.hvcc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d093eebbfacf797903c222c3b85512b25e741f11af1e660be9ec40b03a9cd4ef +size 113 diff --git a/tests/Images/Input/Heif/Hevc/image1-tile1-hm.yuv b/tests/Images/Input/Heif/Hevc/image1-tile1-hm.yuv new file mode 100644 index 0000000000..771b5e4059 --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image1-tile1-hm.yuv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d7a5c74c14d91ffb39cd4846f0b57e8a88fabd31935ab065e2c6f90817ed54db +size 393216 diff --git a/tests/Images/Input/Heif/Hevc/image1-tile1.hvc1 b/tests/Images/Input/Heif/Hevc/image1-tile1.hvc1 new file mode 100644 index 0000000000..a915119990 --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image1-tile1.hvc1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd8846579cc77df772687ebb89dbbda9ff192ad4ffdfaa080d8de7970fecd397 +size 106458 diff --git a/tests/Images/Input/Heif/Hevc/image1-tile1.hvcc b/tests/Images/Input/Heif/Hevc/image1-tile1.hvcc new file mode 100644 index 0000000000..58e61e05ae --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image1-tile1.hvcc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd729786f48693c2584175658291cffbae94cda18502735a383eeb61853b8f0d +size 102 diff --git a/tests/Images/Input/Heif/Hevc/image1-tile2-hm.yuv b/tests/Images/Input/Heif/Hevc/image1-tile2-hm.yuv new file mode 100644 index 0000000000..0f15687f49 --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image1-tile2-hm.yuv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4aa91b707f756ce52d6675283f77bd555352e6738ca0137dfe8526e593c0abb0 +size 393216 diff --git a/tests/Images/Input/Heif/Hevc/image1-tile2.hvc1 b/tests/Images/Input/Heif/Hevc/image1-tile2.hvc1 new file mode 100644 index 0000000000..691e7e4eb9 --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image1-tile2.hvc1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:976b53167fd8881b71b04b3e3e7fb9f9792c05f9e49ca682bac60a8d096bbafa +size 105758 diff --git a/tests/Images/Input/Heif/Hevc/image2-tile.hvcc b/tests/Images/Input/Heif/Hevc/image2-tile.hvcc new file mode 100644 index 0000000000..1e0a32339b --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image2-tile.hvcc @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91b49d1a8ff5e91ea2fafe74df88538a85b76c7dba8086c94799c4a178cf46c2 +size 101 diff --git a/tests/Images/Input/Heif/Hevc/image2-tile1-hm.yuv b/tests/Images/Input/Heif/Hevc/image2-tile1-hm.yuv new file mode 100644 index 0000000000..893c67e2cc --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image2-tile1-hm.yuv @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69b8abe9e4faac13f1b7c212dae7da63716a87aa3cd8d7e9c70f00457b1c3fb2 +size 393216 diff --git a/tests/Images/Input/Heif/Hevc/image2-tile1.hvc1 b/tests/Images/Input/Heif/Hevc/image2-tile1.hvc1 new file mode 100644 index 0000000000..111292e5ec --- /dev/null +++ b/tests/Images/Input/Heif/Hevc/image2-tile1.hvc1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6eecfa72cb4fc895d9421f35f0fb916b0b54704618c66a135a48a7bfb7f1fd5 +size 21099