From 1639550a04500f8a744e53e16cf0c0d1d995a333 Mon Sep 17 00:00:00 2001 From: James Jackson-South Date: Sat, 5 Sep 2026 16:58:56 +1000 Subject: [PATCH] Integrate shared AV1 intra-edge preparation into encoding --- HEIF_IMPLEMENTATION_PLAN.md | 55 +- .../Heif/Av1/Pipeline/Av1FrameEncoder.cs | 2 +- ...traSuperblockEncoder.ChromaModeDecision.cs | 59 ++ .../Av1IntraSuperblockEncoder.ModeDecision.cs | 6 + .../Av1IntraSuperblockEncoder.Operator.cs | 24 + .../Av1/Pipeline/Av1TransformBlockEncoder.cs | 148 +++- .../Av1IntraEdgeFilter.Operations.cs | 210 ++++++ .../Prediction/Av1IntraEdgeFilter.Operator.cs | 113 +++ .../Av1IntraEdgeFilter.Strength1Operator.cs | 36 + .../Av1IntraEdgeFilter.Strength2Operator.cs | 36 + .../Av1IntraEdgeFilter.Strength3Operator.cs | 36 + .../Av1/Prediction/Av1IntraEdgePreparation.cs | 289 +++++++ .../Av1IntraEdgeUpsampler.FourTapOperator.cs | 48 ++ .../Av1IntraEdgeUpsampler.Operations.cs | 278 +++++++ .../Av1IntraEdgeUpsampler.Operator.cs | 91 +++ .../Av1/Prediction/Av1PredictionDecoder.cs | 705 +----------------- .../Formats/Heif/Av1/Av1EncoderFrameTests.cs | 1 + .../Av1/Av1IntraSuperblockEncoderTests.cs | 20 +- .../Formats/Heif/Av1/Av1PredictorTests.cs | 119 ++- .../Heif/Av1/Av1TransformBlockEncoderTests.cs | 4 + 20 files changed, 1545 insertions(+), 735 deletions(-) create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operations.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength1Operator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength2Operator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength3Operator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgePreparation.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.FourTapOperator.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operations.cs create mode 100644 src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operator.cs diff --git a/HEIF_IMPLEMENTATION_PLAN.md b/HEIF_IMPLEMENTATION_PLAN.md index 35c18d22fe..553c96f52e 100644 --- a/HEIF_IMPLEMENTATION_PLAN.md +++ b/HEIF_IMPLEMENTATION_PLAN.md @@ -255,21 +255,46 @@ Intra-reference frame-extent correction after checkpoint `182f39ae5`, verified o **29,668** samples, with **0** samples exceeding one. These are same-bitstream decoder/reconstruction comparisons; they do not establish separate-encoder parity or performance. No benchmark was run. -The intra-edge investigation also confirmed these unresolved integration requirements: - -- `Av1PredictionDecoder.cs:989-1837` owns separate directional preparation, edge smoothing, upsampling, - strength selection, and neighboring-mode selection. Native `reconintra.c:989-1082,1349-1381` uses endpoint - extension, rounded nonnegative smoothing kernels, and clipped signed four-tap half-sample interpolation. - No new numerical discrepancy in those arithmetic kernels has been established by this comparison. -- Encoder `Av1EncoderModeDecisionWorkspace.cs:50-53,133-135` retains four raw edge spans with one prefix sample. - The decoder needs writable prefix positions -1 and -2 and candidate-specific filtering; mutating those raw - encoder spans across mode trials would contaminate later candidates. `Av1EncoderBlockWorkspace.cs:143-144` - exposes transform scratch whose lifetime must be reconciled with directional prediction before sharing it. -- `Av1IntraSuperblockEncoder.ModeDecision.cs:2272-2466` draws tiled edges from both committed reconstruction - and the current candidate mosaic. Enabling filtering must preserve that distinction, coded extents, - chroma neighbor ownership, corner preparation, and smooth-neighbor-dependent thresholds across all callers. - The sequence flag remains disabled pending that complete integration. The decoder's private kernels are - not a substitute for the required shared closed-generic traversal and semantic-operator architecture. +Intra-edge integration after checkpoint `2424ff9f9`, verified on 2026-09-05: + +- Reference `av1/av1_cx_iface.c:333,1561-1562` enables intra-edge filtering by default and propagates it to + sequence configuration (`av1/encoder/encoder.c:641-647`). `Av1FrameEncoder.cs:402` now enables that syntax. + CDEF and restoration remain disabled and unresolved. The starting-tree findings above remain historical evidence. +- Encoder `Av1TransformBlockEncoder.cs:739-800,875-937` now prepares directional edges before prediction. + Luma mode trials, selected-mode transform refinement, split luma transforms, tiled planes, and chroma candidates + propagate both the sequence flag and the neighboring smooth-mode class. Raw references remain separate from + candidate copies; filtering does not mutate references used by subsequent mode or transform trials. +- Neighbor selection at `Av1IntraSuperblockEncoder.ChromaModeDecision.cs:1035-1081` follows native + `av1/common/av1_common_int.h:1359-1415` for the luma units that own subsampled chroma neighbors and + `reconintra.c:958-986` for smooth-mode classification. Inter winners can retain a previous intra trial's UV field + (`Av1IntraSuperblockEncoder.ReferenceModeDecision.cs:925-933`), so that field is only meaningful for an intra neighbor. +- `Av1IntraEdgePreparation.cs:39-116` shares the complete corner/filter/upsampling order between encoder and decoder. + Native `reconintra.c:1132-1147,1204-1243,1512-1548` defines the missing-sole-edge early return and directional + preparation. Strength thresholds follow `reconintra.c:989-1026`; half-sample selection follows `reconintra.h:148-155`. + The shared code preserves a missing sole edge's constant value rather than interpolating its distinct corner. +- `Av1IntraEdgeFilter` and `Av1IntraEdgeUpsampler` have separate closed generic traversals and semantic readonly + operators, with descending 512/256/128-bit widths and scalar tails. Smoothing uses rounded nonnegative kernels; + upsampling uses signed [-1,9,9,-1] arithmetic, rounding, clipping, and linear interleaving. Native definitions are + `reconintra.c:1028-1082,1349-1381`. Inline comments explain endpoint padding, lane ordering, bounds, and scaling. +- Each candidate borrows existing transform scratch (`Av1EncoderBlockWorkspace.cs:143-144`) until prediction and + residual formation finish. Two 160-sample edges retain native prefix sizing; only required edges are copied. + Smoothing uses 132 samples including three endpoint padding positions. Upsampling needs exactly the native + 19 samples, including corner and endpoint extension; vector reads no longer require a larger padded window. + Decoder scratch is 4,548 short samples (about 8.88 KiB), replacing its previous 4,576-sample workspace. + No new owner or per-candidate allocation was added. This source-level sizing result is not a timing claim. +- Existing independent scalar kernel tests now cover all SIMD tiers, lengths around lane boundaries, extrema, + and exact scratch capacities. Eight added preparation cases distinguish smooth-neighbor thresholds and missing + sole edges in both orientations. Production tests assert the emitted sequence flag; mixed-partition tests retain + the four unfiltered cases and add four filtered cases without weakening partition or reconstruction assertions. +- Final Release net11.0 build: zero errors and 1,009 existing warnings. Roslynk: zero compiler errors. + Serialized Visual Studio VSTest passed **314/314** in `intra-edge-final.trx` (30.7293 seconds), including encoder + frames, intra-superblocks, transform-block contracts, predictor SIMD tiers, native decoder fixtures and fallbacks, + HEIF encoder contracts, and the retained empty-transform cost-helper test. +- Fresh optimized-reference decoding of eight regenerated partition streams matches all 16,640 retained luma samples. + Twelve regenerated moving color streams match all 21,348 Y/U/V samples. Combined maximum error is **0** across + **37,988** samples, with **0** exceeding one. These remain bounded same-bitstream reconstruction comparisons; + separate-encoder sample parity, complete decoder coverage, and end-to-end performance are still unverified. + No benchmark was run. Temporary scripts, native output, and reports remain outside the commit. ### Required completion gates diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs index e2db3b5c24..0dd214715f 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1FrameEncoder.cs @@ -399,7 +399,7 @@ internal static class Av1FrameEncoder ForceIntegerMotionVector = Av1Constants.SelectIntegerMotionVector, EnableFilterIntra = effort >= 4, EnableDualFilter = !isStillPicture && effort >= MinimumDualInterpolationEffort, - EnableIntraEdgeFilter = false, + EnableIntraEdgeFilter = true, EnableSuperResolution = false, EnableCdef = false, EnableRestoration = false, diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs index 61ffe4e8a9..f60f263d8a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ChromaModeDecision.cs @@ -266,6 +266,7 @@ internal static partial class Av1IntraSuperblockEncoder redLeft, hasLeft, hasAbove, + this.UseSmoothIntraEdges(macroBlock, lumaOrigin, blockSize, Av1Plane.U), blueContext, redContext, paletteDisabledCost, @@ -882,6 +883,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, predictionMode, angleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + this.UseSmoothIntraEdges(macroBlock, lumaOrigin, blockSize, plane), residual, transformSize, this.bitDepth); @@ -1029,6 +1032,57 @@ internal static partial class Av1IntraSuperblockEncoder return distortion; } + /// + /// Derives the directional edge-filter class from the relevant neighboring coding blocks. + /// + private bool UseSmoothIntraEdges(Av1MacroBlockD macroBlock, Point lumaOrigin, Av1BlockSize blockSize, Av1Plane plane) + { + ObuColorConfig colorConfig = this.picture.Sequence.SequenceHeader.ColorConfig; + int subX = plane == Av1Plane.Y ? 0 : colorConfig.SubSamplingX ? 1 : 0; + int subY = plane == Av1Plane.Y ? 0 : colorConfig.SubSamplingY ? 1 : 0; + int row = lumaOrigin.Y >> Av1Constants.ModeInfoSizeLog2; + int column = lumaOrigin.X >> Av1Constants.ModeInfoSizeLog2; + bool hasAbove = macroBlock.IsUpAvailable; + bool hasLeft = macroBlock.IsLeftAvailable; + if (subX != 0 && blockSize.Get4x4WideCount() < 2) + { + hasLeft = column - 1 > macroBlock.Tile.ModeInfoColumnStart; + } + + if (subY != 0 && blockSize.Get4x4HighCount() < 2) + { + hasAbove = row - 1 > macroBlock.Tile.ModeInfoRowStart; + } + + // Chroma may cover several luma units. Its neighbors are the bottom-right luma units in the + // adjacent chroma regions, measured from the top-left unit covered by the current chroma block. + int baseOffset = -((row & subY) * macroBlock.ModeInfoStride) - (column & subX); + if (hasAbove && IsSmoothIntraNeighbor( + macroBlock.GetRelativeModeInfo(baseOffset - macroBlock.ModeInfoStride + subX).Block, plane)) + { + return true; + } + + return hasLeft && IsSmoothIntraNeighbor( + macroBlock.GetRelativeModeInfo(baseOffset + (subY * macroBlock.ModeInfoStride) - 1).Block, plane); + } + + /// + /// Determines whether a neighboring block supplies the smooth edge-filter class. + /// + private static bool IsSmoothIntraNeighbor(Av1EncoderBlockModeInfo modeInfo, Av1Plane plane) + { + if (plane == Av1Plane.Y) + { + return modeInfo.Mode is Av1PredictionMode.Smooth or Av1PredictionMode.SmoothVertical or Av1PredictionMode.SmoothHorizontal; + } + + // An inter winner can retain the preceding intra trial's UV field. That field has no inter + // meaning, so only an ordinary intra neighbor can select chroma smooth-edge thresholds. + return !modeInfo.UseIntraBlockCopy && modeInfo.Mode < Av1PredictionMode.InterModeStart + && modeInfo.UvMode is Av1ChromaPredictionMode.Smooth or Av1ChromaPredictionMode.SmoothVertical or Av1ChromaPredictionMode.SmoothHorizontal; + } + private long GetChromaCandidateCost( Av1SymbolEncoder writer, Av1MacroBlockModeInfo modeInfo, @@ -1046,6 +1100,7 @@ internal static partial class Av1IntraSuperblockEncoder ReadOnlySpan redLeft, bool hasLeft, bool hasAbove, + bool smoothIntraEdges, Av1TransformBlockContext blueContext, Av1TransformBlockContext redContext, int paletteDisabledCost, @@ -1078,6 +1133,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, predictionMode, angleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + smoothIntraEdges, candidateBlueCoefficients, transformSize, transformType, @@ -1099,6 +1156,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, predictionMode, angleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + smoothIntraEdges, candidateRedCoefficients, transformSize, transformType, diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs index f24353c8d6..8f76f64e25 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.ModeDecision.cs @@ -1408,6 +1408,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + this.UseSmoothIntraEdges(macroBlock, blockOrigin, blockSize, Av1Plane.Y), residual, transformSize, this.bitDepth); @@ -1539,6 +1541,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, bestMode, selectedAngleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + this.UseSmoothIntraEdges(macroBlock, blockOrigin, blockSize, Av1Plane.Y), residual, transformSize, this.bitDepth); @@ -2131,6 +2135,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + this.picture.Sequence.SequenceHeader.EnableIntraEdgeFilter, + this.UseSmoothIntraEdges(macroBlock, blockOrigin, BlockSize, Av1Plane.Y), residual, TransformSize, this.bitDepth); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs index 51bdcf4be8..7bf726c86a 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1IntraSuperblockEncoder.Operator.cs @@ -166,6 +166,8 @@ internal static partial class Av1IntraSuperblockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The candidate entropy-coding coefficients. /// The transform dimensions. /// The compound transform applied to the residual. @@ -187,6 +189,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -210,6 +214,8 @@ internal static partial class Av1IntraSuperblockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The contiguous source-minus-prediction destination. /// The prediction dimensions. /// The coded sample bit depth. @@ -224,6 +230,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span residual, Av1TransformSize transformSize, Av1BitDepth bitDepth); @@ -685,6 +693,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -705,6 +715,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, quantizedCoefficients, transformSize, transformType, @@ -726,6 +738,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span residual, Av1TransformSize transformSize, Av1BitDepth bitDepth) @@ -741,6 +755,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, residual, transformSize); @@ -1197,6 +1213,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -1217,6 +1235,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, quantizedCoefficients, transformSize, transformType, @@ -1239,6 +1259,8 @@ internal static partial class Av1IntraSuperblockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span residual, Av1TransformSize transformSize, Av1BitDepth bitDepth) @@ -1254,6 +1276,8 @@ internal static partial class Av1IntraSuperblockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, residual, transformSize, bitDepth); diff --git a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs index a61ec59951..cf295115e2 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Pipeline/Av1TransformBlockEncoder.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Quantizers; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; @@ -68,6 +69,8 @@ internal static class Av1TransformBlockEncoder hasAbove, Av1PredictionMode.DC, 0, + false, + false, quantizedCoefficients, transformSize, transformType, @@ -91,6 +94,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The candidate entropy-coding coefficients. /// The selected transform dimensions. /// The selected compound transform type. @@ -111,6 +116,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -136,6 +143,8 @@ internal static class Av1TransformBlockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, quantizedCoefficients, transformSize, transformType, @@ -383,6 +392,8 @@ internal static class Av1TransformBlockEncoder hasAbove, Av1PredictionMode.DC, 0, + false, + false, quantizedCoefficients, transformSize, transformType, @@ -407,6 +418,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The candidate entropy-coding coefficients. /// The selected transform dimensions. /// The selected compound transform type. @@ -428,6 +441,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -454,6 +469,8 @@ internal static class Av1TransformBlockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, quantizedCoefficients, transformSize, transformType, @@ -690,6 +707,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The compact source-minus-prediction destination. /// The prediction dimensions. public static void PrepareIntraPrediction( @@ -704,6 +723,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span residual, Av1TransformSize transformSize) { @@ -718,9 +739,53 @@ internal static class Av1TransformBlockEncoder } else if (mode.IsDirectional()) { - // The current encoder disables intra-edge filtering in sequence syntax. Zone-three transposition - // borrows transform scratch because prediction completes before forward transformation starts. - Span directionalScratch = MemoryMarshal.AsBytes(workspace.TransformWorkspace)[..(width * height)]; + int angle = mode.ToAngle() + (angleDelta * Av1Constants.AngleStep); + Span scratch = MemoryMarshal.AsBytes(workspace.TransformWorkspace); + int predictionLength = width * height; + Span directionalScratch = scratch[..predictionLength]; + bool upsampleAbove = false; + bool upsampleLeft = false; + if (enableIntraEdgeFilter) + { + // Mode trials share raw references. Prepare private edge copies after the directional scratch; + // this entire transform workspace is reusable once prediction and residual formation finish. + int edgeLength = Av1IntraEdgePreparation.ReferenceBufferLength; + int prefixLength = Av1IntraEdgePreparation.ReferencePrefixLength; + Span aboveStorage = scratch.Slice(predictionLength, edgeLength); + Span leftStorage = scratch.Slice(predictionLength + edgeLength, edgeLength); + aboveStorage.Fill(127); + leftStorage.Fill(129); + if (angle < 180) + { + above.CopyTo(aboveStorage[prefixLength..]); + aboveStorage[prefixLength - 1] = Unsafe.Subtract(ref MemoryMarshal.GetReference(above), 1); + } + + if (angle > 90) + { + left.CopyTo(leftStorage[prefixLength..]); + leftStorage[prefixLength - 1] = Unsafe.Subtract(ref MemoryMarshal.GetReference(left), 1); + } + + Span filteredAbove = aboveStorage[prefixLength..]; + Span filteredLeft = leftStorage[prefixLength..]; + Av1IntraEdgePreparation.Prepare( + filteredAbove, + filteredLeft, + width, + height, + angle, + hasAbove ? width : 0, + hasLeft ? height : 0, + smoothIntraEdges, + 8, + scratch.Slice(predictionLength + (2 * edgeLength), Av1IntraEdgeFilter.ScratchLength), + out upsampleAbove, + out upsampleLeft); + + above = filteredAbove; + left = filteredLeft; + } Av1DirectionalIntraPredictor.Predict( prediction, @@ -728,9 +793,9 @@ internal static class Av1TransformBlockEncoder transformSize, above, left, - false, - false, - mode.ToAngle() + (angleDelta * Av1Constants.AngleStep), + upsampleAbove, + upsampleLeft, + angle, directionalScratch); } else @@ -764,6 +829,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The compact source-minus-prediction destination. /// The prediction dimensions. /// The coded sample bit depth. @@ -779,6 +846,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span residual, Av1TransformSize transformSize, Av1BitDepth bitDepth) @@ -806,7 +875,54 @@ internal static class Av1TransformBlockEncoder } else if (mode.IsDirectional()) { - Span directionalScratch = MemoryMarshal.Cast(workspace.TransformWorkspace)[..(width * height)]; + int angle = mode.ToAngle() + (angleDelta * Av1Constants.AngleStep); + Span scratch = MemoryMarshal.Cast(workspace.TransformWorkspace); + int predictionLength = width * height; + Span directionalScratch = scratch[..predictionLength]; + bool upsampleAbove = false; + bool upsampleLeft = false; + if (enableIntraEdgeFilter) + { + // Mode trials share raw references. Prepare private edge copies after the directional scratch; + // this entire transform workspace is reusable once prediction and residual formation finish. + int edgeLength = Av1IntraEdgePreparation.ReferenceBufferLength; + int prefixLength = Av1IntraEdgePreparation.ReferencePrefixLength; + Span aboveStorage = scratch.Slice(predictionLength, edgeLength); + Span leftStorage = scratch.Slice(predictionLength + edgeLength, edgeLength); + int midpoint = 128 << (bitDepth.GetBitCount() - 8); + aboveStorage.Fill((short)(midpoint - 1)); + leftStorage.Fill((short)(midpoint + 1)); + if (angle < 180) + { + signedAbove.CopyTo(aboveStorage[prefixLength..]); + aboveStorage[prefixLength - 1] = Unsafe.Subtract(ref MemoryMarshal.GetReference(signedAbove), 1); + } + + if (angle > 90) + { + signedLeft.CopyTo(leftStorage[prefixLength..]); + leftStorage[prefixLength - 1] = Unsafe.Subtract(ref MemoryMarshal.GetReference(signedLeft), 1); + } + + Span filteredAbove = aboveStorage[prefixLength..]; + Span filteredLeft = leftStorage[prefixLength..]; + Av1IntraEdgePreparation.Prepare( + filteredAbove, + filteredLeft, + width, + height, + angle, + hasAbove ? width : 0, + hasLeft ? height : 0, + smoothIntraEdges, + bitDepth.GetBitCount(), + scratch.Slice(predictionLength + (2 * edgeLength), Av1IntraEdgeFilter.ScratchLength), + out upsampleAbove, + out upsampleLeft); + + signedAbove = filteredAbove; + signedLeft = filteredLeft; + } Av1DirectionalIntraPredictor.Predict( signedPrediction, @@ -814,9 +930,9 @@ internal static class Av1TransformBlockEncoder transformSize, signedAbove, signedLeft, - false, - false, - mode.ToAngle() + (angleDelta * Av1Constants.AngleStep), + upsampleAbove, + upsampleLeft, + angle, directionalScratch); } else @@ -850,6 +966,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The retained entropy-coding coefficients. /// The selected transform dimensions. /// The selected compound transform type. @@ -870,6 +988,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -891,6 +1011,8 @@ internal static class Av1TransformBlockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, workspace.Residual, transformSize); @@ -935,6 +1057,8 @@ internal static class Av1TransformBlockEncoder /// Whether the top reference is available. /// The intra prediction mode. /// The signed directional-angle adjustment. + /// Whether sequence syntax enables directional edge filtering. + /// Whether a relevant neighboring block uses smooth prediction. /// The retained entropy-coding coefficients. /// The selected transform dimensions. /// The selected compound transform type. @@ -956,6 +1080,8 @@ internal static class Av1TransformBlockEncoder bool hasAbove, Av1PredictionMode mode, int angleDelta, + bool enableIntraEdgeFilter, + bool smoothIntraEdges, Span quantizedCoefficients, Av1TransformSize transformSize, Av1TransformType transformType, @@ -978,6 +1104,8 @@ internal static class Av1TransformBlockEncoder hasAbove, mode, angleDelta, + enableIntraEdgeFilter, + smoothIntraEdges, workspace.Residual, transformSize, bitDepth); diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operations.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operations.cs new file mode 100644 index 0000000000..f6c48c57cf --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operations.cs @@ -0,0 +1,210 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeFilter +{ + /// + /// Traverses one edge using the arithmetic of a closed smoothing operator. + /// + /// The filter-strength arithmetic. + private static class Filter + where TOperator : struct, IEdgeFilterOperator + { + /// + /// Filters all samples following the preserved first sample. + /// + /// The first edge sample. + /// The number of samples including the preserved sample. + /// The reusable source workspace. + public static void Apply(ref byte edge, int count, Span scratch) + { + // Each convolution reads the original edge. Duplicate its first sample once and its last sample + // twice so the five-tap windows implement endpoint clamping without per-lane boundary branches. + scratch[0] = edge; + MemoryMarshal.CreateReadOnlySpan(ref edge, count).CopyTo(scratch[1..]); + scratch.Slice(count + 1, 2).Fill(Unsafe.Add(ref edge, count - 1)); + + ref byte source = ref MemoryMarshal.GetReference(scratch); + int outputCount = count - 1; + int i = 0; + + // The same offset advances through descending SIMD widths. Adjacent lanes represent adjacent + // output samples, and only complete windows are loaded; the final incomplete window is scalar. + if (Vector512.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector512.Count; + for (; i <= vectorEnd; i += Vector512.Count) + { + Vector512 s0 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 0)), Vector256.Zero)); + + Vector512 s1 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 1)), Vector256.Zero)); + + Vector512 s2 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 2)), Vector256.Zero)); + + Vector512 s3 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 3)), Vector256.Zero)); + + Vector512 s4 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 4)), Vector256.Zero)); + + Vector512 result = TOperator.Apply(s0, s1, s2, s3, s4); + Vector512.Narrow(result, Vector512.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector256.Count; + for (; i <= vectorEnd; i += Vector256.Count) + { + Vector256 s0 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128.Zero)); + + Vector256 s1 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128.Zero)); + + Vector256 s2 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128.Zero)); + + Vector256 s3 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128.Zero)); + + Vector256 s4 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 4)), Vector128.Zero)); + + Vector256 result = TOperator.Apply(s0, s1, s2, s3, s4); + Vector256.Narrow(result, Vector256.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector128.Count; + for (; i <= vectorEnd; i += Vector128.Count) + { + Vector128 s0 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64.Zero)); + + Vector128 s1 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64.Zero)); + + Vector128 s2 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64.Zero)); + + Vector128 s3 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64.Zero)); + + Vector128 s4 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 4)), Vector64.Zero)); + + Vector128 result = TOperator.Apply(s0, s1, s2, s3, s4); + Vector128.Narrow(result, Vector128.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + for (; i < outputCount; i++) + { + int value = TOperator.Apply( + Unsafe.Add(ref source, i), + Unsafe.Add(ref source, i + 1), + Unsafe.Add(ref source, i + 2), + Unsafe.Add(ref source, i + 3), + Unsafe.Add(ref source, i + 4)); + + Unsafe.Add(ref edge, i + 1) = (byte)value; + } + } + + /// + /// Filters all samples following the preserved first sample. + /// + /// The first edge sample. + /// The number of samples including the preserved sample. + /// The reusable source workspace. + public static void Apply(ref short edge, int count, Span scratch) + { + // Each convolution reads the original edge. Duplicate its first sample once and its last sample + // twice so the five-tap windows implement endpoint clamping without per-lane boundary branches. + scratch[0] = edge; + MemoryMarshal.CreateReadOnlySpan(ref edge, count).CopyTo(scratch[1..]); + scratch.Slice(count + 1, 2).Fill(Unsafe.Add(ref edge, count - 1)); + + ref short source = ref MemoryMarshal.GetReference(scratch); + int outputCount = count - 1; + int i = 0; + + // The same offset advances through descending SIMD widths. Adjacent lanes represent adjacent + // output samples, and only complete windows are loaded; the final incomplete window is scalar. + // Nonnegative 12-bit samples have a maximum weighted sum of 65520. The rounding bias keeps + // that below 65536, so unsigned 16-bit lanes preserve the normative result at all strengths. + if (Vector512.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector512.Count; + for (; i <= vectorEnd; i += Vector512.Count) + { + Vector512 s0 = Vector512.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); + Vector512 s1 = Vector512.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); + Vector512 s2 = Vector512.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); + Vector512 s3 = Vector512.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); + Vector512 s4 = Vector512.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); + + Vector512 result = TOperator.Apply(s0, s1, s2, s3, s4); + result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector256.Count; + for (; i <= vectorEnd; i += Vector256.Count) + { + Vector256 s0 = Vector256.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); + Vector256 s1 = Vector256.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); + Vector256 s2 = Vector256.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); + Vector256 s3 = Vector256.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); + Vector256 s4 = Vector256.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); + + Vector256 result = TOperator.Apply(s0, s1, s2, s3, s4); + result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int vectorEnd = outputCount - Vector128.Count; + for (; i <= vectorEnd; i += Vector128.Count) + { + Vector128 s0 = Vector128.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); + Vector128 s1 = Vector128.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); + Vector128 s2 = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); + Vector128 s3 = Vector128.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); + Vector128 s4 = Vector128.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); + + Vector128 result = TOperator.Apply(s0, s1, s2, s3, s4); + result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); + } + } + + for (; i < outputCount; i++) + { + int value = TOperator.Apply( + Unsafe.Add(ref source, i), + Unsafe.Add(ref source, i + 1), + Unsafe.Add(ref source, i + 2), + Unsafe.Add(ref source, i + 3), + Unsafe.Add(ref source, i + 4)); + + Unsafe.Add(ref edge, i + 1) = (short)value; + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operator.cs new file mode 100644 index 0000000000..1c2c20dc2a --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Operator.cs @@ -0,0 +1,113 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Smooths AV1 intra-reference edges while preserving their common-corner sample. +/// +internal static partial class Av1IntraEdgeFilter +{ + /// + /// The sample count required for a maximal edge and its repeated endpoints. + /// + public const int ScratchLength = (2 * Av1Constants.MaxTransformSize) + 4; + + /// + /// Defines the rounded smoothing arithmetic for one AV1 filter strength. + /// + internal interface IEdgeFilterOperator + { + /// + /// Filters one sample using the five neighboring positions. + /// + /// The samples two positions before the output. + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The samples two positions after the output. + /// The rounded filtered samples. + public static abstract int Apply(int a, int b, int c, int d, int e); + + /// + /// Filters eight samples using the five neighboring positions. + /// + /// The samples two positions before the output. + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The samples two positions after the output. + /// The rounded filtered samples. + public static abstract Vector128 Apply(Vector128 a, Vector128 b, Vector128 c, Vector128 d, Vector128 e); + + /// + /// Filters sixteen samples using the five neighboring positions. + /// + /// The samples two positions before the output. + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The samples two positions after the output. + /// The rounded filtered samples. + public static abstract Vector256 Apply(Vector256 a, Vector256 b, Vector256 c, Vector256 d, Vector256 e); + + /// + /// Filters thirty-two samples using the five neighboring positions. + /// + /// The samples two positions before the output. + /// The preceding samples. + /// The centered samples. + /// The following samples. + /// The samples two positions after the output. + /// The rounded filtered samples. + public static abstract Vector512 Apply(Vector512 a, Vector512 b, Vector512 c, Vector512 d, Vector512 e); + } + + /// + /// Filters an edge in place, leaving its first sample unchanged. + /// + /// The first edge sample, including the common corner when present. + /// The number of edge samples. + /// The smoothing strength from zero through three. + /// The source workspace with at least samples. + public static void Apply(ref byte edge, int count, int strength, Span scratch) + { + switch (strength) + { + case 1: + Filter.Apply(ref edge, count, scratch); + break; + case 2: + Filter.Apply(ref edge, count, scratch); + break; + case 3: + Filter.Apply(ref edge, count, scratch); + break; + } + } + + /// + /// Filters an edge in place, leaving its first sample unchanged. + /// + /// The first edge sample, including the common corner when present. + /// The number of edge samples. + /// The smoothing strength from zero through three. + /// The source workspace with at least samples. + public static void Apply(ref short edge, int count, int strength, Span scratch) + { + switch (strength) + { + case 1: + Filter.Apply(ref edge, count, scratch); + break; + case 2: + Filter.Apply(ref edge, count, scratch); + break; + case 3: + Filter.Apply(ref edge, count, scratch); + break; + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength1Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength1Operator.cs new file mode 100644 index 0000000000..bdd76e67da --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength1Operator.cs @@ -0,0 +1,36 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeFilter +{ + /// + /// Applies the strength-1 three-tap edge smoothing kernel. + /// + internal readonly struct Strength1Operator : IEdgeFilterOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Apply(int a, int b, int c, int d, int e) + => (b + (c << 1) + d + 2) >> 2; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Apply(Vector128 a, Vector128 b, Vector128 c, Vector128 d, Vector128 e) + => (b + (c << 1) + d + Vector128.Create((ushort)2)) >> 2; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Apply(Vector256 a, Vector256 b, Vector256 c, Vector256 d, Vector256 e) + => (b + (c << 1) + d + Vector256.Create((ushort)2)) >> 2; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Apply(Vector512 a, Vector512 b, Vector512 c, Vector512 d, Vector512 e) + => (b + (c << 1) + d + Vector512.Create((ushort)2)) >> 2; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength2Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength2Operator.cs new file mode 100644 index 0000000000..c4f2f2f40e --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength2Operator.cs @@ -0,0 +1,36 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeFilter +{ + /// + /// Applies the strength-2 three-tap edge smoothing kernel. + /// + internal readonly struct Strength2Operator : IEdgeFilterOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Apply(int a, int b, int c, int d, int e) + => (((b + d) * 5) + (c * 6) + 8) >> 4; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Apply(Vector128 a, Vector128 b, Vector128 c, Vector128 d, Vector128 e) + => (((b + d) * Vector128.Create((ushort)5)) + (c * Vector128.Create((ushort)6)) + Vector128.Create((ushort)8)) >> 4; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Apply(Vector256 a, Vector256 b, Vector256 c, Vector256 d, Vector256 e) + => (((b + d) * Vector256.Create((ushort)5)) + (c * Vector256.Create((ushort)6)) + Vector256.Create((ushort)8)) >> 4; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Apply(Vector512 a, Vector512 b, Vector512 c, Vector512 d, Vector512 e) + => (((b + d) * Vector512.Create((ushort)5)) + (c * Vector512.Create((ushort)6)) + Vector512.Create((ushort)8)) >> 4; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength3Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength3Operator.cs new file mode 100644 index 0000000000..38aaac7658 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeFilter.Strength3Operator.cs @@ -0,0 +1,36 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeFilter +{ + /// + /// Applies the strength-3 five-tap edge smoothing kernel. + /// + internal readonly struct Strength3Operator : IEdgeFilterOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Apply(int a, int b, int c, int d, int e) + => (a + ((b + c + d) << 1) + e + 4) >> 3; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Apply(Vector128 a, Vector128 b, Vector128 c, Vector128 d, Vector128 e) + => (a + ((b + c + d) << 1) + e + Vector128.Create((ushort)4)) >> 3; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Apply(Vector256 a, Vector256 b, Vector256 c, Vector256 d, Vector256 e) + => (a + ((b + c + d) << 1) + e + Vector256.Create((ushort)4)) >> 3; + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Apply(Vector512 a, Vector512 b, Vector512 c, Vector512 d, Vector512 e) + => (a + ((b + c + d) << 1) + e + Vector512.Create((ushort)4)) >> 3; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgePreparation.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgePreparation.cs new file mode 100644 index 0000000000..1202f39bc5 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgePreparation.cs @@ -0,0 +1,289 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Prepares directional intra-reference edges for AV1 smoothing and half-sample prediction. +/// +internal static class Av1IntraEdgePreparation +{ + /// + /// The number of samples reserved before the first edge sample. + /// + public const int ReferencePrefixLength = 16; + + /// + /// The total sample capacity of one edge including prefix and extension. + /// + public const int ReferenceBufferLength = (2 * Av1Constants.MaxTransformSize) + 32; + + /// + /// Filters and upsamples prepared directional reference edges. + /// + /// The byte or signed high-bit-depth sample type. + /// The top edge with writable prefix and extension. + /// The left edge with writable prefix and extension. + /// The transform width. + /// The transform height. + /// The adjusted directional angle. + /// The number of available top samples before extension. + /// The number of available left samples before extension. + /// Whether a relevant neighbor uses smooth prediction. + /// The coded sample precision. + /// The original-edge workspace with at least samples. + /// Whether the top edge contains half-sample positions. + /// Whether the left edge contains half-sample positions. + public static void Prepare( + Span above, + Span left, + int width, + int height, + int angle, + int topCount, + int leftCount, + bool filterType, + int bitDepth, + Span scratch, + out bool upsampleAbove, + out bool upsampleLeft) + where T : unmanaged, IBinaryInteger + { + bool needAbove = angle < 180; + bool needLeft = angle > 90; + bool needRight = angle < 90; + bool needBottom = angle > 180; + upsampleAbove = false; + upsampleLeft = false; + + // A missing sole edge produces a constant block from the perpendicular sample or midpoint offset. + // Its prepared edge already repeats that value. Upsampling its distinct corner would change it. + if ((!needAbove && leftCount == 0) || (!needLeft && topCount == 0)) + { + return; + } + + if (angle is not 90 and not 180) + { + if (needAbove && needLeft && width + height >= 24) + { + // The corner is one logical sample represented in both edge prefixes. Filter it first, + // then let both edge convolutions read the same rounded [5, 6, 5] corner value. + ref T corner = ref Unsafe.Subtract(ref above[0], 1); + int value = (5 * int.CreateChecked(left[0])) + + (6 * int.CreateChecked(corner)) + + (5 * int.CreateChecked(above[0])); + + corner = T.CreateChecked((value + 8) >> 4); + Unsafe.Subtract(ref left[0], 1) = corner; + } + + if (needAbove && topCount > 0) + { + int strength = IntraEdgeFilterStrength(width, height, angle - 90, filterType); + Filter(ref Unsafe.Subtract(ref above[0], 1), topCount + 1 + (needRight ? height : 0), strength, scratch); + } + + if (needLeft && leftCount > 0) + { + int strength = IntraEdgeFilterStrength(height, width, angle - 180, filterType); + Filter(ref Unsafe.Subtract(ref left[0], 1), leftCount + 1 + (needBottom ? width : 0), strength, scratch); + } + } + + upsampleAbove = UseUpsampling(width, height, angle - 90, filterType); + if (needAbove && upsampleAbove) + { + Upsample(above, width + (needRight ? height : 0), bitDepth, scratch); + } + + upsampleLeft = UseUpsampling(height, width, angle - 180, filterType); + if (needLeft && upsampleLeft) + { + Upsample(left, height + (needBottom ? width : 0), bitDepth, scratch); + } + } + + /// + /// Selects half-sample interpolation for a transform edge. + /// + /// The transform width. + /// The transform height. + /// The angle relative to the edge's cardinal direction. + /// Whether a relevant neighbor uses smooth prediction. + /// Whether the edge uses half-sample interpolation. + private static bool UseUpsampling(int width, int height, int delta, bool filterType) + { + int distance = Math.Abs(delta); + return distance > 0 && distance < 40 && width + height <= (filterType ? 8 : 16); + } + + /// + /// Dispatches edge smoothing to the concrete sample representation. + /// + /// The byte or signed high-bit-depth sample type. + /// The first edge sample, including the corner. + /// The number of edge samples. + /// The smoothing strength. + /// The reusable original-edge workspace. + private static void Filter(ref T edge, int count, int strength, Span scratch) + where T : unmanaged, IBinaryInteger + { + if (typeof(T) == typeof(byte)) + { + Av1IntraEdgeFilter.Apply(ref Unsafe.As(ref edge), count, strength, MemoryMarshal.Cast(scratch)); + } + else + { + Av1IntraEdgeFilter.Apply(ref Unsafe.As(ref edge), count, strength, MemoryMarshal.Cast(scratch)); + } + } + + /// + /// Dispatches half-sample interpolation to the concrete sample representation. + /// + /// The byte or signed high-bit-depth sample type. + /// The edge with writable prefix and extension. + /// The number of original edge samples. + /// The coded precision. + /// The reusable original-edge workspace. + private static void Upsample(Span edge, int count, int bitDepth, Span scratch) + where T : unmanaged, IBinaryInteger + { + if (typeof(T) == typeof(byte)) + { + Av1IntraEdgeUpsampler.Apply(MemoryMarshal.Cast(edge), count, MemoryMarshal.Cast(scratch)); + } + else + { + Av1IntraEdgeUpsampler.Apply(MemoryMarshal.Cast(edge), count, bitDepth, MemoryMarshal.Cast(scratch)); + } + } + + /// + /// Selects the AV1 intra-edge filter strength for the block dimensions and prediction angle. + /// + /// The edge's primary block dimension. + /// The edge's secondary block dimension. + /// The prediction angle relative to the edge's cardinal direction. + /// A value indicating whether a neighboring smooth mode selects the alternate thresholds. + /// The filter strength from zero for no filtering through three for the strongest kernel. + private static int IntraEdgeFilterStrength(int width, int height, int delta, bool filterType) + { + int d = Math.Abs(delta); + int strength = 0; + int widthHeight = width + height; + if (!filterType) + { + if (widthHeight <= 8) + { + if (d >= 56) + { + strength = 1; + } + } + else if (widthHeight <= 12) + { + if (d >= 40) + { + strength = 1; + } + } + else if (widthHeight <= 16) + { + if (d >= 40) + { + strength = 1; + } + } + else if (widthHeight <= 24) + { + if (d >= 8) + { + strength = 1; + } + + if (d >= 16) + { + strength = 2; + } + + if (d >= 32) + { + strength = 3; + } + } + else if (widthHeight <= 32) + { + if (d >= 1) + { + strength = 1; + } + + if (d >= 4) + { + strength = 2; + } + + if (d >= 32) + { + strength = 3; + } + } + else + { + if (d >= 1) + { + strength = 3; + } + } + } + else + { + if (widthHeight <= 8) + { + if (d >= 40) + { + strength = 1; + } + + if (d >= 64) + { + strength = 2; + } + } + else if (widthHeight <= 16) + { + if (d >= 20) + { + strength = 1; + } + + if (d >= 48) + { + strength = 2; + } + } + else if (widthHeight <= 24) + { + if (d >= 4) + { + strength = 3; + } + } + else + { + if (d >= 1) + { + strength = 3; + } + } + } + + return strength; + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.FourTapOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.FourTapOperator.cs new file mode 100644 index 0000000000..b99523ce02 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.FourTapOperator.cs @@ -0,0 +1,48 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeUpsampler +{ + /// + /// Applies the AV1 [-1, 9, 9, -1] interpolation kernel with Q4 rounding and clipping. + /// + internal readonly struct FourTapOperator : IEdgeUpsamplingOperator + { + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Interpolate(int a, int b, int c, int d, int maximum) + => Math.Clamp((((9 * (b + c)) - a - d) + 8) >> 4, 0, maximum); + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Interpolate(Vector128 a, Vector128 b, Vector128 c, Vector128 d, int maximum) + { + // Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710. + Vector128 value = (((Vector128.Create(9) * (b + c)) - a - d) + Vector128.Create(8)) >> 4; + return Vector128.Clamp(value, Vector128.Zero, Vector128.Create(maximum)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Interpolate(Vector256 a, Vector256 b, Vector256 c, Vector256 d, int maximum) + { + // Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710. + Vector256 value = (((Vector256.Create(9) * (b + c)) - a - d) + Vector256.Create(8)) >> 4; + return Vector256.Clamp(value, Vector256.Zero, Vector256.Create(maximum)); + } + + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Interpolate(Vector512 a, Vector512 b, Vector512 c, Vector512 d, int maximum) + { + // Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710. + Vector512 value = (((Vector512.Create(9) * (b + c)) - a - d) + Vector512.Create(8)) >> 4; + return Vector512.Clamp(value, Vector512.Zero, Vector512.Create(maximum)); + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operations.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operations.cs new file mode 100644 index 0000000000..106905c053 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operations.cs @@ -0,0 +1,278 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using SixLabors.ImageSharp.Common.Helpers; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +internal static partial class Av1IntraEdgeUpsampler +{ + /// + /// Traverses a bounded edge using a closed interpolation operator. + /// + /// The four-tap interpolation arithmetic. + private static class Upsampler + where TOperator : struct, IEdgeUpsamplingOperator + { + /// + /// Inserts half samples using the original edge values and repeated endpoints. + /// + /// The edge with prefix and doubled output capacity. + /// The original sample count. + /// The reusable original-sample workspace. + public static void Apply(Span edge, int count, Span scratch) + { + ref byte destination = ref MemoryMarshal.GetReference(edge); + ref byte source = ref MemoryMarshal.GetReference(scratch); + + // Preserve the corner twice and the final sample once. Every SIMD load below covers exactly its + // input lanes, so the native 16+3-sample workspace also suffices for the widest interpolation. + source = Unsafe.Subtract(ref destination, 1); + Unsafe.Add(ref source, 1) = source; + edge[..count].CopyTo(scratch[2..]); + Unsafe.Add(ref source, count + 2) = edge[count - 1]; + Unsafe.Subtract(ref destination, 2) = source; + ref byte firstOutput = ref Unsafe.Subtract(ref destination, 1); + int i = 0; + + if (Vector512.IsHardwareAccelerated) + { + int vectorEnd = count - Vector512.Count; + for (; i <= vectorEnd; i += Vector512.Count) + { + Vector256 w0 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128.Zero)); + + Vector512 s0 = Vector512.WidenLower(Vector512.Create(w0, Vector256.Zero)).AsInt32(); + + Vector256 w1 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128.Zero)); + + Vector512 s1 = Vector512.WidenLower(Vector512.Create(w1, Vector256.Zero)).AsInt32(); + + Vector256 w2 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128.Zero)); + + Vector512 s2 = Vector512.WidenLower(Vector512.Create(w2, Vector256.Zero)).AsInt32(); + + Vector256 w3 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128.Zero)); + + Vector512 s3 = Vector512.WidenLower(Vector512.Create(w3, Vector256.Zero)).AsInt32(); + + Vector512 values = TOperator.Interpolate(s0, s1, s2, s3, 255); + Vector256 halfWords = Vector512.Narrow(values, Vector512.Zero).GetLower().AsUInt16(); + Vector128 halfSamples = Vector256.Narrow(halfWords, Vector256.Zero).GetLower(); + Vector128 originals = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)); + + // Unpack the lower and upper eight pairs independently to retain linear sample order. + Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + Vector128_.UnpackHigh(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 16)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int vectorEnd = count - Vector256.Count; + for (; i <= vectorEnd; i += Vector256.Count) + { + Vector128 w0 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64.Zero)); + + Vector256 s0 = Vector256.WidenLower(Vector256.Create(w0, Vector128.Zero)).AsInt32(); + + Vector128 w1 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64.Zero)); + + Vector256 s1 = Vector256.WidenLower(Vector256.Create(w1, Vector128.Zero)).AsInt32(); + + Vector128 w2 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64.Zero)); + + Vector256 s2 = Vector256.WidenLower(Vector256.Create(w2, Vector128.Zero)).AsInt32(); + + Vector128 w3 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64.Zero)); + + Vector256 s3 = Vector256.WidenLower(Vector256.Create(w3, Vector128.Zero)).AsInt32(); + + Vector256 values = TOperator.Interpolate(s0, s1, s2, s3, 255); + Vector128 halfWords = Vector256.Narrow(values, Vector256.Zero).GetLower().AsUInt16(); + Vector128 halfSamples = Vector128.Narrow(halfWords, Vector128.Zero); + Vector128 originals = Vector128.Create(Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64.Zero); + Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int vectorEnd = count - Vector128.Count; + for (; i <= vectorEnd; i += Vector128.Count) + { + Vector128 b0 = Vector128.CreateScalar(Unsafe.As(ref Unsafe.Add(ref source, i + 0))).AsByte(); + Vector128 w0 = Vector128.WidenLower(b0); + Vector128 s0 = Vector128.WidenLower(w0).AsInt32(); + + Vector128 b1 = Vector128.CreateScalar(Unsafe.As(ref Unsafe.Add(ref source, i + 1))).AsByte(); + Vector128 w1 = Vector128.WidenLower(b1); + Vector128 s1 = Vector128.WidenLower(w1).AsInt32(); + + Vector128 b2 = Vector128.CreateScalar(Unsafe.As(ref Unsafe.Add(ref source, i + 2))).AsByte(); + Vector128 w2 = Vector128.WidenLower(b2); + Vector128 s2 = Vector128.WidenLower(w2).AsInt32(); + + Vector128 b3 = Vector128.CreateScalar(Unsafe.As(ref Unsafe.Add(ref source, i + 3))).AsByte(); + Vector128 w3 = Vector128.WidenLower(b3); + Vector128 s3 = Vector128.WidenLower(w3).AsInt32(); + + Vector128 values = TOperator.Interpolate(s0, s1, s2, s3, 255); + Vector128 halfSamples = Vector128.Narrow( + Vector128.Narrow(values, Vector128.Zero).AsUInt16(), Vector128.Zero); + + Vector128 originals = Vector128.CreateScalar(Unsafe.As(ref Unsafe.Add(ref source, i + 2))).AsByte(); + Vector128_.UnpackLow(halfSamples, originals).GetLower().StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + } + } + + for (; i < count; i++) + { + int value = TOperator.Interpolate( + Unsafe.Add(ref source, i), + Unsafe.Add(ref source, i + 1), + Unsafe.Add(ref source, i + 2), + Unsafe.Add(ref source, i + 3), + 255); + + Unsafe.Add(ref destination, (2 * i) - 1) = (byte)value; + Unsafe.Add(ref destination, 2 * i) = Unsafe.Add(ref source, i + 2); + } + } + + /// + /// Inserts half samples using the original edge values and repeated endpoints. + /// + /// The edge with prefix and doubled output capacity. + /// The original sample count. + /// The maximum coded sample value. + /// The reusable original-sample workspace. + public static void Apply(Span edge, int count, int maximum, Span scratch) + { + ref short destination = ref MemoryMarshal.GetReference(edge); + ref short source = ref MemoryMarshal.GetReference(scratch); + + // Preserve the corner twice and the final sample once. Every SIMD load below covers exactly its + // input lanes, so the native 16+3-sample workspace also suffices for the widest interpolation. + source = Unsafe.Subtract(ref destination, 1); + Unsafe.Add(ref source, 1) = source; + edge[..count].CopyTo(scratch[2..]); + Unsafe.Add(ref source, count + 2) = edge[count - 1]; + Unsafe.Subtract(ref destination, 2) = source; + ref short firstOutput = ref Unsafe.Subtract(ref destination, 1); + int i = 0; + + if (Vector512.IsHardwareAccelerated) + { + int vectorEnd = count - Vector512.Count; + for (; i <= vectorEnd; i += Vector512.Count) + { + Vector512 s0 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 0)), Vector256.Zero)); + + Vector512 s1 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 1)), Vector256.Zero)); + + Vector512 s2 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 2)), Vector256.Zero)); + + Vector512 s3 = Vector512.WidenLower(Vector512.Create( + Vector256.LoadUnsafe(ref source, (nuint)(i + 3)), Vector256.Zero)); + + Vector512 values = TOperator.Interpolate(s0, s1, s2, s3, maximum); + Vector256 halfSamples = Vector512.Narrow(values, Vector512.Zero).GetLower(); + Vector256 originals = Vector256.LoadUnsafe(ref source, (nuint)(i + 2)); + + // Four contiguous groups of four pairs avoid treating lane-local unpack order as one + // linear 256-bit edge. Each store writes only prepared half samples and their originals. + Vector128_.UnpackLow(halfSamples.GetLower(), originals.GetLower()) + .StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + + Vector128_.UnpackHigh(halfSamples.GetLower(), originals.GetLower()) + .StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 8)); + + Vector128_.UnpackLow(halfSamples.GetUpper(), originals.GetUpper()) + .StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 16)); + + Vector128_.UnpackHigh(halfSamples.GetUpper(), originals.GetUpper()) + .StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 24)); + } + } + + if (Vector256.IsHardwareAccelerated) + { + int vectorEnd = count - Vector256.Count; + for (; i <= vectorEnd; i += Vector256.Count) + { + Vector256 s0 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128.Zero)); + + Vector256 s1 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128.Zero)); + + Vector256 s2 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128.Zero)); + + Vector256 s3 = Vector256.WidenLower(Vector256.Create( + Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128.Zero)); + + Vector256 values = TOperator.Interpolate(s0, s1, s2, s3, maximum); + Vector128 halfSamples = Vector256.Narrow(values, Vector256.Zero).GetLower(); + Vector128 originals = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)); + Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + Vector128_.UnpackHigh(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 8)); + } + } + + if (Vector128.IsHardwareAccelerated) + { + int vectorEnd = count - Vector128.Count; + for (; i <= vectorEnd; i += Vector128.Count) + { + Vector128 s0 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64.Zero)); + + Vector128 s1 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64.Zero)); + + Vector128 s2 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64.Zero)); + + Vector128 s3 = Vector128.WidenLower(Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64.Zero)); + + Vector128 values = TOperator.Interpolate(s0, s1, s2, s3, maximum); + Vector128 halfSamples = Vector128.Narrow(values, Vector128.Zero); + Vector128 originals = Vector128.Create( + Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64.Zero); + + Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); + } + } + + for (; i < count; i++) + { + int value = TOperator.Interpolate( + Unsafe.Add(ref source, i), + Unsafe.Add(ref source, i + 1), + Unsafe.Add(ref source, i + 2), + Unsafe.Add(ref source, i + 3), + maximum); + + Unsafe.Add(ref destination, (2 * i) - 1) = (short)value; + Unsafe.Add(ref destination, 2 * i) = Unsafe.Add(ref source, i + 2); + } + } + } +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operator.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operator.cs new file mode 100644 index 0000000000..a8ed1fb5b2 --- /dev/null +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraEdgeUpsampler.Operator.cs @@ -0,0 +1,91 @@ +// Copyright (c) Six Labors. +// Licensed under the Six Labors Split License. + +using System.Runtime.Intrinsics; + +namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; + +/// +/// Inserts clipped half-sample positions into AV1 intra-reference edges. +/// +internal static partial class Av1IntraEdgeUpsampler +{ + /// + /// The maximum number of original samples permitted in an upsampled edge. + /// + public const int MaximumCount = 16; + + /// + /// The sample count required for the original edge, corner, and repeated endpoints. + /// + public const int ScratchLength = MaximumCount + 3; + + /// + /// Defines signed four-tap interpolation before sample narrowing. + /// + internal interface IEdgeUpsamplingOperator + { + /// + /// Interpolates half samples and clamps them to the coded range. + /// + /// The preceding samples. + /// The first central samples. + /// The second central samples. + /// The following samples. + /// The maximum coded sample. + /// The rounded and clipped half samples. + public static abstract int Interpolate(int a, int b, int c, int d, int maximum); + + /// + /// Interpolates half samples and clamps them to the coded range. + /// + /// The preceding samples. + /// The first central samples. + /// The second central samples. + /// The following samples. + /// The maximum coded sample. + /// The rounded and clipped half samples. + public static abstract Vector128 Interpolate(Vector128 a, Vector128 b, Vector128 c, Vector128 d, int maximum); + + /// + /// Interpolates half samples and clamps them to the coded range. + /// + /// The preceding samples. + /// The first central samples. + /// The second central samples. + /// The following samples. + /// The maximum coded sample. + /// The rounded and clipped half samples. + public static abstract Vector256 Interpolate(Vector256 a, Vector256 b, Vector256 c, Vector256 d, int maximum); + + /// + /// Interpolates half samples and clamps them to the coded range. + /// + /// The preceding samples. + /// The first central samples. + /// The second central samples. + /// The following samples. + /// The maximum coded sample. + /// The rounded and clipped half samples. + public static abstract Vector512 Interpolate(Vector512 a, Vector512 b, Vector512 c, Vector512 d, int maximum); + } + + /// + /// Inserts half samples before each original edge sample. + /// + /// The edge with writable prefix samples at -2 and -1 and room for the doubled extent. + /// The number of original edge samples, at most . + /// The original-sample workspace with at least samples. + public static void Apply(Span edge, int count, Span scratch) + => Upsampler.Apply(edge, count, scratch); + + /// + /// Inserts half samples before each original edge sample. + /// + /// The edge with writable prefix samples at -2 and -1 and room for the doubled extent. + /// The number of original edge samples, at most . + /// The coded precision used to clamp interpolation. + /// The original-sample workspace with at least samples. + public static void Apply(Span edge, int count, int bitDepth, Span scratch) + => Upsampler.Apply(edge, count, (1 << bitDepth) - 1, scratch); +} diff --git a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs index a4319d945b..92928ab500 100644 --- a/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs +++ b/src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs @@ -4,8 +4,6 @@ using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using SixLabors.ImageSharp.Common.Helpers; using SixLabors.ImageSharp.Formats.Heif.Av1.OpenBitstreamUnit; using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling; @@ -25,20 +23,15 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; /// internal sealed class Av1PredictionDecoder { - /// - /// The largest edge length for which AV1 permits intra-edge upsampling. - /// - private const int MaxUpsampleSize = 16; - /// /// The number of samples reserved for one prepared AV1 intra-prediction edge. /// - private const int ReferenceBufferLength = (Av1Constants.MaxTransformSize * 2) + 32; + private const int ReferenceBufferLength = Av1IntraEdgePreparation.ReferenceBufferLength; /// /// The padded sample count required by the widest intra-edge SIMD loads. /// - private const int EdgeScratchLength = 160; + private const int EdgeScratchLength = Av1IntraEdgeFilter.ScratchLength; /// /// The number of high-bit-depth samples required by the reusable prediction workspace. @@ -716,8 +709,8 @@ internal sealed class Av1PredictionDecoder // and writes upsampled edge samples as far back as -2. aboveData.Fill(T.CreateChecked(baseValue - 1)); leftData.Fill(T.CreateChecked(baseValue + 1)); - Span aboveRow = aboveData[16..]; - Span leftColumn = leftData[16..]; + Span aboveRow = aboveData[Av1IntraEdgePreparation.ReferencePrefixLength..]; + Span leftColumn = leftData[Av1IntraEdgePreparation.ReferencePrefixLength..]; int transformWidth = transformSize.GetWidth(); int transformHeight = transformSize.GetHeight(); bool isDirectionalMode = mode.IsDirectional(); @@ -918,49 +911,19 @@ internal sealed class Av1PredictionDecoder bool upsampleLeft = false; if (!disableEdgeFilter) { - bool needRight = angle < 90; - bool needBottom = angle > 180; - - bool filterType = GetFilterType(ref partitionInfo, plane); - - if (angle is not 90 and not 180) - { - int ab_le = needAboveLeft ? 1 : 0; - if (needAbove && needLeft && (transformWidth + transformHeight >= 24)) - { - FilterIntraEdgeCorner(aboveRow, leftColumn); - } - - if (needAbove && topPixelCount > 0) - { - int strength = IntraEdgeFilterStrength(transformWidth, transformHeight, angle - 90, filterType); - int pixelCount = topPixelCount + ab_le + (needRight ? transformHeight : 0); - FilterIntraEdge(ref Unsafe.Subtract(ref aboveRow[0], ab_le), pixelCount, strength, edgeScratch); - } - - if (needLeft && leftPixelCount > 0) - { - int strength = IntraEdgeFilterStrength(transformHeight, transformWidth, angle - 180, filterType); - int pixelCount = leftPixelCount + ab_le + (needBottom ? transformWidth : 0); - FilterIntraEdge(ref Unsafe.Subtract(ref leftColumn[0], ab_le), pixelCount, strength, edgeScratch); - } - } - - upsampleAbove = UseIntraEdgeUpsample(transformWidth, transformHeight, angle - 90, filterType); - if (needAbove && upsampleAbove) - { - int pixelCount = transformWidth + (needRight ? transformHeight : 0); - - UpsampleIntraEdge(aboveRow, pixelCount, bitDepth, edgeScratch); - } - - upsampleLeft = UseIntraEdgeUpsample(transformHeight, transformWidth, angle - 180, filterType); - if (needLeft && upsampleLeft) - { - int pixelCount = transformHeight + (needBottom ? transformWidth : 0); - - UpsampleIntraEdge(leftColumn, pixelCount, bitDepth, edgeScratch); - } + Av1IntraEdgePreparation.Prepare( + aboveRow, + leftColumn, + transformWidth, + transformHeight, + angle, + topPixelCount, + leftPixelCount, + GetFilterType(ref partitionInfo, plane), + bitDepth, + edgeScratch, + out upsampleAbove, + out upsampleLeft); } this.DirectionalPredictor(destination, destinationStride, transformSize, aboveRow, leftColumn, upsampleAbove, upsampleLeft, angle); @@ -1151,640 +1114,6 @@ internal sealed class Av1PredictionDecoder } } - /// - /// Inserts half-sample positions into a prepared intra-prediction edge. - /// - /// The 8-bit or high-bit-depth sample type. - /// The edge buffer, including writable prefix storage at indices -2 and -1. - /// The number of original edge samples to upsample. - /// The number of bits used to clamp interpolated samples. - /// The reusable padded source workspace. - private static void UpsampleIntraEdge(Span buffer, int count, int bitDepth, Span scratch) - where T : unmanaged, IBinaryInteger - { - DebugGuard.MustBeLessThanOrEqualTo(count, MaxUpsampleSize, nameof(count)); - - // DecodeBuildIntraPredictors is closed only over byte and short. Keeping that dispatch outside the - // kernels gives the JIT concrete vector element types and removes generic arithmetic from their loops. - if (typeof(T) == typeof(byte)) - { - UpsampleIntraEdge(MemoryMarshal.Cast(buffer), count, MemoryMarshal.Cast(scratch)); - } - else - { - UpsampleIntraEdge(MemoryMarshal.Cast(buffer), count, bitDepth, MemoryMarshal.Cast(scratch)); - } - } - - /// - /// Inserts half-sample positions into an 8-bit intra-prediction edge. - /// - /// The edge buffer, including writable prefix storage at indices -2 and -1. - /// The number of original edge samples to upsample. - /// The reusable padded source workspace. - public static void UpsampleIntraEdge(Span buffer, int count, Span scratch) - { - ref byte bufferBase = ref MemoryMarshal.GetReference(buffer); - ref byte inputBase = ref MemoryMarshal.GetReference(scratch); - byte beforeBuffer = Unsafe.Subtract(ref bufferBase, 1); - byte finalSample = Unsafe.Add(ref bufferBase, count - 1); - - // Vector loads intentionally extend past the logical edge. Initializing the complete load window with - // the final sample provides the AV1 endpoint extension and keeps every unaligned read inside scratch. - scratch[..32].Fill(finalSample); - inputBase = beforeBuffer; - Unsafe.Add(ref inputBase, 1) = beforeBuffer; - buffer[..count].CopyTo(scratch[2..]); - Unsafe.Subtract(ref bufferBase, 2) = beforeBuffer; - - int i = 0; - if (Vector128.IsHardwareAccelerated) - { - int eightSamplesFromEnd = count - 8; - for (; i <= eightSamplesFromEnd; i += 8) - { - Vector128 interpolated = InterpolateEightBytes(ref inputBase, i); - Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); - Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); - interleaved.StoreUnsafe(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)); - } - - // AV1 upsampled edges are normally multiples of four. A half-vector store handles that common - // remainder without overwriting the prepared extension beyond the logical output edge. - if (i <= count - 4) - { - Vector128 interpolated = InterpolateEightBytes(ref inputBase, i); - Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); - Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); - Unsafe.As(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)) = interleaved.AsUInt64().ToScalar(); - i += 4; - } - } - - UpsampleIntraEdgeScalar(ref bufferBase, ref inputBase, i, count, 255); - } - - /// - /// Inserts half-sample positions into a high-bit-depth intra-prediction edge. - /// - /// The edge buffer, including writable prefix storage at indices -2 and -1. - /// The number of original edge samples to upsample. - /// The number of bits used to clamp interpolated samples. - /// The reusable padded source workspace. - public static void UpsampleIntraEdge(Span buffer, int count, int bitDepth, Span scratch) - { - ref short bufferBase = ref MemoryMarshal.GetReference(buffer); - ref short inputBase = ref MemoryMarshal.GetReference(scratch); - short beforeBuffer = Unsafe.Subtract(ref bufferBase, 1); - short finalSample = Unsafe.Add(ref bufferBase, count - 1); - - // The same padded layout is used for 10- and 12-bit edges. Arithmetic widens to Int32 before applying - // the four-tap kernel because the 12-bit intermediate exceeds the unsigned 16-bit range. - scratch[..32].Fill(finalSample); - inputBase = beforeBuffer; - Unsafe.Add(ref inputBase, 1) = beforeBuffer; - buffer[..count].CopyTo(scratch[2..]); - Unsafe.Subtract(ref bufferBase, 2) = beforeBuffer; - - int maximum = (1 << bitDepth) - 1; - int i = 0; - if (Vector128.IsHardwareAccelerated) - { - int eightSamplesFromEnd = count - 8; - for (; i <= eightSamplesFromEnd; i += 8) - { - Vector128 interpolated = InterpolateEightHighBitDepthSamples(ref inputBase, i, maximum); - Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); - Vector128 interleavedLow = Vector128_.UnpackLow(interpolated, originals); - Vector128 interleavedHigh = Vector128_.UnpackHigh(interpolated, originals); - ref short destination = ref Unsafe.Add(ref bufferBase, (2 * i) - 1); - - interleavedLow.StoreUnsafe(ref destination); - interleavedHigh.StoreUnsafe(ref destination, (nuint)Vector128.Count); - } - - if (i <= count - 4) - { - Vector128 interpolated = InterpolateEightHighBitDepthSamples(ref inputBase, i, maximum); - Vector128 originals = Vector128.LoadUnsafe(ref inputBase, (nuint)(i + 2)); - Vector128 interleaved = Vector128_.UnpackLow(interpolated, originals); - interleaved.StoreUnsafe(ref Unsafe.Add(ref bufferBase, (2 * i) - 1)); - i += 4; - } - } - - UpsampleIntraEdgeScalar(ref bufferBase, ref inputBase, i, count, maximum); - } - - /// - /// Calculates eight 8-bit half-sample values in parallel. - /// - /// The first padded input sample. - /// The first output sample index. - /// The interpolated samples in the lower eight lanes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 InterpolateEightBytes(ref byte input, int offset) - { - Vector128 source0 = Vector128.LoadUnsafe(ref input, (nuint)offset); - Vector128 source1 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 1)); - Vector128 source2 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 2)); - Vector128 source3 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 3)); - (Vector128 source0Low, _) = Vector128.Widen(source0); - (Vector128 source1Low, _) = Vector128.Widen(source1); - (Vector128 source2Low, _) = Vector128.Widen(source2); - (Vector128 source3Low, _) = Vector128.Widen(source3); - Vector128 interpolation = (((source1Low + source2Low) * Vector128.Create((ushort)9)) - (source0Low + source3Low)).AsInt16(); - - interpolation = Vector128.Clamp((interpolation + Vector128.Create((short)8)) >> 4, Vector128.Zero, Vector128.Create((short)255)); - return Vector128.Narrow(interpolation.AsUInt16(), Vector128.Zero); - } - - /// - /// Calculates eight high-bit-depth half-sample values in parallel. - /// - /// The first padded input sample. - /// The first output sample index. - /// The maximum reconstructed sample value. - /// The interpolated samples. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 InterpolateEightHighBitDepthSamples(ref short input, int offset, int maximum) - { - Vector128 source0 = Vector128.LoadUnsafe(ref input, (nuint)offset); - Vector128 source1 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 1)); - Vector128 source2 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 2)); - Vector128 source3 = Vector128.LoadUnsafe(ref input, (nuint)(offset + 3)); - (Vector128 source0Low, Vector128 source0High) = Vector128.Widen(source0); - (Vector128 source1Low, Vector128 source1High) = Vector128.Widen(source1); - (Vector128 source2Low, Vector128 source2High) = Vector128.Widen(source2); - (Vector128 source3Low, Vector128 source3High) = Vector128.Widen(source3); - Vector128 coefficient = Vector128.Create(9); - Vector128 rounding = Vector128.Create(8); - Vector128 maximumVector = Vector128.Create(maximum); - Vector128 low = ((((source1Low + source2Low) * coefficient) - (source0Low + source3Low)) + rounding) >> 4; - Vector128 high = ((((source1High + source2High) * coefficient) - (source0High + source3High)) + rounding) >> 4; - - low = Vector128.Clamp(low, Vector128.Zero, maximumVector); - high = Vector128.Clamp(high, Vector128.Zero, maximumVector); - return Vector128.Narrow(low, high); - } - - /// - /// Inserts the scalar remainder of an intra-edge upsample operation. - /// - /// The byte or 16-bit sample type. - /// The first original edge sample. - /// The first padded input sample. - /// The first sample not processed by SIMD. - /// The number of original edge samples. - /// The maximum reconstructed sample value. - private static void UpsampleIntraEdgeScalar(ref T buffer, ref T input, int start, int count, int maximum) - where T : unmanaged, IBinaryInteger - { - for (int i = start; i < count; i++) - { - int interpolation = -int.CreateChecked(Unsafe.Add(ref input, i)) - + (9 * int.CreateChecked(Unsafe.Add(ref input, i + 1))) - + (9 * int.CreateChecked(Unsafe.Add(ref input, i + 2))) - - int.CreateChecked(Unsafe.Add(ref input, i + 3)); - - interpolation = Av1Math.Clamp((interpolation + 8) >> 4, 0, maximum); - - Unsafe.Add(ref buffer, (2 * i) - 1) = T.CreateChecked(interpolation); - Unsafe.Add(ref buffer, 2 * i) = Unsafe.Add(ref input, i + 2); - } - } - - /// - /// Determines whether AV1 intra-edge upsampling applies to a directional prediction edge. - /// - /// The edge's primary block dimension. - /// The edge's secondary block dimension. - /// The prediction angle relative to the edge's cardinal direction. - /// A value indicating whether a neighboring smooth mode selects the alternate thresholds. - /// when the edge must be upsampled; otherwise, . - private static bool UseIntraEdgeUpsample(int width, int height, int delta, bool filterType) - { - int d = Math.Abs(delta); - if (d is <= 0 or >= 40) - { - return false; - } - - int widthHeight = width + height; - return filterType ? (widthHeight <= 8) : (widthHeight <= 16); - } - - /// - /// Applies the AV1 intra-edge smoothing kernel at the requested strength. - /// - /// The 8-bit or high-bit-depth sample type. - /// A reference to the first edge sample to filter. - /// The number of edge samples. - /// The AV1 filter-strength index from zero through three. - /// The reusable padded source workspace. - private static void FilterIntraEdge(ref T buffer, int count, int strength, Span scratch) - where T : unmanaged, IBinaryInteger - { - if (strength == 0) - { - return; - } - - // As with edge upsampling, closing the kernel over a concrete sample type keeps vector arithmetic - // outside the generic decoder while the valid strength-zero no-op remains at the owning boundary. - if (typeof(T) == typeof(byte)) - { - FilterIntraEdge(ref Unsafe.As(ref buffer), count, strength, MemoryMarshal.Cast(scratch)); - } - else - { - FilterIntraEdge(ref Unsafe.As(ref buffer), count, strength, MemoryMarshal.Cast(scratch)); - } - } - - /// - /// Applies an AV1 intra-edge smoothing kernel to 8-bit samples. - /// - /// A reference to the first edge sample to filter. - /// The number of edge samples. - /// The AV1 filter-strength index from one through three. - /// The reusable padded source workspace. - public static void FilterIntraEdge(ref byte buffer, int count, int strength, Span scratch) - { - byte finalSample = Unsafe.Add(ref buffer, count - 1); - - // The original edge is retained because each convolution window must observe unfiltered neighbors. - // Padding both endpoints also makes every complete vector use the same contiguous load pattern. - scratch[..EdgeScratchLength].Fill(finalSample); - scratch[0] = buffer; - MemoryMarshal.CreateReadOnlySpan(ref buffer, count).CopyTo(scratch[1..]); - - ref byte edge = ref MemoryMarshal.GetReference(scratch); - int outputCount = count - 1; - int processed = 0; - if (Vector128.IsHardwareAccelerated) - { - int eightSamplesFromEnd = outputCount - 8; - switch (strength) - { - case 1: - for (; processed <= eightSamplesFromEnd; processed += 8) - { - Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); - Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); - Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); - Vector128 result = Vector128.Narrow(FilterEdgeStrength1(source0, source1, source2), Vector128.Zero); - Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().ToScalar(); - } - - break; - case 2: - for (; processed <= eightSamplesFromEnd; processed += 8) - { - Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); - Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); - Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); - Vector128 result = Vector128.Narrow(FilterEdgeStrength2(source0, source1, source2), Vector128.Zero); - Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().ToScalar(); - } - - break; - default: - for (; processed <= eightSamplesFromEnd; processed += 8) - { - Vector128 source0 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)processed)); - Vector128 source1 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1))); - Vector128 source2 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2))); - Vector128 source3 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3))); - Vector128 source4 = WidenLower(Vector128.LoadUnsafe(ref edge, (nuint)(processed + 4))); - Vector128 result = Vector128.Narrow(FilterEdgeStrength3(source0, source1, source2, source3, source4), Vector128.Zero); - Unsafe.As(ref Unsafe.Add(ref buffer, processed + 1)) = result.AsUInt64().ToScalar(); - } - - break; - } - } - - FilterIntraEdgeScalar(ref buffer, ref edge, processed, outputCount, strength); - } - - /// - /// Applies an AV1 intra-edge smoothing kernel to high-bit-depth samples. - /// - /// A reference to the first edge sample to filter. - /// The number of edge samples. - /// The AV1 filter-strength index from one through three. - /// The reusable padded source workspace. - public static void FilterIntraEdge(ref short buffer, int count, int strength, Span scratch) - { - short finalSample = Unsafe.Add(ref buffer, count - 1); - - scratch[..EdgeScratchLength].Fill(finalSample); - scratch[0] = buffer; - MemoryMarshal.CreateReadOnlySpan(ref buffer, count).CopyTo(scratch[1..]); - - ref short edge = ref MemoryMarshal.GetReference(scratch); - int outputCount = count - 1; - int processed = 0; - - // The largest 12-bit weighted sum is 65520; the greatest rounding bias raises that only to 65528. - // Unsigned 16-bit lanes therefore preserve every normative strength without widening to 32-bit vectors. - if (Vector128.IsHardwareAccelerated) - { - int vectorEnd = (int)(Numerics.Vector128Count(outputCount) * (nuint)Vector128.Count); - switch (strength) - { - case 1: - for (; processed < vectorEnd; processed += Vector128.Count) - { - Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); - Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); - Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); - FilterEdgeStrength1(source0, source1, source2).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); - } - - break; - case 2: - for (; processed < vectorEnd; processed += Vector128.Count) - { - Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); - Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); - Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); - FilterEdgeStrength2(source0, source1, source2).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); - } - - break; - default: - for (; processed < vectorEnd; processed += Vector128.Count) - { - Vector128 source0 = Vector128.LoadUnsafe(ref edge, (nuint)processed).AsUInt16(); - Vector128 source1 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 1)).AsUInt16(); - Vector128 source2 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 2)).AsUInt16(); - Vector128 source3 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 3)).AsUInt16(); - Vector128 source4 = Vector128.LoadUnsafe(ref edge, (nuint)(processed + 4)).AsUInt16(); - FilterEdgeStrength3(source0, source1, source2, source3, source4).AsInt16().StoreUnsafe(ref buffer, (nuint)(processed + 1)); - } - - break; - } - } - - FilterIntraEdgeScalar(ref buffer, ref edge, processed, outputCount, strength); - } - - /// - /// Widens the lower eight lanes of a byte vector for edge-filter arithmetic. - /// - /// The packed source samples. - /// The widened samples. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 WidenLower(Vector128 source) - { - (Vector128 lower, _) = Vector128.Widen(source); - return lower; - } - - /// - /// Applies the strength-one three-tap edge filter to eight samples. - /// - /// The preceding samples. - /// The centered samples. - /// The following samples. - /// The filtered samples. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 FilterEdgeStrength1(Vector128 source0, Vector128 source1, Vector128 source2) - => (source0 + (source1 << 1) + source2 + Vector128.Create((ushort)2)) >> 2; - - /// - /// Applies the strength-two three-tap edge filter to eight samples. - /// - /// The preceding samples. - /// The centered samples. - /// The following samples. - /// The filtered samples. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 FilterEdgeStrength2(Vector128 source0, Vector128 source1, Vector128 source2) - => (((source0 + source2) * Vector128.Create((ushort)5)) + (source1 * Vector128.Create((ushort)6)) + Vector128.Create((ushort)8)) >> 4; - - /// - /// Applies the strength-three five-tap edge filter to eight samples. - /// - /// The samples two positions before each output. - /// The preceding samples. - /// The centered samples. - /// The following samples. - /// The samples two positions after each output. - /// The filtered samples. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 FilterEdgeStrength3( - Vector128 source0, - Vector128 source1, - Vector128 source2, - Vector128 source3, - Vector128 source4) - => (source0 + ((source1 + source2 + source3) << 1) + source4 + Vector128.Create((ushort)4)) >> 3; - - /// - /// Applies an AV1 edge filter to samples not consumed by the vector loop. - /// - /// The byte or 16-bit sample type. - /// The first destination sample. - /// The first padded source sample. - /// The first output index not processed by SIMD. - /// The number of filtered outputs following the preserved first sample. - /// The AV1 filter-strength index. - private static void FilterIntraEdgeScalar(ref T buffer, ref T edge, int start, int count, int strength) - where T : unmanaged, IBinaryInteger - { - switch (strength) - { - case 1: - for (int i = start; i < count; i++) - { - int sourceOffset = i + 1; - int value = int.CreateChecked(Unsafe.Add(ref edge, sourceOffset)) - + (2 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 1))) - + int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 2)); - - Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 2) >> 2); - } - - break; - case 2: - for (int i = start; i < count; i++) - { - int sourceOffset = i + 1; - int value = (5 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset))) - + (6 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 1))) - + (5 * int.CreateChecked(Unsafe.Add(ref edge, sourceOffset + 2))); - - Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 8) >> 4); - } - - break; - default: - for (int i = start; i < count; i++) - { - int value = int.CreateChecked(Unsafe.Add(ref edge, i)) - + (2 * (int.CreateChecked(Unsafe.Add(ref edge, i + 1)) - + int.CreateChecked(Unsafe.Add(ref edge, i + 2)) - + int.CreateChecked(Unsafe.Add(ref edge, i + 3)))) - + int.CreateChecked(Unsafe.Add(ref edge, i + 4)); - - Unsafe.Add(ref buffer, i + 1) = T.CreateChecked((value + 4) >> 3); - } - - break; - } - } - - /// - /// Selects the AV1 intra-edge filter strength for the block dimensions and prediction angle. - /// - /// The edge's primary block dimension. - /// The edge's secondary block dimension. - /// The prediction angle relative to the edge's cardinal direction. - /// A value indicating whether a neighboring smooth mode selects the alternate thresholds. - /// The filter strength from zero for no filtering through three for the strongest kernel. - private static int IntraEdgeFilterStrength(int width, int height, int delta, bool filterType) - { - int d = Math.Abs(delta); - int strength = 0; - int widthHeight = width + height; - if (!filterType) - { - if (widthHeight <= 8) - { - if (d >= 56) - { - strength = 1; - } - } - else if (widthHeight <= 12) - { - if (d >= 40) - { - strength = 1; - } - } - else if (widthHeight <= 16) - { - if (d >= 40) - { - strength = 1; - } - } - else if (widthHeight <= 24) - { - if (d >= 8) - { - strength = 1; - } - - if (d >= 16) - { - strength = 2; - } - - if (d >= 32) - { - strength = 3; - } - } - else if (widthHeight <= 32) - { - if (d >= 1) - { - strength = 1; - } - - if (d >= 4) - { - strength = 2; - } - - if (d >= 32) - { - strength = 3; - } - } - else - { - if (d >= 1) - { - strength = 3; - } - } - } - else - { - if (widthHeight <= 8) - { - if (d >= 40) - { - strength = 1; - } - - if (d >= 64) - { - strength = 2; - } - } - else if (widthHeight <= 16) - { - if (d >= 20) - { - strength = 1; - } - - if (d >= 48) - { - strength = 2; - } - } - else if (widthHeight <= 24) - { - if (d >= 4) - { - strength = 3; - } - } - else - { - if (d >= 1) - { - strength = 3; - } - } - } - - return strength; - } - - /// - /// Smooths the shared top-left reference sample where the prepared top and left edges meet. - /// - /// The 8-bit or high-bit-depth sample type. - /// The prepared top edge with writable top-left prefix storage. - /// The prepared left edge with writable top-left prefix storage. - private static void FilterIntraEdgeCorner(Span above, Span left) - where T : unmanaged, IBinaryInteger - { - int[] kernel = [5, 6, 5]; - - ref T aboveRef = ref above[0]; - ref T leftRef = ref left[0]; - ref T abovePreviousRef = ref Unsafe.Subtract(ref aboveRef, 1); - ref T leftPreviousRef = ref Unsafe.Subtract(ref leftRef, 1); - int s = (int.CreateChecked(leftRef) * kernel[0]) + (int.CreateChecked(abovePreviousRef) * kernel[1]) + (int.CreateChecked(aboveRef) * kernel[2]); - s = (s + 8) >> 4; - - // Both edge spans reserve their own prefix location for the same logical corner, - // so keep the two scratch representations synchronized after filtering. - abovePreviousRef = T.CreateChecked(s); - leftPreviousRef = T.CreateChecked(s); - } - /// /// Determines the directional edge-filter threshold class from neighboring prediction modes. /// diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs index 1716de243e..6cec574eaf 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1EncoderFrameTests.cs @@ -409,6 +409,7 @@ public class Av1EncoderFrameTests sample.Position = 0; sample.CopyTo(output); decoder.DecodeSequenceReference(sample.ToArray(), null, null); + Assert.True(Assert.IsType(decoder.SequenceHeader).EnableIntraEdgeFilter); Av1FrameBuffer decoded = Assert.IsType>(decoder.FrameBuffer); Assert.Equal(Width, decoded.Width); Assert.Equal(Height, decoded.Height); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs index 6dda775e2b..36f982c1bb 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1IntraSuperblockEncoderTests.cs @@ -2918,11 +2918,15 @@ public class Av1IntraSuperblockEncoderTests /// Verifies that mixed partition trials and final writing retain the decoder's reconstruction order. /// [Theory] - [InlineData(false, 32)] - [InlineData(true, 32)] - [InlineData(false, 56)] - [InlineData(true, 56)] - public void ProductionMixedPartitionsPreserveReconstructionOrder(bool transpose, int size) + [InlineData(false, 32, false)] + [InlineData(true, 32, false)] + [InlineData(false, 56, false)] + [InlineData(true, 56, false)] + [InlineData(false, 32, true)] + [InlineData(true, 32, true)] + [InlineData(false, 56, true)] + [InlineData(true, 56, true)] + public void ProductionMixedPartitionsPreserveReconstructionOrder(bool transpose, int size, bool enableIntraEdgeFilter) { const int QIndex = 4; ObuColorConfig colorConfig = new() @@ -2956,6 +2960,7 @@ public class Av1IntraSuperblockEncoderTests ClearPlane(reconstruction.Luma); using Av1EncoderModeInfoBuffer modeInfo = new(Configuration.Default, size, size, disallow4x4AllFrames: false); Av1PictureControlSet template = CreatePicture(modeInfo, colorConfig, use128x128Superblock: false, QIndex); + template.Sequence.SequenceHeader.EnableIntraEdgeFilter = enableIntraEdgeFilter; using Av1EncoderPictureBuffer picture = new( Configuration.Default, template.Sequence.SequenceHeader, template.Parent.FrameHeader, size, size, disallow4x4AllFrames: false); @@ -2969,6 +2974,7 @@ public class Av1IntraSuperblockEncoderTests byte[] payload = WriteCompleteTileObu(picture.Picture, tileWriter, size, size); using Av1Decoder decoder = new(Configuration.Default); decoder.DecodeSequenceReference(payload, null, null); + Assert.Equal(enableIntraEdgeFilter, Assert.IsType(decoder.SequenceHeader).EnableIntraEdgeFilter); Av1FrameInfo decodedInfo = Assert.IsType(decoder.FrameInfo); Av1FrameBuffer decodedFrame = Assert.IsType>(decoder.FrameBuffer); Buffer2DRegion decodedPlane = decodedFrame.DeriveBlockPointer(Av1Plane.Y, 0, 0); @@ -2996,8 +3002,8 @@ public class Av1IntraSuperblockEncoderTests TestEnvironment.ActualOutputDirectoryFullPath, "Heif", "Av1", nameof(this.ProductionMixedPartitionsPreserveReconstructionOrder)); Directory.CreateDirectory(directory); - File.WriteAllBytes(Path.Combine(directory, $"{size}-{transpose}.obu"), payload); - using FileStream raw = File.Create(Path.Combine(directory, $"{size}-{transpose}.retained.yuv")); + File.WriteAllBytes(Path.Combine(directory, $"{size}-{transpose}-{enableIntraEdgeFilter}.obu"), payload); + using FileStream raw = File.Create(Path.Combine(directory, $"{size}-{transpose}-{enableIntraEdgeFilter}.retained.yuv")); for (int y = 0; y < size; y++) { raw.Write(retainedPlane.DangerousGetRowSpan(y)); diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs index eb6e368b09..222fdde671 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1PredictorTests.cs @@ -108,18 +108,109 @@ public class Av1PredictorTests => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateFilterIntraPredictors, PredictorConfigurations); /// - /// Verifies intra-edge upsampling with Vector128 and the scalar fallback. + /// Verifies intra-edge upsampling with each register-width tier and the scalar fallback. /// [Fact] public void EdgeUpsamplingMatchesReference() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeUpsampling, HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic); + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeUpsampling, PredictorConfigurations); /// - /// Verifies intra-edge filtering with Vector128 and the scalar fallback. + /// Verifies intra-edge filtering with each register-width tier and the scalar fallback. /// [Fact] public void EdgeFilteringMatchesReference() - => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeFiltering, HwIntrinsics.AllowAll | HwIntrinsics.DisableHWIntrinsic); + => FeatureTestRunner.RunWithHwIntrinsicsFeature(ValidateEdgeFiltering, PredictorConfigurations); + + /// + /// Verifies the different edge preparation selected by a smooth neighbor on a 4x8 directional block. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void EdgePreparationUsesSmoothNeighborThresholds(bool transpose, bool smoothNeighbor) + { + const int Count = 12; + byte[] edge = CreateUpsampleByteEdge(Count); + byte[] expected = (byte[])edge.Clone(); + byte[] unusedEdge = CreateUpsampleByteEdge(Count); + byte[] expectedUnused = (byte[])unusedEdge.Clone(); + byte[] scratch = new byte[Av1IntraEdgeFilter.ScratchLength]; + + // At 23 degrees from the cardinal direction and width + height = 12, an ordinary neighbor + // selects unfiltered half samples. A smooth neighbor selects strength one without upsampling. + // The expected samples use the independent scalar kernels, never the production selector. + if (smoothNeighbor) + { + byte[] source = edge.AsSpan(1, Count + 1).ToArray(); + byte[] filtered = (byte[])source.Clone(); + FilterEdgeScalar(source, filtered, 1); + filtered.CopyTo(expected, 1); + } + else + { + UpsampleEdgeScalar(expected, Count, 8); + } + + Av1IntraEdgePreparation.Prepare( + (transpose ? unusedEdge : edge).AsSpan(2), + (transpose ? edge : unusedEdge).AsSpan(2), + transpose ? 8 : 4, + transpose ? 4 : 8, + transpose ? 203 : 67, + transpose ? 8 : 4, + transpose ? 4 : 8, + smoothNeighbor, + 8, + scratch, + out bool upsampleAbove, + out bool upsampleLeft); + + Assert.Equal(!transpose && !smoothNeighbor, upsampleAbove); + Assert.Equal(transpose && !smoothNeighbor, upsampleLeft); + Assert.Equal(expected, edge); + Assert.Equal(expectedUnused, unusedEdge); + } + + /// + /// Verifies that an unavailable sole directional edge retains its constant prediction and distinct corner. + /// + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void EdgePreparationPreservesUnavailableSoleEdge(bool transpose, bool perpendicularAvailable) + { + byte[] edge = CreateUpsampleByteEdge(12); + byte[] perpendicular = CreateUpsampleByteEdge(12); + edge.AsSpan(2, 12).Fill(perpendicularAvailable ? perpendicular[2] : transpose ? (byte)129 : (byte)127); + byte[] expected = (byte[])edge.Clone(); + byte[] expectedPerpendicular = (byte[])perpendicular.Clone(); + byte[] scratch = new byte[Av1IntraEdgeFilter.ScratchLength]; + + // These angles normally enable half-sample interpolation. Native prediction exits before that + // stage when its sole edge is unavailable; the corner must not introduce a nonconstant sample. + Av1IntraEdgePreparation.Prepare( + (transpose ? perpendicular : edge).AsSpan(2), + (transpose ? edge : perpendicular).AsSpan(2), + transpose ? 8 : 4, + transpose ? 4 : 8, + transpose ? 203 : 67, + transpose && perpendicularAvailable ? 8 : 0, + !transpose && perpendicularAvailable ? 8 : 0, + false, + 8, + scratch, + out bool upsampleAbove, + out bool upsampleLeft); + + Assert.False(upsampleAbove); + Assert.False(upsampleLeft); + Assert.Equal(expected, edge); + Assert.Equal(expectedPerpendicular, perpendicular); + } /// /// Verifies the traversal-order bits that distinguish current libaom's mixed-vertical square tables. @@ -608,15 +699,15 @@ public class Av1PredictorTests /// private static void ValidateEdgeUpsampling() { - ReadOnlySpan counts = [4, 8, 12, 16]; + ReadOnlySpan counts = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; foreach (int count in counts) { byte[] actual = CreateUpsampleByteEdge(count); byte[] expected = (byte[])actual.Clone(); - byte[] scratch = new byte[160]; + byte[] scratch = new byte[Av1IntraEdgeUpsampler.ScratchLength]; UpsampleEdgeScalar(expected, count, 8); - Av1PredictionDecoder.UpsampleIntraEdge(actual.AsSpan(2), count, scratch); + Av1IntraEdgeUpsampler.Apply(actual.AsSpan(2), count, scratch); Assert.Equal(expected, actual); @@ -626,10 +717,10 @@ public class Av1PredictorTests { short[] actualHigh = CreateUpsampleHighBitDepthEdge(count, bitDepth); short[] expectedHigh = (short[])actualHigh.Clone(); - short[] scratchHigh = new short[160]; + short[] scratchHigh = new short[Av1IntraEdgeUpsampler.ScratchLength]; UpsampleEdgeScalar(expectedHigh, count, bitDepth); - Av1PredictionDecoder.UpsampleIntraEdge(actualHigh.AsSpan(2), count, bitDepth, scratchHigh); + Av1IntraEdgeUpsampler.Apply(actualHigh.AsSpan(2), count, bitDepth, scratchHigh); Assert.Equal(expectedHigh, actualHigh); } @@ -641,7 +732,7 @@ public class Av1PredictorTests /// private static void ValidateEdgeFiltering() { - ReadOnlySpan counts = [4, 8, 9, 16, 31, 64, 129]; + ReadOnlySpan counts = [4, 8, 9, 16, 17, 31, 32, 33, 64, 65, 129]; foreach (int count in counts) { for (int strength = 1; strength <= 3; strength++) @@ -649,20 +740,20 @@ public class Av1PredictorTests byte[] actual = CreateByteSamples(count, 31); byte[] expected = (byte[])actual.Clone(); byte[] source = (byte[])actual.Clone(); - byte[] scratch = new byte[160]; + byte[] scratch = new byte[Av1IntraEdgeFilter.ScratchLength]; FilterEdgeScalar(source, expected, strength); - Av1PredictionDecoder.FilterIntraEdge(ref actual[0], count, strength, scratch); + Av1IntraEdgeFilter.Apply(ref actual[0], count, strength, scratch); Assert.Equal(expected, actual); short[] actualHigh = CreateHighBitDepthSamples(count, 31); short[] expectedHigh = (short[])actualHigh.Clone(); short[] sourceHigh = (short[])actualHigh.Clone(); - short[] scratchHigh = new short[160]; + short[] scratchHigh = new short[Av1IntraEdgeFilter.ScratchLength]; FilterEdgeScalar(sourceHigh, expectedHigh, strength); - Av1PredictionDecoder.FilterIntraEdge(ref actualHigh[0], count, strength, scratchHigh); + Av1IntraEdgeFilter.Apply(ref actualHigh[0], count, strength, scratchHigh); Assert.Equal(expectedHigh, actualHigh); } diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs index 7b08d44def..5d5beede63 100644 --- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1TransformBlockEncoderTests.cs @@ -334,6 +334,8 @@ public class Av1TransformBlockEncoderTests hasAbove: true, Av1PredictionMode.Vertical, 0, + enableIntraEdgeFilter: false, + smoothIntraEdges: false, quantized, Av1TransformSize.Size8x8, Av1TransformType.DctDct, @@ -410,6 +412,8 @@ public class Av1TransformBlockEncoderTests hasAbove: true, Av1PredictionMode.Directional135Degrees, angleDelta, + enableIntraEdgeFilter: false, + smoothIntraEdges: false, quantized, Av1TransformSize.Size8x8, Av1TransformType.DctDct,