Browse Source

Document HEIF SIMD architecture

pull/2633/head
James Jackson-South 2 months ago
parent
commit
f8ab68f588
  1. 6
      src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs
  2. 23
      src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs
  3. 5
      src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainOverlap.cs
  4. 12
      src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs
  5. 6
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs
  6. 21
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs
  7. 8
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs
  8. 6
      src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1WienerFilter.cs
  9. 14
      src/ImageSharp/Formats/Heif/Av1/Pipeline/SuperResolution/Av1SuperResolutionFilter.cs
  10. 9
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcIntraPredictor.cs
  11. 14
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Operations.cs
  12. 8
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Transpose.cs
  13. 18
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs
  14. 3
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.HorizontalOperator.cs
  15. 14
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs
  16. 3
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs
  17. 5
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Packing.cs
  18. 3
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.PaethOperator.cs
  19. 8
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothHorizontalOperator.cs
  20. 8
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothOperator.cs
  21. 7
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothVerticalOperator.cs
  22. 3
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.VerticalOperator.cs
  23. 6
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PalettePredictor.cs
  24. 5
      src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs
  25. 4
      src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.Operations.cs
  26. 6
      src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaPredictor.cs
  27. 6
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs
  28. 5
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs
  29. 18
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs
  30. 9
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs
  31. 5
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs
  32. 5
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs
  33. 6
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs
  34. 10
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs
  35. 5
      src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs
  36. 16
      src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs
  37. 5
      src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs
  38. 7
      src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs
  39. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs
  40. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs
  41. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs
  42. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs
  43. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs
  44. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs
  45. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs
  46. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs
  47. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs
  48. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs
  49. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs
  50. 4
      src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs
  51. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.CoefficientOperator.cs
  52. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.ConstantLuminanceOperator.cs
  53. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.ICtCpOperator.cs
  54. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.IdentityOperator.cs
  55. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.IptC2Operator.cs
  56. 8
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Operator.cs
  57. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Smpte2085Operator.cs
  58. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.YCgCoOperator.cs
  59. 3
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.YCgCoReversibleOperator.cs
  60. 5
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs
  61. 4
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.CoefficientOperator.cs
  62. 4
      src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Operator.cs
  63. 5
      src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs
  64. 6
      src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs
  65. 10
      src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs
  66. 6
      src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs
  67. 11
      src/ImageSharp/Formats/Heif/Hevc/HevcInverseQuantizer.cs
  68. 15
      src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs
  69. 19
      src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs
  70. 10
      src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetFilter.cs

6
src/ImageSharp/Formats/Heif/Av1/Pipeline/Cdef/Av1CdefFilter.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.Cdef;
/// <summary>
/// Applies AV1 constrained directional enhancement filtering and derives the dominant direction of reconstructed blocks.
/// </summary>
/// <remarks>
/// Filtering uses signed 16-bit working samples. The 256-bit kernel packs either two complete eight-sample rows or
/// four complete four-sample rows, keeping directional offsets within 128-bit lanes. Direction analysis instead packs
/// one eight-by-eight block per 128-bit lane so AVX2 can evaluate two independent blocks together. Scalar kernels retain
/// the same constrain, clipping, and tie-breaking rules for unsupported hardware and partial edge blocks.
/// </remarks>
internal static class Av1CdefFilter
{
/// <summary>

23
src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainNoise.cs

@ -12,6 +12,13 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain;
/// <summary>
/// Applies selected AV1 grain blocks to restored luma and chroma samples.
/// </summary>
/// <remarks>
/// SIMD lanes follow consecutive samples within one plane row. Grain values and native samples are widened to signed
/// 32-bit lanes before the scaling-table lookup and fixed-point addition, then clipped and narrowed only once. AVX2
/// uses indexed gathers for the 256-entry scaling table. Portable 128-bit traversal is retained where high-bit-depth
/// interpolation provides enough arithmetic to offset its scalar table reads; all remaining columns use the identical
/// scalar equation.
/// </remarks>
internal static class Av1FilmGrainNoise
{
/// <summary>
@ -186,6 +193,8 @@ internal static class Av1FilmGrainNoise
int maximum)
where TSample : unmanaged
{
// Scaling is an indexed lookup, so AVX2 is selected by gather support rather than generic preferred vector
// width. The portable path is deliberately limited by CanVectorizeWithoutGather for the same reason.
if (Avx2.IsSupported)
{
ApplyLuma(
@ -256,6 +265,9 @@ internal static class Av1FilmGrainNoise
int grainRowOffset = row * grainStride;
int column = 0;
int vectorEnd = width - Vector256<int>.Count;
// Samples, grain, and scale indices share the same lane coordinate. No permutation is required between
// the lookup, fixed-point multiply, clipping, and native-sample store.
for (; column <= vectorEnd; column += Vector256<int>.Count)
{
ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column);
@ -308,6 +320,9 @@ internal static class Av1FilmGrainNoise
int grainRowOffset = row * grainStride;
int column = 0;
int vectorEnd = width - Vector128<int>.Count;
// Four scalar table reads assemble the scale vector; the rest of the normative grain equation remains
// lane-wise, including interpolation for 10- and 12-bit coordinates.
for (; column <= vectorEnd; column += Vector128<int>.Count)
{
ref TSample destination = ref Unsafe.Add(ref sampleBase, sampleRowOffset + column);
@ -402,6 +417,8 @@ internal static class Av1FilmGrainNoise
int maximum)
where TSample : unmanaged
{
// Chroma uses the same indexed scaling-table constraint as luma, so gather support determines the primary
// width and the portable path remains restricted to workloads that amortize scalar table reads.
if (Avx2.IsSupported)
{
ApplyChroma(
@ -554,6 +571,9 @@ internal static class Av1FilmGrainNoise
int grainRowOffset = row * grainStride;
int column = 0;
int vectorEnd = width - Vector256<int>.Count;
// Each lane represents one chroma coordinate and its corresponding reconstructed-luma coordinate. The Q6
// luma/chroma blend is clamped to a legal sample code before it becomes a scaling-table index.
for (; column <= vectorEnd; column += Vector256<int>.Count)
{
ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX);
@ -682,6 +702,9 @@ internal static class Av1FilmGrainNoise
int grainRowOffset = row * grainStride;
int column = 0;
int vectorEnd = width - Vector128<int>.Count;
// The four-lane path preserves the same coordinate alignment and Q6 scaling-index arithmetic. Only the
// table read changes from a hardware gather to four scalar reads assembled into a vector.
for (; column <= vectorEnd; column += Vector128<int>.Count)
{
ref TSample lumaSource = ref Unsafe.Add(ref lumaRow, column << subsamplingX);

5
src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainOverlap.cs

@ -11,6 +11,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain;
/// <summary>
/// Blends grain samples across adjacent synthesis blocks.
/// </summary>
/// <remarks>
/// Vertical boundaries contain only one or two strided columns and use the fixed scalar kernels. Horizontal boundaries
/// are contiguous and progress from the runtime's preferred native width through smaller vector widths before the
/// scalar tail. Every lane applies the same Q5 overlap weights, rounding offset, and signed grain clamp.
/// </remarks>
internal static class Av1FilmGrainOverlap
{
/// <summary>

12
src/ImageSharp/Formats/Heif/Av1/Pipeline/FilmGrain/Av1FilmGrainSampleOperator.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.FilmGrain;
/// Loads and stores native AV1 samples for the film-grain arithmetic pipeline.
/// </summary>
/// <typeparam name="TSample">The native sample type.</typeparam>
/// <remarks>
/// The decoder closes this operator over <see cref="byte"/> for eight-bit planes and <see cref="ushort"/> for
/// high-bit-depth planes. The JIT removes the unused type branch, so widening and narrowing remain branch-free in the
/// row loops. All arithmetic uses signed 32-bit lanes; decoded samples are nonnegative and at most twelve bits, making
/// the intermediate signed 16-bit views safe wherever pairwise operations require them.
/// </remarks>
internal readonly struct Av1FilmGrainSampleOperator<TSample>
where TSample : unmanaged
{
@ -25,6 +31,7 @@ internal readonly struct Av1FilmGrainSampleOperator<TSample>
{
if (typeof(TSample) == typeof(byte))
{
// Reading exactly eight bytes avoids touching the following row when the visible width has no padding.
ref byte sourceBytes = ref Unsafe.As<TSample, byte>(ref source);
ulong packed = Unsafe.ReadUnaligned<ulong>(ref sourceBytes);
return Avx2.ConvertToVector256Int32(Vector128.CreateScalarUnsafe(packed).AsByte());
@ -44,6 +51,7 @@ internal readonly struct Av1FilmGrainSampleOperator<TSample>
{
if (typeof(TSample) == typeof(byte))
{
// The source occupies only the low four byte lanes; two lower-half widens preserve their original order.
ref byte sourceBytes = ref Unsafe.As<TSample, byte>(ref source);
uint packedBytes = Unsafe.ReadUnaligned<uint>(ref sourceBytes);
Vector128<ushort> widened = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packedBytes).AsByte());
@ -126,6 +134,8 @@ internal readonly struct Av1FilmGrainSampleOperator<TSample>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Store8(ref TSample destination, Vector256<int> values)
{
// AddNoise has already clipped every lane to the native sample range. Unsigned narrowing is therefore exact
// and serves only to repack lane width; it does not supply saturation or alter out-of-range values.
Vector128<ushort> packed = Vector128.Narrow(values.GetLower().AsUInt32(), values.GetUpper().AsUInt32());
if (typeof(TSample) == typeof(byte))
{
@ -146,6 +156,8 @@ internal readonly struct Av1FilmGrainSampleOperator<TSample>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static void Store4(ref TSample destination, Vector128<int> values)
{
// Only four destination samples are valid. Narrow through the low half, then write four bytes or four UInt16
// values so the store cannot overwrite the next row or an adjacent plane allocation.
Vector64<ushort> packed = Vector128.Narrow(values.AsUInt32(), Vector128<uint>.Zero).GetLower();
if (typeof(TSample) == typeof(byte))
{

6
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopFilter/Av1DeblockingFilter.cs

@ -10,6 +10,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopFilter;
/// <summary>
/// Applies the AV1 deblocking kernels to four-sample edge segments.
/// </summary>
/// <remarks>
/// Each 32-bit vector lane represents one sample position along the edge. The orientation-specific operators gather or
/// load the corresponding p6..p0,q0..q6 neighborhood, after which masks, flatness tests, and filter equations remain
/// lane-wise. Conditional selection preserves unfiltered lanes while allowing four adjacent edge positions to share
/// one kernel invocation.
/// </remarks>
internal static class Av1DeblockingFilter
{
/// <summary>

21
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.Operations.cs

@ -8,6 +8,12 @@ using System.Runtime.Intrinsics.X86;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration;
/// <content>
/// Provides the 128- and 256-bit self-guided restoration pipelines. Consecutive lanes represent neighboring output
/// columns throughout integral-image construction, coefficient generation, filtering, and projection. Each vector
/// loop passes its final horizontal prefix to the scalar tail, preserving one continuous summed-area row without
/// recomputing already processed samples.
/// </content>
internal static partial class Av1SelfGuidedFilter
{
/// <summary>
@ -37,6 +43,9 @@ internal static partial class Av1SelfGuidedFilter
Span<int> scratch,
Vector256<int> vector)
{
// The caller-owned span contains two visible filtered planes followed by four identically strided coefficient
// planes. Keeping these regions disjoint allows projection to read either radius after the integral buffers
// have been reused as immutable inputs, without per-unit allocation or copying.
int filteredLength = width * height;
int bufferLength = GetBufferLength(width, height);
int bufferStride = GetBufferStride(width);
@ -134,6 +143,8 @@ internal static partial class Av1SelfGuidedFilter
Span<int> scratch,
Vector128<int> vector)
{
// Use the same scratch partition as the 256-bit path. Vector width changes only the number of adjacent columns
// advanced by each stage; all offsets and fixed-point representations remain identical.
int filteredLength = width * height;
int bufferLength = GetBufferLength(width, height);
int bufferStride = GetBufferStride(width);
@ -671,6 +682,8 @@ internal static partial class Av1SelfGuidedFilter
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static unsafe Vector256<int> LookupBlendFactors(Vector256<uint> indices)
{
// Variance normalization bounds every index to the 256-entry table. AVX2 gather keeps the eight independent
// column lookups in the vector pipeline instead of materializing an intermediate scalar scale buffer.
fixed (int* table = XByXPlusOne)
{
return Avx2.GatherVector256(table, indices.AsInt32(), sizeof(int));
@ -685,6 +698,8 @@ internal static partial class Av1SelfGuidedFilter
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<int> LookupBlendFactors(Vector128<uint> indices)
{
// Portable 128-bit APIs do not provide indexed loads. Four bounded scalar reads are assembled directly into
// the result vector, avoiding both an allocation and a second pass over the coefficient row.
ReadOnlySpan<int> table = XByXPlusOne;
return Vector128.Create(
table[(int)indices.GetElement(0)],
@ -787,6 +802,9 @@ internal static partial class Av1SelfGuidedFilter
Vector256<int> rounding = Vector256.Create(1 << (roundingBits - 1));
int column = 0;
int vectorEnd = width - Vector256<int>.Count;
// Filtered signals use Q4 precision. Projection applies the signaled Q7 weights to their difference from
// the unfiltered Q4 sample, then performs the combined Q11 rounding shift once before clipping.
for (; column <= vectorEnd; column += Vector256<int>.Count)
{
Vector256<int> factors = CrossSum(blendFactors, coefficientRowOffset + column, bufferStride, row, vector);
@ -843,6 +861,9 @@ internal static partial class Av1SelfGuidedFilter
Vector128<int> rounding = Vector128.Create(1 << (roundingBits - 1));
int column = 0;
int vectorEnd = width - Vector128<int>.Count;
// The 128-bit path uses the same Q4/Q7 projection equation. The four-sample load and store are deliberately
// 64 bits wide so a tightly strided destination row never requires writable padding.
for (; column <= vectorEnd; column += Vector128<int>.Count)
{
Vector128<int> factors = CrossSum(blendFactors, coefficientRowOffset + column, bufferStride, row, vector);

8
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1SelfGuidedFilter.cs

@ -10,6 +10,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration;
/// <summary>
/// Applies the normative AV1 self-guided restoration filter and projection.
/// </summary>
/// <remarks>
/// The accelerated paths process adjacent output columns in SIMD lanes and use the caller-provided scratch span for
/// filtered samples, local coefficients, and padded integral images. AVX2 is selected where variance-to-blend lookup
/// can use a native gather; the portable 128-bit path performs four scalar table reads while retaining vectorized
/// window and projection arithmetic. The scalar path uses the same fixed-point units and scratch partition.
/// </remarks>
internal static partial class Av1SelfGuidedFilter
{
/// <summary>
@ -151,6 +157,8 @@ internal static partial class Av1SelfGuidedFilter
ReadOnlySpan<int> projectionCoefficients,
Span<int> scratch)
{
// The closed vector overloads share the same scratch layout and fixed-point equations. AVX2 is preferred here
// because its coefficient stage can gather four noncontiguous entries from the 256-value blend table.
if (Avx2.IsSupported)
{
FilterBlock(

6
src/ImageSharp/Formats/Heif/Av1/Pipeline/LoopRestoration/Av1WienerFilter.cs

@ -11,6 +11,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.LoopRestoration;
/// <summary>
/// Applies the normative separable Wiener filter used by AV1 loop restoration.
/// </summary>
/// <remarks>
/// The horizontal pass consumes one contiguous eight-tap window at a time. SIMD pairwise multiply-add widens adjacent
/// unsigned-sample and signed-coefficient products into four 32-bit partial sums, which are reduced before normative
/// rounding and clipping. The resulting caller-owned intermediate plane supplies contiguous columns to the vertical
/// pass without allocating per stripe.
/// </remarks>
internal static class Av1WienerFilter
{
/// <summary>

14
src/ImageSharp/Formats/Heif/Av1/Pipeline/SuperResolution/Av1SuperResolutionFilter.cs

@ -13,6 +13,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Pipeline.SuperResolution;
/// <summary>
/// Applies the normative horizontal filter used by AV1 super-resolution upscaling.
/// </summary>
/// <remarks>
/// Four output coordinates are evaluated together, but each coordinate starts as its own eight-lane vector of source
/// taps. Pairwise multiply-add produces four partial sums per output, and two horizontal reductions transpose those
/// four independent dot products into consecutive output lanes. This layout supports arbitrary fixed-point source
/// steps and filter phases without gathers or a temporary coefficient matrix.
/// </remarks>
internal static class Av1SuperResolutionFilter
{
/// <summary>
@ -245,6 +251,9 @@ internal static class Av1SuperResolutionFilter
GetOffsets(sourcePosition + (step * 3), out int sourceOffset3, out int filterOffset3);
ref short filter = ref MemoryMarshal.GetReference(Filters);
// Replicated edge storage guarantees all eight taps are contiguous even for the first and last output. Exact
// 64-bit reads avoid depending on additional row padding before widening each tap set to signed Int16 lanes.
Vector128<short> samples0 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref source, sourceOffset0))).AsByte()).AsInt16();
Vector128<short> samples1 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref source, sourceOffset1))).AsByte()).AsInt16();
Vector128<short> samples2 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref source, sourceOffset2))).AsByte()).AsInt16();
@ -276,6 +285,9 @@ internal static class Av1SuperResolutionFilter
GetOffsets(sourcePosition + (step * 3), out int sourceOffset3, out int filterOffset3);
ref short filter = ref MemoryMarshal.GetReference(Filters);
// AV1 high-bit-depth samples are at most twelve bits, so their signed Int16 view remains positive. Keeping the
// tap and coefficient types equal enables the same pairwise multiply-add reduction as the eight-bit path.
return FilterFour(
Vector128.LoadUnsafe(ref source, (nuint)sourceOffset0).AsInt16(),
Vector128.LoadUnsafe(ref source, (nuint)sourceOffset1).AsInt16(),
@ -348,6 +360,8 @@ internal static class Av1SuperResolutionFilter
return Vector128.Create(leftPairs, rightPairs);
}
// The portable fallback selects even and odd lanes separately. ShuffleNative is valid here because every index
// is in range; adding the two permutations forms [x0+x1, x2+x3] for each source vector.
Vector128<int> evenIndices = Vector128.Create(0, 2, 0, 2);
Vector128<int> oddIndices = Vector128.Create(1, 3, 1, 3);
Vector128<int> leftPairsFallback = Vector128.ShuffleNative(left, evenIndices) + Vector128.ShuffleNative(left, oddIndices);

9
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DcIntraPredictor.cs

@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <summary>
/// Reconstructs AV1 DC intra-prediction blocks from the available neighboring samples.
/// </summary>
/// <remarks>
/// Reference reduction uses the widest available integer lanes, then one rounded scalar mean is broadcast across each
/// destination row. Row filling is delegated to span operations so the runtime supplies its native vectorized store;
/// explicit scalar entry points remain available for feature-disabled conformance tests.
/// </remarks>
internal static class Av1DcIntraPredictor
{
/// <summary>
@ -124,6 +129,8 @@ internal static class Av1DcIntraPredictor
int sum = 0;
int index = 0;
// Widening prevents the packed-byte reduction from overflowing before the horizontal sum. The shared index
// advances through every available vector width and leaves only the incomplete final group to the scalar loop.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = samples.Length - Vector512<byte>.Count;
@ -173,6 +180,8 @@ internal static class Av1DcIntraPredictor
int sum = 0;
int index = 0;
// Signed 16-bit storage is nonnegative for supported bit depths. Widening to Int32 preserves the exact sum of
// the largest permitted reference edge before the rounded mean is calculated once outside this loop.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = samples.Length - Vector512<short>.Count;

14
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Operations.cs

@ -8,7 +8,11 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides packed projection kernels for AV1 directional intra prediction.
/// Provides packed projection kernels for AV1 directional intra prediction. Contiguous zone-one references map one
/// output sample to each lane. Upsampled references use native byte or 16-bit table shuffles to select alternating
/// half-sample positions, while zone-two left projections construct four independent coordinates per vector because
/// their bases advance by the directional derivative rather than by a fixed memory stride. Every interpolation uses
/// Q5 weights and the scalar continuation preserves the same rounding and endpoint-extension rules.
/// </content>
internal static partial class Av1DirectionalIntraPredictor
{
@ -189,6 +193,8 @@ internal static partial class Av1DirectionalIntraPredictor
if (!upsample)
{
// A single index is advanced through all supported widths. A narrower path consumes only the remainder
// left by the wider path, so the row is written once without requiring padded destination storage.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = validCount - Vector512<byte>.Count;
@ -240,6 +246,8 @@ internal static partial class Av1DirectionalIntraPredictor
if (Vector128.IsHardwareAccelerated)
{
// Four-lane construction covers both the final non-upsampled remainder and targets without a native
// gather. Each lane carries an independently projected coordinate but shares the row's interpolation weight.
int oneVectorFromEnd = validCount - 4;
for (; index <= oneVectorFromEnd; index += 4)
{
@ -293,6 +301,8 @@ internal static partial class Av1DirectionalIntraPredictor
if (!upsample)
{
// High-bit-depth samples stay in signed 16-bit storage, but interpolation widens to Int32 before the Q5
// weighted sum. The largest supported 12-bit sample therefore cannot overflow an intermediate lane.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = validCount - Vector512<short>.Count;
@ -391,6 +401,8 @@ internal static partial class Av1DirectionalIntraPredictor
if (Vector128.IsHardwareAccelerated)
{
// Zone-two left references are not contiguous across output columns. Constructing the four source pairs
// directly avoids a temporary gather-index buffer and keeps the scalar continuation at the same offset.
int oneVectorFromEnd = destination.Length - 4;
for (; index <= oneVectorFromEnd; index += 4)
{

8
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1DirectionalIntraPredictor.Transpose.cs

@ -9,7 +9,10 @@ using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides SIMD block transposition for zone 3 directional prediction.
/// Provides SIMD block transposition for zone 3 directional prediction. AV1 block dimensions are multiples of four,
/// allowing the traversal to use complete eight-by-eight tiles where possible and complete four-by-four tiles for the
/// remaining small blocks. Unpack stages exchange coordinate bits inside registers; exact-width loads and stores keep
/// every access within the logical block even when the destination has no writable row padding.
/// </content>
internal static partial class Av1DirectionalIntraPredictor
{
@ -25,6 +28,8 @@ internal static partial class Av1DirectionalIntraPredictor
{
if (Vector128.IsHardwareAccelerated)
{
// Selecting one tile size for the complete block keeps both loop increments aligned with the AV1 block
// dimensions. No partial SIMD tile reaches a neighboring prediction block.
int tileSize = sourceWidth >= 8 && sourceHeight >= 8 ? 8 : 4;
for (int y = 0; y < sourceHeight; y += tileSize)
{
@ -65,6 +70,7 @@ internal static partial class Av1DirectionalIntraPredictor
{
if (Vector128.IsHardwareAccelerated)
{
// The same tiling invariant applies to two-byte samples; only the register unpack granularity differs.
int tileSize = sourceWidth >= 8 && sourceHeight >= 8 ? 8 : 4;
for (int y = 0; y < sourceHeight; y += tileSize)
{

18
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1FilterIntraPredictor.Operator.cs

@ -8,12 +8,21 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Tiling;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides recursive filter-intra traversal for the closed coefficient operators.
/// </content>
internal abstract partial class Av1FilterIntraPredictorBase
{
/// <summary>
/// Applies one closed filter-intra coefficient operator using the widest useful SIMD width.
/// </summary>
/// <typeparam name="TOperator">The filter-intra coefficient set.</typeparam>
/// <remarks>
/// AV1 filter-intra predicts a two-row by four-column group from seven samples that may include previously
/// predicted groups. The fixed-stride scratch surface preserves those dependencies with a one-sample top and left
/// border. SIMD lanes hold the eight outputs of one group in row-major order; they do not span independent groups,
/// because the next group can depend on the values just produced.
/// </remarks>
internal sealed class Av1FilterIntraPredictor<TOperator> : Av1FilterIntraPredictorBase
where TOperator : struct, IAv1FilterIntraPredictionOperator
{
@ -32,6 +41,8 @@ internal abstract partial class Av1FilterIntraPredictorBase
if (Vector256.IsHardwareAccelerated)
{
// Each tap vector contains the coefficient at one tap position for the eight row-major outputs in a
// 2-by-4 group. Broadcasting the seven reconstructed inputs therefore evaluates all outputs together.
Vector256<int> tap0 = CreateTapVector256(ref taps, 0);
Vector256<int> tap1 = CreateTapVector256(ref taps, 1);
Vector256<int> tap2 = CreateTapVector256(ref taps, 2);
@ -70,6 +81,8 @@ internal abstract partial class Av1FilterIntraPredictorBase
}
else if (Vector128.IsHardwareAccelerated)
{
// A 128-bit vector covers one four-sample output row. Low and high coefficient vectors describe the
// first and second rows respectively while sharing the same seven reconstructed input broadcasts.
Vector128<int> tap0Low = CreateTapVector128(ref taps, 0, 0);
Vector128<int> tap1Low = CreateTapVector128(ref taps, 1, 0);
Vector128<int> tap2Low = CreateTapVector128(ref taps, 2, 0);
@ -128,6 +141,8 @@ internal abstract partial class Av1FilterIntraPredictorBase
if (Vector256.IsHardwareAccelerated)
{
// High-bit-depth storage changes only the final clamp and narrowing. The Int32 accumulator layout is
// identical to the eight-bit path, preserving all signed coefficient products before Q4 rounding.
Vector256<int> tap0 = CreateTapVector256(ref taps, 0);
Vector256<int> tap1 = CreateTapVector256(ref taps, 1);
Vector256<int> tap2 = CreateTapVector256(ref taps, 2);
@ -245,6 +260,8 @@ internal abstract partial class Av1FilterIntraPredictorBase
/// <param name="topLeft">The shared top-left reference.</param>
private static void Initialize(Span<byte> buffer, ReadOnlySpan<byte> above, ReadOnlySpan<byte> left, int width, int height, byte topLeft)
{
// Predictions use one-based coordinates in the workspace. Row zero and column zero retain the prepared
// references while later groups overwrite only the interior values on which following groups depend.
buffer[0] = topLeft;
above[..width].CopyTo(buffer[1..]);
for (int row = 0; row < height; row++)
@ -264,6 +281,7 @@ internal abstract partial class Av1FilterIntraPredictorBase
/// <param name="topLeft">The shared top-left reference.</param>
private static void Initialize(Span<short> buffer, ReadOnlySpan<short> above, ReadOnlySpan<short> left, int width, int height, short topLeft)
{
// Match the eight-bit one-based workspace so the recursive source offsets remain representation-agnostic.
buffer[0] = topLeft;
above[..width].CopyTo(buffer[1..]);
for (int row = 0; row < height; row++)

3
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.HorizontalOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides horizontal intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>

14
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Operator.cs

@ -7,12 +7,20 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides the width-progressive SIMD traversal shared by closed non-directional prediction operators.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>
/// Applies one closed non-directional AV1 prediction operator using the widest available SIMD width.
/// </summary>
/// <typeparam name="TOperator">The prediction-mode-specific arithmetic.</typeparam>
/// <remarks>
/// Each lane produces one output column. Top samples and column weights vary by lane, while the current row's
/// left sample and row weight are broadcast. <typeparamref name="TOperator"/> declares which references it uses;
/// because the operator type is closed, the JIT can remove unused loads and broadcasts from each prediction mode.
/// </remarks>
internal sealed class Av1IntraPredictor<TOperator> : Av1IntraPredictorBase
where TOperator : struct, IAv1IntraPredictionOperator
{
@ -41,6 +49,8 @@ internal abstract partial class Av1IntraPredictorBase
byte bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default;
int processedColumns = 0;
// Widths are cumulative rather than mutually exclusive. A wide vector advances the row prefix, then the
// narrower paths consume any complete vectors left before the scalar tail handles the final columns.
if (Vector512.IsHardwareAccelerated)
{
int vectorizedColumns = width - (width % Vector512<byte>.Count);
@ -168,6 +178,8 @@ internal abstract partial class Av1IntraPredictorBase
short bottomLeft = usesBottomLeft ? Unsafe.Add(ref leftBase, height - 1) : default;
int processedColumns = 0;
// High-bit-depth samples use signed storage but remain nonnegative. Each vector lane follows one output
// column, so the same width-progressive traversal is valid without inter-lane packing or saturation.
if (Vector512.IsHardwareAccelerated)
{
int vectorizedColumns = width - (width % Vector512<short>.Count);
@ -256,6 +268,8 @@ internal abstract partial class Av1IntraPredictorBase
}
}
// Retain a scalar continuation for widths smaller than the available vectors and for forced-scalar test
// execution. AV1 block dimensions keep this tail short during normal hardware-accelerated decoding.
for (int row = 0; row < height; row++)
{
short leftSample = usesLeft ? Unsafe.Add(ref leftBase, row) : default;

3
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.OperatorContract.cs

@ -5,6 +5,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Defines the neighbor-usage flags and scalar/SIMD contract for closed intra-prediction operators.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>

5
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.Packing.cs

@ -9,6 +9,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <summary>
/// Widens AV1 sample vectors for fixed-point prediction arithmetic and narrows completed results.
/// </summary>
/// <remarks>
/// Widening preserves consecutive sample order while splitting packed vectors into equal low-to-high groups. Callers
/// perform interpolation in signed 32-bit lanes and clip before narrowing, so the unsigned byte overloads may use
/// unsigned narrowing and the high-bit-depth overloads may use signed narrowing without additional saturation logic.
/// </remarks>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.PaethOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides Paeth intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>

8
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothHorizontalOperator.cs

@ -6,11 +6,19 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides horizontal smooth intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>
/// Implements horizontal AV1 smooth intra prediction for scalar and SIMD lanes.
/// </summary>
/// <remarks>
/// Each lane uses its Q8 column weight to interpolate between the current row's left sample and the top-right
/// endpoint. Rewriting the complementary weight around 256 leaves one multiply per lane plus a shared endpoint and
/// rounding bias.
/// </remarks>
internal readonly struct SmoothHorizontalOperator : IAv1IntraPredictionOperator
{
/// <summary>

8
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothOperator.cs

@ -6,11 +6,19 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides two-dimensional smooth intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>
/// Implements two-dimensional AV1 smooth intra prediction for scalar and SIMD lanes.
/// </summary>
/// <remarks>
/// Horizontal and vertical Q8 interpolations are accumulated before one Q9 rounding shift. Expanding each
/// complementary weight around 256 reduces the lane equation to two products and a shared endpoint bias while
/// preserving the normative result exactly.
/// </remarks>
internal readonly struct SmoothOperator : IAv1IntraPredictionOperator
{
/// <summary>

7
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.SmoothVerticalOperator.cs

@ -6,11 +6,18 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides vertical smooth intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>
/// Implements vertical AV1 smooth intra prediction for scalar and SIMD lanes.
/// </summary>
/// <remarks>
/// The row's Q8 weight is common to every lane, while top references vary by column. Rewriting the complementary
/// weight around 256 leaves one product per lane plus a broadcast bottom-left endpoint and rounding bias.
/// </remarks>
internal readonly struct SmoothVerticalOperator : IAv1IntraPredictionOperator
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1IntraPredictor.VerticalOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <content>
/// Provides vertical intra prediction for scalar and SIMD sample representations.
/// </content>
internal abstract partial class Av1IntraPredictorBase
{
/// <summary>

6
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PalettePredictor.cs

@ -10,6 +10,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// <summary>
/// Reconstructs AV1 palette-predicted sample blocks from decoded color-index maps.
/// </summary>
/// <remarks>
/// AV1 palettes contain at most eight entries, so a complete 8-bit palette fits in each native shuffle lane and a
/// complete high-bit-depth palette fits as sixteen bytes. Color indices become byte-shuffle controls; replicating the
/// table per 128-bit lane keeps every lookup lane-local at 128, 256, and 512 bits. Exact-width tail loads and stores
/// avoid requiring writable padding around small transform blocks.
/// </remarks>
internal static class Av1PalettePredictor
{
/// <summary>

5
src/ImageSharp/Formats/Heif/Av1/Prediction/Av1PredictionDecoder.cs

@ -17,7 +17,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction;
/// Reconstructs AV1 intra-predicted transform blocks from neighboring samples and decoded mode information.
/// </summary>
/// <remarks>
/// This type implements the intra prediction portion of the AV1 reconstruction process for 8-, 10-, and 12-bit samples.
/// This type implements the intra prediction portion of the AV1 reconstruction process for 8-, 10-, and 12-bit
/// samples. Intra-edge filtering and upsampling operate on caller-owned padded scratch: adjacent reference samples map
/// to adjacent SIMD lanes, exact-width stores interleave filtered half samples with the original edge, and scalar
/// continuations handle only incomplete vectors. The completed edges then feed the closed prediction operators.
/// </remarks>
internal class Av1PredictionDecoder
{

4
src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaContext.Operations.cs

@ -12,7 +12,9 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
/// <content>
/// Provides packed luma subsampling and mean-subtraction operations for chroma-from-luma prediction.
/// Provides packed luma subsampling and mean-subtraction operations for chroma-from-luma prediction. Consecutive lanes
/// represent chroma coordinates. Horizontal and vertical luma sums are converted directly to Q3, after which a vector
/// reduction derives the common rounded mean and lane-wise subtraction leaves the zero-mean AC predictor surface.
/// </content>
internal partial class Av1ChromaFromLumaContext
{

6
src/ImageSharp/Formats/Heif/Av1/Prediction/ChromaFromLuma/Av1ChromaFromLumaPredictor.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma;
/// <summary>
/// Applies an AV1 chroma-from-luma residual to a DC-predicted chroma block.
/// </summary>
/// <remarks>
/// Each signed 16-bit lane contains one zero-mean Q3 luma value. Packed rounded-high multiplication converts the product
/// with the Q3 alpha parameter directly to a signed integer adjustment; the alpha/luma sign mask restores the product
/// sign after the magnitude operation. The common DC prediction is broadcast, then results are clipped and narrowed to
/// the destination sample representation.
/// </remarks>
internal static class Av1ChromaFromLumaPredictor
{
/// <summary>

6
src/ImageSharp/Formats/Heif/Av1/Transform/Av1ForwardTransformer.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Converts spatial residual samples into AV1 transform coefficients.
/// </summary>
/// <remarks>
/// The SIMD pipeline transposes rows into lanes before invoking the one-dimensional operators. One vector then holds
/// the same transform position from several independent axes, allowing the complete stage network to run lane-wise.
/// Eight-bit blocks use saturating 16-bit stages where their normative ranges permit it; high-bit-depth and scalar
/// fallback paths retain 32-bit stages. Both representations produce the same row-major coefficient contract.
/// </remarks>
internal static class Av1ForwardTransformer
{
/// <summary>

5
src/ImageSharp/Formats/Heif/Av1/Transform/Av1IdentityTransform1d.cs

@ -9,6 +9,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Provides the shared SIMD arithmetic for AV1 identity-transform operators.
/// </summary>
/// <remarks>
/// Transform-vector fields represent positions along an identity transform and lanes represent independent axes.
/// Scaling is lane-local; a zero fractional-bit count uses exact multiplication, while fixed-point variants use the
/// same rounded butterfly primitive as DCT and ADST operators.
/// </remarks>
internal static class Av1IdentityTransform1d
{
/// <summary>

18
src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs

@ -11,6 +11,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Applies separable two-dimensional AV1 inverse transforms and reconstructs decoded samples.
/// </summary>
/// <remarks>
/// Coefficients are transposed so that each SIMD lane represents an independent transform axis and each vector field
/// represents one coefficient position. The column and row operators can then use the scalar stage graph without
/// cross-lane permutations. Reconstruction adds the final residuals to their matching prediction lanes before
/// narrowing to the decoded sample depth.
/// </remarks>
internal static class Av1Inverse2dTransformer
{
/// <summary>
@ -320,6 +326,9 @@ internal static class Av1Inverse2dTransformer
byte rowClampBits = (byte)(bitDepth + 8);
byte columnClampBits = (byte)Math.Max(bitDepth + 6, 16);
// Three transform vectors occupy the fixed prefix of the caller-owned workspace. Reinterpreting that storage
// gives constant field offsets to the one-dimensional operators; the remaining raster buffer holds the first
// axis result without allocating or aliasing any active stage vector.
ref int workspaceBase = ref MemoryMarshal.GetReference(workspace);
ref Av1TransformVector<Vector256<int>> tempIn = ref Unsafe.As<int, Av1TransformVector<Vector256<int>>>(ref workspaceBase);
ref Av1TransformVector<Vector256<int>> tempOut = ref Unsafe.As<int, Av1TransformVector<Vector256<int>>>(ref Unsafe.Add(ref workspaceBase, vectorLength));
@ -404,7 +413,8 @@ internal static class Av1Inverse2dTransformer
ref TSample readBase = ref MemoryMarshal.GetReference(outputForRead);
ref TSample writeBase = ref MemoryMarshal.GetReference(outputForWrite);
// The intermediate rows already contain contiguous column groups, avoiding a second transpose.
// The intermediate rows already contain contiguous column groups, avoiding a second transpose. Horizontal and
// vertical flips are folded into these loads and row selections so flipped transforms need no reversal pass.
for (int column = 0; column < width; column += laneCount)
{
int sourceColumn = config.FlipLeftToRight ? width - column - laneCount : column;
@ -472,6 +482,8 @@ internal static class Av1Inverse2dTransformer
byte rowClampBits = (byte)(bitDepth + 8);
byte columnClampBits = (byte)Math.Max(bitDepth + 6, 16);
// The 128-bit workspace has the same three-vector plus raster-buffer layout as the 256-bit path. Only the
// number of independent axes represented by each vector changes from eight to four.
ref int workspaceBase = ref MemoryMarshal.GetReference(workspace);
ref Av1TransformVector<Vector128<int>> tempIn = ref Unsafe.As<int, Av1TransformVector<Vector128<int>>>(ref workspaceBase);
ref Av1TransformVector<Vector128<int>> tempOut = ref Unsafe.As<int, Av1TransformVector<Vector128<int>>>(ref Unsafe.Add(ref workspaceBase, vectorLength));
@ -480,6 +492,8 @@ internal static class Av1Inverse2dTransformer
ref int inputBase = ref MemoryMarshal.GetReference(input);
ref int bufferBase = ref MemoryMarshal.GetReference(buffer);
// A 4-by-4 transpose changes four raster rows into four coefficient-position vectors. Each lane then remains
// one independent row throughout the complete first-axis stage network.
for (int row = 0; row < height; row += laneCount)
{
for (int column = 0; column < width; column += laneCount)
@ -531,6 +545,8 @@ internal static class Av1Inverse2dTransformer
ref TSample readBase = ref MemoryMarshal.GetReference(outputForRead);
ref TSample writeBase = ref MemoryMarshal.GetReference(outputForWrite);
// Contiguous four-column groups become the independent lanes for the second axis. Flip selection is applied
// while reading the intermediate block and selecting completed rows, avoiding any extra copy or reversal.
for (int column = 0; column < width; column += laneCount)
{
int sourceColumn = config.FlipLeftToRight ? width - column - laneCount : column;

9
src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs

@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Reconstructs AV1 samples from predicted values and inverse-transform residuals.
/// </summary>
/// <remarks>
/// Each SIMD lane represents one consecutive reconstructed sample. Packed byte or 16-bit predictions are widened to
/// signed 32-bit lanes before residual addition, clipped to the coded sample range, and narrowed into exact-width
/// stores. The closed <typeparamref name="TSample"/> specialization removes storage-type branches from hot loops.
/// </remarks>
/// <typeparam name="TSample">The decoded sample storage type.</typeparam>
internal readonly struct Av1InverseTransformOutputOperator<TSample> : IAv1InverseTransformOutputOperator<TSample>
where TSample : unmanaged
@ -36,6 +41,8 @@ internal readonly struct Av1InverseTransformOutputOperator<TSample> : IAv1Invers
{
if (typeof(TSample) == typeof(byte))
{
// Read and write exactly four bytes. The unused upper lanes only participate in narrowing and never reach
// memory, which keeps reconstruction valid at a tightly packed row boundary.
ref byte source = ref Unsafe.As<TSample, byte>(ref prediction);
uint packed = Unsafe.ReadUnaligned<uint>(ref source);
Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
@ -63,6 +70,8 @@ internal readonly struct Av1InverseTransformOutputOperator<TSample> : IAv1Invers
{
if (typeof(TSample) == typeof(byte))
{
// Eight byte predictions widen through UInt16 into the eight Int32 residual lanes. The final 64-bit store
// covers only those reconstructed samples and does not require destination padding.
ref byte source = ref Unsafe.As<TSample, byte>(ref prediction);
ulong packed = Unsafe.ReadUnaligned<ulong>(ref source);
Vector128<ushort> predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());

5
src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs

@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Applies the reversible four-by-four inverse Walsh-Hadamard transform used by lossless AV1 segments.
/// </summary>
/// <remarks>
/// The vector path stores one transform row in each <see cref="Vector128{T}"/> and one column position in each lane.
/// Register transposes exchange the two transform dimensions between identical reversible butterflies. Reconstruction
/// then adds four consecutive residual lanes to each prediction row with exact-width output stores.
/// </remarks>
internal static class Av1InverseWalshHadamardTransformer
{
/// <summary>

5
src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform1dMath.cs

@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Provides the fixed-point arithmetic shared by the scalar and SIMD AV1 one-dimensional transform kernels.
/// </summary>
/// <remarks>
/// The general transform path keeps one independent axis in each signed 32-bit lane. Low-bit-depth forward transforms
/// additionally use signed 16-bit lanes and whole-butterfly AVX2 or AVX-512 multiply-add operations, matching AV1's
/// stage saturation points before packing. Scalar overloads preserve the same rounding and serve as the fallback.
/// </remarks>
internal static class Av1Transform1dMath
{
/// <summary>

6
src/ImageSharp/Formats/Heif/Av1/Transform/Av1Transform2dOperations.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Provides the SIMD data-layout operations shared by AV1 two-dimensional transforms.
/// </summary>
/// <remarks>
/// The one-dimensional operators expect one transform position per vector and one independent axis per lane. These
/// routines transpose rectangular sample tiles into that structure, then transpose the completed axes back to raster
/// order. Every shuffle is consequently an index-bit exchange between row and column coordinates; it does not alter
/// the signed fixed-point sample representation.
/// </remarks>
internal static class Av1Transform2dOperations
{
/// <summary>

10
src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformVector.cs

@ -11,12 +11,18 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// Stores the fixed set of SIMD values used by one bulk AV1 transform axis.
/// </summary>
/// <typeparam name="TVector">The SIMD vector type used for parallel transform lanes.</typeparam>
/// <remarks>
/// Field <c>Vn</c> stores transform position <c>n</c> for every independent axis in the vector lanes. A
/// <see cref="System.Runtime.Intrinsics.Vector128{T}"/> therefore carries four 32-bit axes, while a
/// <see cref="System.Runtime.Intrinsics.Vector256{T}"/> carries eight. Transform stages operate vertically through
/// these fields and never mix lanes, so their scalar fixed-point rounding and clamping rules remain unchanged.
/// </remarks>
[StructLayout(LayoutKind.Sequential)]
internal struct Av1TransformVector<TVector>
where TVector : struct
{
// Explicit fields give the JIT constant offsets inside large transform operators. The previous inline-array
// helpers were not inlined once those operators exceeded the JIT's expansion budget, causing a call per access.
// Explicit fields make each transform position a constant field offset. An indexed inline-array accessor was not
// expanded inside the larger stage networks, which introduced a helper call for every coefficient access.
public TVector V0;
public TVector V1;
public TVector V2;

5
src/ImageSharp/Formats/Heif/Av1/Transform/Av1TransformWorkspace.cs

@ -8,6 +8,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Describes the reusable storage required by an AV1 two-dimensional transform operation.
/// </summary>
/// <remarks>
/// The workspace reserves three maximum-length transform vectors for input, output, and stage exchange, followed by
/// full-block raster buffers and the fixed AVX-512 transpose area. All storage is expressed as integers so callers can
/// rent one buffer and reinterpret aligned prefixes for 16-bit or 32-bit SIMD lanes without per-block allocations.
/// </remarks>
internal static class Av1TransformWorkspace
{
/// <summary>

16
src/ImageSharp/Formats/Heif/Av1/Transform/Forward/Av1ForwardTransformArithmetic.cs

@ -11,6 +11,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Forward;
/// Provides the sample-type and vector-width arithmetic used by the shared AV1 forward-transform stage networks.
/// </summary>
/// <typeparam name="TValue">The scalar or SIMD value containing independent transform axes.</typeparam>
/// <remarks>
/// Each closed <typeparamref name="TValue"/> is either one scalar axis or a vector of independent axes. The
/// <see langword="typeof"/> branches are resolved when the generic type is compiled, so they select arithmetic once
/// without adding per-stage runtime dispatch. Signed 16-bit representations use the saturating operations required by
/// the packed transform pipeline; 32-bit representations retain the normative wrapping fixed-point arithmetic.
/// </remarks>
internal static class Av1ForwardTransformArithmetic<TValue>
where TValue : struct
{
@ -25,6 +31,8 @@ internal static class Av1ForwardTransformArithmetic<TValue>
int value = 1 << (cosBit - 1);
Av1TransformRounding rounding = default;
// The closed TValue makes this a compile-time shape selection. Only the matching explicit-layout field is
// initialized and subsequently read, keeping the broadcast outside every butterfly in the stage network.
if (typeof(TValue) == typeof(Vector128<short>) || typeof(TValue) == typeof(Vector128<int>))
{
rounding.Vector128 = Vector128.Create(value);
@ -740,6 +748,8 @@ internal static class Av1ForwardTransformArithmetic<TValue>
int cosBit,
Vector512<int> rounding)
{
// Unpacking forms adjacent (input0, input1) pairs independently inside each 128-bit lane. Pairwise multiply-add
// widens those pairs to Int32 for rounding, and the final pack restores original lane order with saturation.
Vector512<short> lowerInputs = Avx512BW.UnpackLow(input0, input1);
Vector512<short> upperInputs = Avx512BW.UnpackHigh(input0, input1);
Vector512<short> weights = Avx512BW.UnpackLow(Vector512.Create((short)weight0), Vector512.Create((short)weight1));
@ -764,6 +774,8 @@ internal static class Av1ForwardTransformArithmetic<TValue>
int cosBit,
Vector128<int> rounding)
{
// Four products can exceed Int16 even though the completed stage value cannot. Widening each input first keeps
// the full fixed-point sum until rounding; explicit clamping supplies the required saturating demotion.
(Vector128<int> input0Lower, Vector128<int> input0Upper) = Vector128.Widen(input0);
(Vector128<int> input1Lower, Vector128<int> input1Upper) = Vector128.Widen(input1);
(Vector128<int> input2Lower, Vector128<int> input2Upper) = Vector128.Widen(input2);
@ -804,6 +816,8 @@ internal static class Av1ForwardTransformArithmetic<TValue>
int cosBit,
Vector256<int> rounding)
{
// The two unpack streams contain alternating input pairs for the lower and upper lane groups. Adding the two
// pairwise products completes each four-term dot product before the rounded saturating pack restores Int16.
Vector256<short> weights01 = Avx2.UnpackLow(Vector256.Create((short)weight0), Vector256.Create((short)weight1));
Vector256<short> weights23 = Avx2.UnpackLow(Vector256.Create((short)weight2), Vector256.Create((short)weight3));
Vector256<int> lower = Avx2.MultiplyAddAdjacent(Avx2.UnpackLow(input0, input1), weights01)
@ -833,6 +847,8 @@ internal static class Av1ForwardTransformArithmetic<TValue>
int cosBit,
Vector512<int> rounding)
{
// AVX-512BW preserves the same lane-local pair layout as the 256-bit path. Two pairwise dot products form each
// four-term result in Int32, after which rounding and signed saturation return thirty-two independent axes.
Vector512<short> weights01 = Avx512BW.UnpackLow(Vector512.Create((short)weight0), Vector512.Create((short)weight1));
Vector512<short> weights23 = Avx512BW.UnpackLow(Vector512.Create((short)weight2), Vector512.Create((short)weight3));
Vector512<int> lower = Avx512BW.MultiplyAddAdjacent(Avx512BW.UnpackLow(input0, input1), weights01)

5
src/ImageSharp/Formats/Heif/Av1/Transform/IAv1InverseTransformOutputOperator.cs

@ -8,6 +8,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// <summary>
/// Defines how inverse-transform residuals are added to a decoded sample representation.
/// </summary>
/// <remarks>
/// Residual lanes correspond to consecutive reconstructed samples. Implementations must widen packed predictions,
/// add and clip in signed 32-bit lanes, then store exactly four or eight results so callers do not require writable
/// padding beyond the transform block. The closed sample type allows byte and high-bit-depth storage to specialize.
/// </remarks>
/// <typeparam name="TSample">The decoded sample storage type.</typeparam>
internal interface IAv1InverseTransformOutputOperator<TSample>
where TSample : unmanaged

7
src/ImageSharp/Formats/Heif/Av1/Transform/IAv1Transform1dOperator.cs

@ -9,8 +9,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
/// Defines the scalar and SIMD arithmetic for one AV1 one-dimensional transform.
/// </summary>
/// <remarks>
/// Each overload performs the same transform. The two-dimensional traversal selects a concrete operator and the
/// widest supported lane width once per block, allowing the JIT to specialize every static interface call.
/// Each overload performs the same staged fixed-point transform. In the SIMD overloads, each vector field identifies
/// one coefficient position and each lane identifies an independent row or column. Butterfly arithmetic is therefore
/// lane-local: vectorization changes only how many axes advance together, not coefficient order, rounding, or stage
/// clamping. The two-dimensional traversal selects the concrete operator and lane width once per block, allowing the
/// JIT to specialize every static interface call outside the stage network.
/// </remarks>
internal interface IAv1Transform1dOperator
{

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst16Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the 16-point AV1 inverse asymmetric discrete sine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Adst16Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst4Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the four-point AV1 inverse asymmetric discrete sine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Adst4Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Adst8Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the eight-point AV1 inverse asymmetric discrete sine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged rotations, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Adst8Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct16Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the 16-point AV1 inverse discrete cosine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Dct16Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct32Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the 32-point AV1 inverse discrete cosine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Dct32Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct4Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the four-point AV1 inverse discrete cosine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Dct4Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct64Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the 64-point AV1 inverse discrete cosine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Dct64Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Dct8Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the eight-point AV1 inverse discrete cosine transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. The SIMD overloads apply
/// the same staged butterflies, fixed-point rounding, and range clamps as the scalar overload without mixing axes.
/// </remarks>
internal readonly partial struct Av1Dct8Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity16Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the sixteen-point AV1 inverse identity transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
/// </remarks>
internal readonly partial struct Av1Identity16Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity32Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the thirty-two-point AV1 inverse identity transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
/// </remarks>
internal readonly partial struct Av1Identity32Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity4Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the four-point AV1 inverse identity transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
/// </remarks>
internal readonly partial struct Av1Identity4Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

4
src/ImageSharp/Formats/Heif/Av1/Transform/Inverse/Av1Identity8Inverse1dOperator.cs

@ -8,6 +8,10 @@ namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform.Inverse;
/// <summary>
/// Defines the eight-point AV1 inverse identity transform operator.
/// </summary>
/// <remarks>
/// Vector fields represent transform positions and vector lanes represent independent axes. Scaling is lane-local,
/// so the SIMD overloads preserve the scalar fixed-point multiplier and rounding for every axis.
/// </remarks>
internal readonly partial struct Av1Identity8Inverse1dOperator : IAv1Transform1dOperator
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.CoefficientOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides coefficient-matrix YCbCr conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.ConstantLuminanceOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides constant-luminance YCbCr conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.ICtCpOperator.cs

@ -7,6 +7,9 @@ using SixLabors.ImageSharp.Metadata.Profiles.Cicp;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides ICtCp transfer-domain matrix conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.IdentityOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides identity-matrix component reordering for scalar and SIMD color conversion.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.IptC2Operator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides IPT-C2 transfer-domain matrix conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

8
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Operator.cs

@ -8,13 +8,19 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides the static operator contract and SIMD traversal used by HEIF color converters.
/// Provides the static operator contract and SIMD traversal used by HEIF color converters. Each lane carries one pixel
/// and the three vectors remain planar component rows throughout conversion. Descending vector widths consume a single
/// shared offset, preserving SIMD execution for the remainder without overlapping stores or requiring row padding.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>
/// Defines color-model arithmetic for scalar and SIMD lanes in both conversion directions.
/// </summary>
/// <remarks>
/// Operator methods are lane-local and must preserve input order. The closed operator type lets the JIT bind the
/// matrix or lifting transform once per converter, keeping color-model dispatch outside every row loop.
/// </remarks>
internal interface IHeifColorOperator
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.Smpte2085Operator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides SMPTE ST 2085 YDzDx conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.YCgCoOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides YCgCo matrix conversion for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

3
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifColorConverter.YCgCoReversibleOperator.cs

@ -6,6 +6,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides reversible YCgCo integer lifting for scalar and SIMD lanes.
/// </content>
internal abstract partial class HeifColorConverterBase
{
/// <summary>

5
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifTransferFunctions.Operator.cs

@ -8,7 +8,10 @@ using static SixLabors.ImageSharp.Formats.Heif.Components.HeifTransferVectorOper
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides fixed-width vector overloads and shared H.273 transfer operations for HEIF color conversion.
/// Provides fixed-width vector overloads and shared H.273 transfer operations for HEIF color conversion. One lane
/// represents one normalized color component. Closed vector operators bind the 128-, 256-, or 512-bit implementation
/// once per row kernel, while conditional selection evaluates piecewise transfer curves without per-lane branches.
/// Inputs to logarithms and powers are bounded before evaluation because SIMD selection evaluates both branches.
/// </content>
internal static partial class HeifTransferFunctions
{

4
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.CoefficientOperator.cs

@ -7,7 +7,9 @@ using System.Runtime.Intrinsics;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Provides full-range fixed-point coefficient conversion.
/// Provides full-range fixed-point coefficient conversion. Each signed 32-bit lane carries one luma or duplicated
/// chroma sample. Matrix coefficients use a common fixed-point scale; every component rounds once after its complete
/// weighted sum, then clips before the enclosing row kernel narrows and packs the RGB result.
/// </content>
internal static partial class HeifYuv420ToRgb8Converter
{

4
src/ImageSharp/Formats/Heif/Components/ColorConverters/HeifYuv420ToRgb8Converter.Operator.cs

@ -9,7 +9,9 @@ using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// <content>
/// Defines fixed-point operators and row traversal for eight-bit 4:2:0 conversion.
/// Defines fixed-point operators and row traversal for eight-bit 4:2:0 conversion. Consecutive lanes represent output
/// pixels; each native chroma sample is duplicated into the two lanes covered by horizontal subsampling before the
/// closed color operator runs. All products remain in signed 32-bit lanes until clipped RGB values are narrowed.
/// </content>
internal static partial class HeifYuv420ToRgb8Converter
{

5
src/ImageSharp/Formats/Heif/Components/ColorConverters/IHeifSampleConverter.cs

@ -10,6 +10,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Components;
/// Defines SIMD widening and narrowing operations for one native HEIF sample representation.
/// </summary>
/// <typeparam name="TSample">The native sample type.</typeparam>
/// <remarks>
/// Every vector lane represents one consecutive planar sample. Implementations widen native integer storage to
/// normalized single-precision arithmetic and narrow only values already scaled and clipped for the destination sample
/// type. The containing row traversal selects vector width; these operators do not reorder pixels or cross row bounds.
/// </remarks>
internal interface IHeifSampleConverter<TSample>
where TSample : unmanaged
{

6
src/ImageSharp/Formats/Heif/Hevc/HevcDeblockingFilter.cs

@ -9,6 +9,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <summary>
/// Applies the HEVC luma and chroma deblocking kernels to four-sample edge segments.
/// </summary>
/// <remarks>
/// Each 32-bit lane represents one position along an edge segment. Orientation-specific operators gather the samples
/// at a common signed distance across that edge; the luma and chroma masks and adjustments then execute lane-wise.
/// Loads with fewer than four valid positions populate only the low lanes, which the matching store writes without
/// touching samples beyond the picture boundary.
/// </remarks>
internal static class HevcDeblockingFilter
{
/// <summary>

10
src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.Operations.cs

@ -10,7 +10,10 @@ using SixLabors.ImageSharp.Common.Helpers;
namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <content>
/// Provides shared SIMD operations used by the closed prediction operators.
/// Provides shared SIMD operations used by the closed prediction operators. Each lane represents one output column;
/// planar and angular interpolation widen 16-bit references before their Q5 weighted sums, then narrow only after the
/// normative rounding shift. Horizontal prediction reuses the vertical row kernel through a caller-owned contiguous
/// block and an eight-by-eight transpose, keeping the arithmetic identical without gathering strided destination rows.
/// </content>
internal static partial class HevcIntraPredictor
{
@ -120,6 +123,9 @@ internal static partial class HevcIntraPredictor
ref ushort samplesBase = ref MemoryMarshal.GetReference(samples);
uint sum = 0;
int i = 0;
// Widen before reduction because a complete 64-sample, 12-bit reference edge exceeds UInt16. The shared index
// lets narrower vectors consume only the remainder from the widest available path.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = samples.Length - Vector512<ushort>.Count;
@ -275,6 +281,8 @@ internal static partial class HevcIntraPredictor
uint rightWeight = (uint)fraction;
int i = 0;
// Adjacent source vectors overlap by one sample, aligning each left/right reference pair in the same lane.
// Widening keeps the largest 12-bit Q5 weighted sum below the UInt32 limit before narrowing to sample storage.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = destination.Length - Vector512<ushort>.Count;

6
src/ImageSharp/Formats/Heif/Hevc/HevcIntraPredictor.cs

@ -12,6 +12,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <summary>
/// Reconstructs HEVC intra-prediction blocks from prepared neighboring samples.
/// </summary>
/// <remarks>
/// Closed static operators select planar, DC, or angular arithmetic once per block. SIMD rows keep neighboring output
/// columns in consecutive lanes; broadcast left endpoints and vector top references then evaluate the interpolation
/// without per-sample mode dispatch. Horizontal angular prediction reuses the vertical kernel in contiguous scratch
/// storage and transposes once into the strided destination.
/// </remarks>
internal static partial class HevcIntraPredictor
{
/// <summary>

11
src/ImageSharp/Formats/Heif/Hevc/HevcInverseQuantizer.cs

@ -11,6 +11,11 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <summary>
/// Reconstructs dequantized HEVC transform coefficients.
/// </summary>
/// <remarks>
/// Consecutive quantized coefficients occupy consecutive 32-bit lanes. Scaling-list values are widened to the same lane
/// shape before the inverse scale, quantization shift, rounding, and transform-range clamp are applied. Vector-width
/// loops advance the complete coefficient prefix and leave only the final incomplete group to the scalar equation.
/// </remarks>
internal static class HevcInverseQuantizer
{
/// <summary>
@ -130,6 +135,8 @@ internal static class HevcInverseQuantizer
ref int destinationBase = ref MemoryMarshal.GetReference(destination);
int i = 0;
// Descending widths advance one shared coefficient offset. Smaller registers consume complete groups left by a
// wider path, so the scalar loop sees fewer than four values without any overlapping dequantization stores.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = source.Length - Vector512<int>.Count;
@ -197,6 +204,8 @@ internal static class HevcInverseQuantizer
ref byte matrixBase = ref MemoryMarshal.GetReference(matrix);
int i = 0;
// Scaling matrices use one unsigned byte per coefficient. Each width loads exactly its matching byte count,
// widens in source order, and multiplies by the common inverse-quantization scale before signed arithmetic.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = source.Length - Vector512<int>.Count;
@ -304,6 +313,8 @@ internal static class HevcInverseQuantizer
int outputMinimum,
int outputMaximum)
{
// A positive rightShift applies nearest-integer rounding before division. A nonpositive value represents an
// exact left shift; the earlier input clamp guarantees that multiplication and shifting remain in Int32 range.
values = Vector512.Clamp(values, Vector512.Create(inputMinimum), Vector512.Create(inputMaximum));
Vector512<int> result = values * weights;
result = rightShift > 0

15
src/ImageSharp/Formats/Heif/Hevc/HevcInverseTransformer.Operations.cs

@ -8,6 +8,11 @@ using SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <content>
/// Provides the shared HEVC inverse-transform stage and transpose operations. Vector lanes represent independent
/// transform lines, while consecutive scratch rows represent frequency groups in the partial-butterfly factorization.
/// Arithmetic never mixes lines; transposition is the only operation that exchanges row and column coordinates.
/// </content>
internal static partial class HevcInverseTransformer
{
/// <summary>
@ -77,6 +82,8 @@ internal static partial class HevcInverseTransformer
ref int destinationBase = ref MemoryMarshal.GetReference(destination);
int x = 0;
// Source storage is frequency-major: advancing one lane moves to the same frequency in another independent
// transform line. The shared X offset lets each narrower width continue exactly where the wider loop stopped.
if (Vector512.IsHardwareAccelerated)
{
int oneVectorFromEnd = lineCount - Vector512<int>.Count;
@ -168,6 +175,8 @@ internal static partial class HevcInverseTransformer
ReadOnlySpan<int> even = currentIsInitial ? initial : alternate;
Span<int> destination = currentIsInitial ? alternate : initial;
// Odd rows remain in the initial disjoint-group buffer while expanded even rows alternate buffers. This
// preserves every source row needed by later hierarchy levels without allocating another transform block.
for (int position = 0; position < combinedSize; position++)
{
ReadOnlySpan<int> evenRow = even.Slice((combinedStart + position) * lineCount, lineCount);
@ -385,6 +394,9 @@ internal static partial class HevcInverseTransformer
{
ref int sourceBase = ref MemoryMarshal.GetReference(source);
ref int destinationBase = ref MemoryMarshal.GetReference(destination);
// Every supported HEVC transform dimension is a multiple of four. Complete four-by-four tiles therefore
// transpose the rectangular block without masked loads, partial stores, or access to row padding.
for (int y = 0; y < sourceHeight; y += 4)
{
for (int x = 0; x < sourceWidth; x += 4)
@ -435,6 +447,8 @@ internal static partial class HevcInverseTransformer
if (Vector256.IsHardwareAccelerated)
{
// Eight UInt16 predictions widen into one Int32 vector so residual addition cannot overflow sample
// storage. Narrowing occurs only after clipping and stores exactly the eight logical destination values.
int oneVectorFromEnd = width - Vector256<int>.Count;
for (; x <= oneVectorFromEnd; x += Vector256<int>.Count)
{
@ -451,6 +465,7 @@ internal static partial class HevcInverseTransformer
if (Vector128.IsHardwareAccelerated)
{
// The four-sample path uses exact 64-bit loads and stores; it does not depend on writable row padding.
int oneVectorFromEnd = width - Vector128<int>.Count;
for (; x <= oneVectorFromEnd; x += Vector128<int>.Count)
{

19
src/ImageSharp/Formats/Heif/Hevc/HevcResidualReconstructor.cs

@ -10,6 +10,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <summary>
/// Reconstructs HEVC transform-skipped, bypassed, and differential residual blocks.
/// </summary>
/// <remarks>
/// Consecutive residual samples are widened to signed 32-bit lanes for normalization and prediction addition. Closed
/// static operators encode the selected transform-skip shift so the JIT specializes left-shift, rounded-right-shift,
/// and identity cases outside the row loops. Saturation to the residual range and clipping to sample depth occur at the
/// same stage in every vector width and in the scalar tail.
/// </remarks>
internal static class HevcResidualReconstructor
{
/// <summary>
@ -287,6 +293,8 @@ internal static class HevcResidualReconstructor
int count = coefficients.Length;
int index = 0;
// Rotation reverses the complete raster sequence, not the lanes of independently loaded forward chunks. Each
// load therefore starts at the mirrored chunk and shuffles its lanes before the common destination traversal.
if (Vector512.IsHardwareAccelerated)
{
for (; index <= count - Vector512<int>.Count; index += Vector512<int>.Count)
@ -333,6 +341,8 @@ internal static class HevcResidualReconstructor
int count = source.Length;
int index = 0;
// The descending source loads and ascending destination stores never overlap because callers provide distinct
// coefficient and residual spans. The shared index permits a scalar tail for non-vector-sized blocks.
if (Vector512.IsHardwareAccelerated)
{
for (; index <= count - Vector512<int>.Count; index += Vector512<int>.Count)
@ -433,6 +443,9 @@ internal static class HevcResidualReconstructor
{
ref int residualBase = ref MemoryMarshal.GetReference(residual);
int x = 0;
// Lanes are independent columns. Carrying the reconstructed row above in the accumulator removes the need for
// a horizontal shuffle while preserving the top-to-bottom dependency of residual DPCM.
if (Vector512.IsHardwareAccelerated)
{
Vector512<int> minimum = Vector512.Create(ResidualMinimum);
@ -508,6 +521,8 @@ internal static class HevcResidualReconstructor
int x = 0;
int accumulator = 0;
// PrefixSum resolves dependencies inside a vector. The final lane then seeds the next vector width or the
// scalar tail, so changing SIMD width cannot change the left-to-right accumulation order.
if (Vector512.IsHardwareAccelerated)
{
Vector512<int> minimum = Vector512.Create(ResidualMinimum);
@ -564,6 +579,8 @@ internal static class HevcResidualReconstructor
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<int> PrefixSum(Vector512<int> values)
{
// Out-of-range shuffle indices create zero lanes. Distances 1, 2, 4, and 8 form an inclusive Hillis-Steele
// scan without carrying values backward across the start of the vector.
values += Vector512.Shuffle(values, Vector512.Create(16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14));
values += Vector512.Shuffle(values, Vector512.Create(16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13));
values += Vector512.Shuffle(values, Vector512.Create(16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11));
@ -578,6 +595,7 @@ internal static class HevcResidualReconstructor
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector256<int> PrefixSum(Vector256<int> values)
{
// Out-of-range index eight supplies the zero lanes needed at each doubling step.
values += Vector256.Shuffle(values, Vector256.Create(8, 0, 1, 2, 3, 4, 5, 6));
values += Vector256.Shuffle(values, Vector256.Create(8, 8, 0, 1, 2, 3, 4, 5));
return values + Vector256.Shuffle(values, Vector256.Create(8, 8, 8, 8, 0, 1, 2, 3));
@ -591,6 +609,7 @@ internal static class HevcResidualReconstructor
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector128<int> PrefixSum(Vector128<int> values)
{
// Out-of-range index four supplies the zero lanes needed at distances one and two.
values += Vector128.Shuffle(values, Vector128.Create(4, 0, 1, 2));
return values + Vector128.Shuffle(values, Vector128.Create(4, 4, 0, 1));
}

10
src/ImageSharp/Formats/Heif/Hevc/HevcSampleAdaptiveOffsetFilter.cs

@ -10,6 +10,12 @@ namespace SixLabors.ImageSharp.Formats.Heif.Hevc;
/// <summary>
/// Applies HEVC sample-adaptive offsets to reconstructed component blocks.
/// </summary>
/// <remarks>
/// Each SIMD lane classifies one reconstructed sample. Band-offset operators derive the class directly from the current
/// value, while edge-offset operators compare aligned lanes from the two neighboring coordinates. The resulting class
/// indices select one of the signaled offsets, after which addition and bit-depth clipping remain lane-wise. A scalar
/// continuation handles only incomplete vectors at picture edges.
/// </remarks>
internal static class HevcSampleAdaptiveOffsetFilter
{
/// <summary>
@ -508,6 +514,8 @@ internal static class HevcSampleAdaptiveOffsetFilter
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static Vector512<short> SelectOffset(Vector512<short> classes, in KernelParameters kernel)
{
// The table contains only five values and there is no portable 16-bit gather. A comparison chain keeps every
// class lane in registers and leaves unrecognized classes at the required zero offset.
Vector512<short> selected = Vector512<short>.Zero;
selected = Vector512.ConditionalSelect(Vector512.Equals(classes, Vector512.Create((short)0)), Vector512.Create(kernel.Offset0), selected);
selected = Vector512.ConditionalSelect(Vector512.Equals(classes, Vector512.Create((short)1)), Vector512.Create(kernel.Offset1), selected);
@ -625,6 +633,8 @@ internal static class HevcSampleAdaptiveOffsetFilter
Vector512<short> neighbor1,
in KernelParameters kernel)
{
// Each comparison pair produces -1, 0, or 1. Adding two maps the normative edge classes onto the
// contiguous zero-through-four offset-table indices used by the selection kernel.
Vector512<short> one = Vector512.Create((short)1);
Vector512<short> sign0 = (Vector512.GreaterThan(current, neighbor0) & one) - (Vector512.LessThan(current, neighbor0) & one);
Vector512<short> sign1 = (Vector512.GreaterThan(current, neighbor1) & one) - (Vector512.LessThan(current, neighbor1) & one);

Loading…
Cancel
Save