mirror of https://github.com/SixLabors/ImageSharp
James Jackson-South
4 weeks ago
20 changed files with 1545 additions and 735 deletions
@ -0,0 +1,210 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.InteropServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeFilter |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Traverses one edge using the arithmetic of a closed smoothing operator.
|
||||
|
/// </summary>
|
||||
|
/// <typeparam name="TOperator">The filter-strength arithmetic.</typeparam>
|
||||
|
private static class Filter<TOperator> |
||||
|
where TOperator : struct, IEdgeFilterOperator |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Filters all samples following the preserved first sample.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The first edge sample.</param>
|
||||
|
/// <param name="count">The number of samples including the preserved sample.</param>
|
||||
|
/// <param name="scratch">The reusable source workspace.</param>
|
||||
|
public static void Apply(ref byte edge, int count, Span<byte> scratch) |
||||
|
{ |
||||
|
// Each convolution reads the original edge. Duplicate its first sample once and its last sample
|
||||
|
// twice so the five-tap windows implement endpoint clamping without per-lane boundary branches.
|
||||
|
scratch[0] = edge; |
||||
|
MemoryMarshal.CreateReadOnlySpan(ref edge, count).CopyTo(scratch[1..]); |
||||
|
scratch.Slice(count + 1, 2).Fill(Unsafe.Add(ref edge, count - 1)); |
||||
|
|
||||
|
ref byte source = ref MemoryMarshal.GetReference(scratch); |
||||
|
int outputCount = count - 1; |
||||
|
int i = 0; |
||||
|
|
||||
|
// The same offset advances through descending SIMD widths. Adjacent lanes represent adjacent
|
||||
|
// output samples, and only complete windows are loaded; the final incomplete window is scalar.
|
||||
|
if (Vector512.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector512<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector512<ushort>.Count) |
||||
|
{ |
||||
|
Vector512<ushort> s0 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 0)), Vector256<byte>.Zero)); |
||||
|
|
||||
|
Vector512<ushort> s1 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 1)), Vector256<byte>.Zero)); |
||||
|
|
||||
|
Vector512<ushort> s2 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 2)), Vector256<byte>.Zero)); |
||||
|
|
||||
|
Vector512<ushort> s3 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 3)), Vector256<byte>.Zero)); |
||||
|
|
||||
|
Vector512<ushort> s4 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 4)), Vector256<byte>.Zero)); |
||||
|
|
||||
|
Vector512<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
Vector512.Narrow(result, Vector512<ushort>.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector256.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector256<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector256<ushort>.Count) |
||||
|
{ |
||||
|
Vector256<ushort> s0 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector256<ushort> s1 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector256<ushort> s2 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector256<ushort> s3 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector256<ushort> s4 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 4)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector256<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
Vector256.Narrow(result, Vector256<ushort>.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector128.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector128<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector128<ushort>.Count) |
||||
|
{ |
||||
|
Vector128<ushort> s0 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector128<ushort> s1 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector128<ushort> s2 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector128<ushort> s3 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector128<ushort> s4 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 4)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector128<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
Vector128.Narrow(result, Vector128<ushort>.Zero).GetLower().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for (; i < outputCount; i++) |
||||
|
{ |
||||
|
int value = TOperator.Apply( |
||||
|
Unsafe.Add(ref source, i), |
||||
|
Unsafe.Add(ref source, i + 1), |
||||
|
Unsafe.Add(ref source, i + 2), |
||||
|
Unsafe.Add(ref source, i + 3), |
||||
|
Unsafe.Add(ref source, i + 4)); |
||||
|
|
||||
|
Unsafe.Add(ref edge, i + 1) = (byte)value; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters all samples following the preserved first sample.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The first edge sample.</param>
|
||||
|
/// <param name="count">The number of samples including the preserved sample.</param>
|
||||
|
/// <param name="scratch">The reusable source workspace.</param>
|
||||
|
public static void Apply(ref short edge, int count, Span<short> scratch) |
||||
|
{ |
||||
|
// Each convolution reads the original edge. Duplicate its first sample once and its last sample
|
||||
|
// twice so the five-tap windows implement endpoint clamping without per-lane boundary branches.
|
||||
|
scratch[0] = edge; |
||||
|
MemoryMarshal.CreateReadOnlySpan(ref edge, count).CopyTo(scratch[1..]); |
||||
|
scratch.Slice(count + 1, 2).Fill(Unsafe.Add(ref edge, count - 1)); |
||||
|
|
||||
|
ref short source = ref MemoryMarshal.GetReference(scratch); |
||||
|
int outputCount = count - 1; |
||||
|
int i = 0; |
||||
|
|
||||
|
// The same offset advances through descending SIMD widths. Adjacent lanes represent adjacent
|
||||
|
// output samples, and only complete windows are loaded; the final incomplete window is scalar.
|
||||
|
// Nonnegative 12-bit samples have a maximum weighted sum of 65520. The rounding bias keeps
|
||||
|
// that below 65536, so unsigned 16-bit lanes preserve the normative result at all strengths.
|
||||
|
if (Vector512.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector512<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector512<ushort>.Count) |
||||
|
{ |
||||
|
Vector512<ushort> s0 = Vector512.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); |
||||
|
Vector512<ushort> s1 = Vector512.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); |
||||
|
Vector512<ushort> s2 = Vector512.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); |
||||
|
Vector512<ushort> s3 = Vector512.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); |
||||
|
Vector512<ushort> s4 = Vector512.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); |
||||
|
|
||||
|
Vector512<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector256.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector256<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector256<ushort>.Count) |
||||
|
{ |
||||
|
Vector256<ushort> s0 = Vector256.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); |
||||
|
Vector256<ushort> s1 = Vector256.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); |
||||
|
Vector256<ushort> s2 = Vector256.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); |
||||
|
Vector256<ushort> s3 = Vector256.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); |
||||
|
Vector256<ushort> s4 = Vector256.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); |
||||
|
|
||||
|
Vector256<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector128.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = outputCount - Vector128<ushort>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector128<ushort>.Count) |
||||
|
{ |
||||
|
Vector128<ushort> s0 = Vector128.LoadUnsafe(ref source, (nuint)(i + 0)).AsUInt16(); |
||||
|
Vector128<ushort> s1 = Vector128.LoadUnsafe(ref source, (nuint)(i + 1)).AsUInt16(); |
||||
|
Vector128<ushort> s2 = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)).AsUInt16(); |
||||
|
Vector128<ushort> s3 = Vector128.LoadUnsafe(ref source, (nuint)(i + 3)).AsUInt16(); |
||||
|
Vector128<ushort> s4 = Vector128.LoadUnsafe(ref source, (nuint)(i + 4)).AsUInt16(); |
||||
|
|
||||
|
Vector128<ushort> result = TOperator.Apply(s0, s1, s2, s3, s4); |
||||
|
result.AsInt16().StoreUnsafe(ref edge, (nuint)(i + 1)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for (; i < outputCount; i++) |
||||
|
{ |
||||
|
int value = TOperator.Apply( |
||||
|
Unsafe.Add(ref source, i), |
||||
|
Unsafe.Add(ref source, i + 1), |
||||
|
Unsafe.Add(ref source, i + 2), |
||||
|
Unsafe.Add(ref source, i + 3), |
||||
|
Unsafe.Add(ref source, i + 4)); |
||||
|
|
||||
|
Unsafe.Add(ref edge, i + 1) = (short)value; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,113 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Smooths AV1 intra-reference edges while preserving their common-corner sample.
|
||||
|
/// </summary>
|
||||
|
internal static partial class Av1IntraEdgeFilter |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// The sample count required for a maximal edge and its repeated endpoints.
|
||||
|
/// </summary>
|
||||
|
public const int ScratchLength = (2 * Av1Constants.MaxTransformSize) + 4; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Defines the rounded smoothing arithmetic for one AV1 filter strength.
|
||||
|
/// </summary>
|
||||
|
internal interface IEdgeFilterOperator |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Filters one sample using the five neighboring positions.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The samples two positions before the output.</param>
|
||||
|
/// <param name="b">The preceding samples.</param>
|
||||
|
/// <param name="c">The centered samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="e">The samples two positions after the output.</param>
|
||||
|
/// <returns>The rounded filtered samples.</returns>
|
||||
|
public static abstract int Apply(int a, int b, int c, int d, int e); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters eight samples using the five neighboring positions.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The samples two positions before the output.</param>
|
||||
|
/// <param name="b">The preceding samples.</param>
|
||||
|
/// <param name="c">The centered samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="e">The samples two positions after the output.</param>
|
||||
|
/// <returns>The rounded filtered samples.</returns>
|
||||
|
public static abstract Vector128<ushort> Apply(Vector128<ushort> a, Vector128<ushort> b, Vector128<ushort> c, Vector128<ushort> d, Vector128<ushort> e); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters sixteen samples using the five neighboring positions.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The samples two positions before the output.</param>
|
||||
|
/// <param name="b">The preceding samples.</param>
|
||||
|
/// <param name="c">The centered samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="e">The samples two positions after the output.</param>
|
||||
|
/// <returns>The rounded filtered samples.</returns>
|
||||
|
public static abstract Vector256<ushort> Apply(Vector256<ushort> a, Vector256<ushort> b, Vector256<ushort> c, Vector256<ushort> d, Vector256<ushort> e); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters thirty-two samples using the five neighboring positions.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The samples two positions before the output.</param>
|
||||
|
/// <param name="b">The preceding samples.</param>
|
||||
|
/// <param name="c">The centered samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="e">The samples two positions after the output.</param>
|
||||
|
/// <returns>The rounded filtered samples.</returns>
|
||||
|
public static abstract Vector512<ushort> Apply(Vector512<ushort> a, Vector512<ushort> b, Vector512<ushort> c, Vector512<ushort> d, Vector512<ushort> e); |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters an edge in place, leaving its first sample unchanged.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The first edge sample, including the common corner when present.</param>
|
||||
|
/// <param name="count">The number of edge samples.</param>
|
||||
|
/// <param name="strength">The smoothing strength from zero through three.</param>
|
||||
|
/// <param name="scratch">The source workspace with at least <see cref="ScratchLength"/> samples.</param>
|
||||
|
public static void Apply(ref byte edge, int count, int strength, Span<byte> scratch) |
||||
|
{ |
||||
|
switch (strength) |
||||
|
{ |
||||
|
case 1: |
||||
|
Filter<Strength1Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
case 2: |
||||
|
Filter<Strength2Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
case 3: |
||||
|
Filter<Strength3Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters an edge in place, leaving its first sample unchanged.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The first edge sample, including the common corner when present.</param>
|
||||
|
/// <param name="count">The number of edge samples.</param>
|
||||
|
/// <param name="strength">The smoothing strength from zero through three.</param>
|
||||
|
/// <param name="scratch">The source workspace with at least <see cref="ScratchLength"/> samples.</param>
|
||||
|
public static void Apply(ref short edge, int count, int strength, Span<short> scratch) |
||||
|
{ |
||||
|
switch (strength) |
||||
|
{ |
||||
|
case 1: |
||||
|
Filter<Strength1Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
case 2: |
||||
|
Filter<Strength2Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
case 3: |
||||
|
Filter<Strength3Operator>.Apply(ref edge, count, scratch); |
||||
|
break; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeFilter |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Applies the strength-1 three-tap edge smoothing kernel.
|
||||
|
/// </summary>
|
||||
|
internal readonly struct Strength1Operator : IEdgeFilterOperator |
||||
|
{ |
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static int Apply(int a, int b, int c, int d, int e) |
||||
|
=> (b + (c << 1) + d + 2) >> 2; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector128<ushort> Apply(Vector128<ushort> a, Vector128<ushort> b, Vector128<ushort> c, Vector128<ushort> d, Vector128<ushort> e) |
||||
|
=> (b + (c << 1) + d + Vector128.Create((ushort)2)) >> 2; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector256<ushort> Apply(Vector256<ushort> a, Vector256<ushort> b, Vector256<ushort> c, Vector256<ushort> d, Vector256<ushort> e) |
||||
|
=> (b + (c << 1) + d + Vector256.Create((ushort)2)) >> 2; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector512<ushort> Apply(Vector512<ushort> a, Vector512<ushort> b, Vector512<ushort> c, Vector512<ushort> d, Vector512<ushort> e) |
||||
|
=> (b + (c << 1) + d + Vector512.Create((ushort)2)) >> 2; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeFilter |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Applies the strength-2 three-tap edge smoothing kernel.
|
||||
|
/// </summary>
|
||||
|
internal readonly struct Strength2Operator : IEdgeFilterOperator |
||||
|
{ |
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static int Apply(int a, int b, int c, int d, int e) |
||||
|
=> (((b + d) * 5) + (c * 6) + 8) >> 4; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector128<ushort> Apply(Vector128<ushort> a, Vector128<ushort> b, Vector128<ushort> c, Vector128<ushort> d, Vector128<ushort> e) |
||||
|
=> (((b + d) * Vector128.Create((ushort)5)) + (c * Vector128.Create((ushort)6)) + Vector128.Create((ushort)8)) >> 4; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector256<ushort> Apply(Vector256<ushort> a, Vector256<ushort> b, Vector256<ushort> c, Vector256<ushort> d, Vector256<ushort> e) |
||||
|
=> (((b + d) * Vector256.Create((ushort)5)) + (c * Vector256.Create((ushort)6)) + Vector256.Create((ushort)8)) >> 4; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector512<ushort> Apply(Vector512<ushort> a, Vector512<ushort> b, Vector512<ushort> c, Vector512<ushort> d, Vector512<ushort> e) |
||||
|
=> (((b + d) * Vector512.Create((ushort)5)) + (c * Vector512.Create((ushort)6)) + Vector512.Create((ushort)8)) >> 4; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeFilter |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Applies the strength-3 five-tap edge smoothing kernel.
|
||||
|
/// </summary>
|
||||
|
internal readonly struct Strength3Operator : IEdgeFilterOperator |
||||
|
{ |
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static int Apply(int a, int b, int c, int d, int e) |
||||
|
=> (a + ((b + c + d) << 1) + e + 4) >> 3; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector128<ushort> Apply(Vector128<ushort> a, Vector128<ushort> b, Vector128<ushort> c, Vector128<ushort> d, Vector128<ushort> e) |
||||
|
=> (a + ((b + c + d) << 1) + e + Vector128.Create((ushort)4)) >> 3; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector256<ushort> Apply(Vector256<ushort> a, Vector256<ushort> b, Vector256<ushort> c, Vector256<ushort> d, Vector256<ushort> e) |
||||
|
=> (a + ((b + c + d) << 1) + e + Vector256.Create((ushort)4)) >> 3; |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector512<ushort> Apply(Vector512<ushort> a, Vector512<ushort> b, Vector512<ushort> c, Vector512<ushort> d, Vector512<ushort> e) |
||||
|
=> (a + ((b + c + d) << 1) + e + Vector512.Create((ushort)4)) >> 3; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,289 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Numerics; |
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.InteropServices; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Prepares directional intra-reference edges for AV1 smoothing and half-sample prediction.
|
||||
|
/// </summary>
|
||||
|
internal static class Av1IntraEdgePreparation |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// The number of samples reserved before the first edge sample.
|
||||
|
/// </summary>
|
||||
|
public const int ReferencePrefixLength = 16; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// The total sample capacity of one edge including prefix and extension.
|
||||
|
/// </summary>
|
||||
|
public const int ReferenceBufferLength = (2 * Av1Constants.MaxTransformSize) + 32; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Filters and upsamples prepared directional reference edges.
|
||||
|
/// </summary>
|
||||
|
/// <typeparam name="T">The byte or signed high-bit-depth sample type.</typeparam>
|
||||
|
/// <param name="above">The top edge with writable prefix and extension.</param>
|
||||
|
/// <param name="left">The left edge with writable prefix and extension.</param>
|
||||
|
/// <param name="width">The transform width.</param>
|
||||
|
/// <param name="height">The transform height.</param>
|
||||
|
/// <param name="angle">The adjusted directional angle.</param>
|
||||
|
/// <param name="topCount">The number of available top samples before extension.</param>
|
||||
|
/// <param name="leftCount">The number of available left samples before extension.</param>
|
||||
|
/// <param name="filterType">Whether a relevant neighbor uses smooth prediction.</param>
|
||||
|
/// <param name="bitDepth">The coded sample precision.</param>
|
||||
|
/// <param name="scratch">The original-edge workspace with at least <see cref="Av1IntraEdgeFilter.ScratchLength"/> samples.</param>
|
||||
|
/// <param name="upsampleAbove">Whether the top edge contains half-sample positions.</param>
|
||||
|
/// <param name="upsampleLeft">Whether the left edge contains half-sample positions.</param>
|
||||
|
public static void Prepare<T>( |
||||
|
Span<T> above, |
||||
|
Span<T> left, |
||||
|
int width, |
||||
|
int height, |
||||
|
int angle, |
||||
|
int topCount, |
||||
|
int leftCount, |
||||
|
bool filterType, |
||||
|
int bitDepth, |
||||
|
Span<T> scratch, |
||||
|
out bool upsampleAbove, |
||||
|
out bool upsampleLeft) |
||||
|
where T : unmanaged, IBinaryInteger<T> |
||||
|
{ |
||||
|
bool needAbove = angle < 180; |
||||
|
bool needLeft = angle > 90; |
||||
|
bool needRight = angle < 90; |
||||
|
bool needBottom = angle > 180; |
||||
|
upsampleAbove = false; |
||||
|
upsampleLeft = false; |
||||
|
|
||||
|
// A missing sole edge produces a constant block from the perpendicular sample or midpoint offset.
|
||||
|
// Its prepared edge already repeats that value. Upsampling its distinct corner would change it.
|
||||
|
if ((!needAbove && leftCount == 0) || (!needLeft && topCount == 0)) |
||||
|
{ |
||||
|
return; |
||||
|
} |
||||
|
|
||||
|
if (angle is not 90 and not 180) |
||||
|
{ |
||||
|
if (needAbove && needLeft && width + height >= 24) |
||||
|
{ |
||||
|
// The corner is one logical sample represented in both edge prefixes. Filter it first,
|
||||
|
// then let both edge convolutions read the same rounded [5, 6, 5] corner value.
|
||||
|
ref T corner = ref Unsafe.Subtract(ref above[0], 1); |
||||
|
int value = (5 * int.CreateChecked(left[0])) |
||||
|
+ (6 * int.CreateChecked(corner)) |
||||
|
+ (5 * int.CreateChecked(above[0])); |
||||
|
|
||||
|
corner = T.CreateChecked((value + 8) >> 4); |
||||
|
Unsafe.Subtract(ref left[0], 1) = corner; |
||||
|
} |
||||
|
|
||||
|
if (needAbove && topCount > 0) |
||||
|
{ |
||||
|
int strength = IntraEdgeFilterStrength(width, height, angle - 90, filterType); |
||||
|
Filter(ref Unsafe.Subtract(ref above[0], 1), topCount + 1 + (needRight ? height : 0), strength, scratch); |
||||
|
} |
||||
|
|
||||
|
if (needLeft && leftCount > 0) |
||||
|
{ |
||||
|
int strength = IntraEdgeFilterStrength(height, width, angle - 180, filterType); |
||||
|
Filter(ref Unsafe.Subtract(ref left[0], 1), leftCount + 1 + (needBottom ? width : 0), strength, scratch); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
upsampleAbove = UseUpsampling(width, height, angle - 90, filterType); |
||||
|
if (needAbove && upsampleAbove) |
||||
|
{ |
||||
|
Upsample(above, width + (needRight ? height : 0), bitDepth, scratch); |
||||
|
} |
||||
|
|
||||
|
upsampleLeft = UseUpsampling(height, width, angle - 180, filterType); |
||||
|
if (needLeft && upsampleLeft) |
||||
|
{ |
||||
|
Upsample(left, height + (needBottom ? width : 0), bitDepth, scratch); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Selects half-sample interpolation for a transform edge.
|
||||
|
/// </summary>
|
||||
|
/// <param name="width">The transform width.</param>
|
||||
|
/// <param name="height">The transform height.</param>
|
||||
|
/// <param name="delta">The angle relative to the edge's cardinal direction.</param>
|
||||
|
/// <param name="filterType">Whether a relevant neighbor uses smooth prediction.</param>
|
||||
|
/// <returns>Whether the edge uses half-sample interpolation.</returns>
|
||||
|
private static bool UseUpsampling(int width, int height, int delta, bool filterType) |
||||
|
{ |
||||
|
int distance = Math.Abs(delta); |
||||
|
return distance > 0 && distance < 40 && width + height <= (filterType ? 8 : 16); |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Dispatches edge smoothing to the concrete sample representation.
|
||||
|
/// </summary>
|
||||
|
/// <typeparam name="T">The byte or signed high-bit-depth sample type.</typeparam>
|
||||
|
/// <param name="edge">The first edge sample, including the corner.</param>
|
||||
|
/// <param name="count">The number of edge samples.</param>
|
||||
|
/// <param name="strength">The smoothing strength.</param>
|
||||
|
/// <param name="scratch">The reusable original-edge workspace.</param>
|
||||
|
private static void Filter<T>(ref T edge, int count, int strength, Span<T> scratch) |
||||
|
where T : unmanaged, IBinaryInteger<T> |
||||
|
{ |
||||
|
if (typeof(T) == typeof(byte)) |
||||
|
{ |
||||
|
Av1IntraEdgeFilter.Apply(ref Unsafe.As<T, byte>(ref edge), count, strength, MemoryMarshal.Cast<T, byte>(scratch)); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
Av1IntraEdgeFilter.Apply(ref Unsafe.As<T, short>(ref edge), count, strength, MemoryMarshal.Cast<T, short>(scratch)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Dispatches half-sample interpolation to the concrete sample representation.
|
||||
|
/// </summary>
|
||||
|
/// <typeparam name="T">The byte or signed high-bit-depth sample type.</typeparam>
|
||||
|
/// <param name="edge">The edge with writable prefix and extension.</param>
|
||||
|
/// <param name="count">The number of original edge samples.</param>
|
||||
|
/// <param name="bitDepth">The coded precision.</param>
|
||||
|
/// <param name="scratch">The reusable original-edge workspace.</param>
|
||||
|
private static void Upsample<T>(Span<T> edge, int count, int bitDepth, Span<T> scratch) |
||||
|
where T : unmanaged, IBinaryInteger<T> |
||||
|
{ |
||||
|
if (typeof(T) == typeof(byte)) |
||||
|
{ |
||||
|
Av1IntraEdgeUpsampler.Apply(MemoryMarshal.Cast<T, byte>(edge), count, MemoryMarshal.Cast<T, byte>(scratch)); |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
Av1IntraEdgeUpsampler.Apply(MemoryMarshal.Cast<T, short>(edge), count, bitDepth, MemoryMarshal.Cast<T, short>(scratch)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Selects the AV1 intra-edge filter strength for the block dimensions and prediction angle.
|
||||
|
/// </summary>
|
||||
|
/// <param name="width">The edge's primary block dimension.</param>
|
||||
|
/// <param name="height">The edge's secondary block dimension.</param>
|
||||
|
/// <param name="delta">The prediction angle relative to the edge's cardinal direction.</param>
|
||||
|
/// <param name="filterType">A value indicating whether a neighboring smooth mode selects the alternate thresholds.</param>
|
||||
|
/// <returns>The filter strength from zero for no filtering through three for the strongest kernel.</returns>
|
||||
|
private static int IntraEdgeFilterStrength(int width, int height, int delta, bool filterType) |
||||
|
{ |
||||
|
int d = Math.Abs(delta); |
||||
|
int strength = 0; |
||||
|
int widthHeight = width + height; |
||||
|
if (!filterType) |
||||
|
{ |
||||
|
if (widthHeight <= 8) |
||||
|
{ |
||||
|
if (d >= 56) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 12) |
||||
|
{ |
||||
|
if (d >= 40) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 16) |
||||
|
{ |
||||
|
if (d >= 40) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 24) |
||||
|
{ |
||||
|
if (d >= 8) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
|
||||
|
if (d >= 16) |
||||
|
{ |
||||
|
strength = 2; |
||||
|
} |
||||
|
|
||||
|
if (d >= 32) |
||||
|
{ |
||||
|
strength = 3; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 32) |
||||
|
{ |
||||
|
if (d >= 1) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
|
||||
|
if (d >= 4) |
||||
|
{ |
||||
|
strength = 2; |
||||
|
} |
||||
|
|
||||
|
if (d >= 32) |
||||
|
{ |
||||
|
strength = 3; |
||||
|
} |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
if (d >= 1) |
||||
|
{ |
||||
|
strength = 3; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
if (widthHeight <= 8) |
||||
|
{ |
||||
|
if (d >= 40) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
|
||||
|
if (d >= 64) |
||||
|
{ |
||||
|
strength = 2; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 16) |
||||
|
{ |
||||
|
if (d >= 20) |
||||
|
{ |
||||
|
strength = 1; |
||||
|
} |
||||
|
|
||||
|
if (d >= 48) |
||||
|
{ |
||||
|
strength = 2; |
||||
|
} |
||||
|
} |
||||
|
else if (widthHeight <= 24) |
||||
|
{ |
||||
|
if (d >= 4) |
||||
|
{ |
||||
|
strength = 3; |
||||
|
} |
||||
|
} |
||||
|
else |
||||
|
{ |
||||
|
if (d >= 1) |
||||
|
{ |
||||
|
strength = 3; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
return strength; |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,48 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeUpsampler |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Applies the AV1 [-1, 9, 9, -1] interpolation kernel with Q4 rounding and clipping.
|
||||
|
/// </summary>
|
||||
|
internal readonly struct FourTapOperator : IEdgeUpsamplingOperator |
||||
|
{ |
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static int Interpolate(int a, int b, int c, int d, int maximum) |
||||
|
=> Math.Clamp((((9 * (b + c)) - a - d) + 8) >> 4, 0, maximum); |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector128<int> Interpolate(Vector128<int> a, Vector128<int> b, Vector128<int> c, Vector128<int> d, int maximum) |
||||
|
{ |
||||
|
// Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710.
|
||||
|
Vector128<int> value = (((Vector128.Create(9) * (b + c)) - a - d) + Vector128.Create(8)) >> 4; |
||||
|
return Vector128.Clamp(value, Vector128<int>.Zero, Vector128.Create(maximum)); |
||||
|
} |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector256<int> Interpolate(Vector256<int> a, Vector256<int> b, Vector256<int> c, Vector256<int> d, int maximum) |
||||
|
{ |
||||
|
// Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710.
|
||||
|
Vector256<int> value = (((Vector256.Create(9) * (b + c)) - a - d) + Vector256.Create(8)) >> 4; |
||||
|
return Vector256.Clamp(value, Vector256<int>.Zero, Vector256.Create(maximum)); |
||||
|
} |
||||
|
|
||||
|
/// <inheritdoc/>
|
||||
|
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
||||
|
public static Vector512<int> Interpolate(Vector512<int> a, Vector512<int> b, Vector512<int> c, Vector512<int> d, int maximum) |
||||
|
{ |
||||
|
// Signed 32-bit lanes preserve negative overshoot and the 12-bit central sum, which can reach 73710.
|
||||
|
Vector512<int> value = (((Vector512.Create(9) * (b + c)) - a - d) + Vector512.Create(8)) >> 4; |
||||
|
return Vector512.Clamp(value, Vector512<int>.Zero, Vector512.Create(maximum)); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,278 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.CompilerServices; |
||||
|
using System.Runtime.InteropServices; |
||||
|
using System.Runtime.Intrinsics; |
||||
|
using SixLabors.ImageSharp.Common.Helpers; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
internal static partial class Av1IntraEdgeUpsampler |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Traverses a bounded edge using a closed interpolation operator.
|
||||
|
/// </summary>
|
||||
|
/// <typeparam name="TOperator">The four-tap interpolation arithmetic.</typeparam>
|
||||
|
private static class Upsampler<TOperator> |
||||
|
where TOperator : struct, IEdgeUpsamplingOperator |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Inserts half samples using the original edge values and repeated endpoints.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The edge with prefix and doubled output capacity.</param>
|
||||
|
/// <param name="count">The original sample count.</param>
|
||||
|
/// <param name="scratch">The reusable original-sample workspace.</param>
|
||||
|
public static void Apply(Span<byte> edge, int count, Span<byte> scratch) |
||||
|
{ |
||||
|
ref byte destination = ref MemoryMarshal.GetReference(edge); |
||||
|
ref byte source = ref MemoryMarshal.GetReference(scratch); |
||||
|
|
||||
|
// Preserve the corner twice and the final sample once. Every SIMD load below covers exactly its
|
||||
|
// input lanes, so the native 16+3-sample workspace also suffices for the widest interpolation.
|
||||
|
source = Unsafe.Subtract(ref destination, 1); |
||||
|
Unsafe.Add(ref source, 1) = source; |
||||
|
edge[..count].CopyTo(scratch[2..]); |
||||
|
Unsafe.Add(ref source, count + 2) = edge[count - 1]; |
||||
|
Unsafe.Subtract(ref destination, 2) = source; |
||||
|
ref byte firstOutput = ref Unsafe.Subtract(ref destination, 1); |
||||
|
int i = 0; |
||||
|
|
||||
|
if (Vector512.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector512<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector512<int>.Count) |
||||
|
{ |
||||
|
Vector256<ushort> w0 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector512<int> s0 = Vector512.WidenLower(Vector512.Create(w0, Vector256<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector256<ushort> w1 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector512<int> s1 = Vector512.WidenLower(Vector512.Create(w1, Vector256<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector256<ushort> w2 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector512<int> s2 = Vector512.WidenLower(Vector512.Create(w2, Vector256<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector256<ushort> w3 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128<byte>.Zero)); |
||||
|
|
||||
|
Vector512<int> s3 = Vector512.WidenLower(Vector512.Create(w3, Vector256<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector512<int> values = TOperator.Interpolate(s0, s1, s2, s3, 255); |
||||
|
Vector256<ushort> halfWords = Vector512.Narrow(values, Vector512<int>.Zero).GetLower().AsUInt16(); |
||||
|
Vector128<byte> halfSamples = Vector256.Narrow(halfWords, Vector256<ushort>.Zero).GetLower(); |
||||
|
Vector128<byte> originals = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)); |
||||
|
|
||||
|
// Unpack the lower and upper eight pairs independently to retain linear sample order.
|
||||
|
Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
Vector128_.UnpackHigh(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 16)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector256.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector256<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector256<int>.Count) |
||||
|
{ |
||||
|
Vector128<ushort> w0 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector256<int> s0 = Vector256.WidenLower(Vector256.Create(w0, Vector128<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector128<ushort> w1 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector256<int> s1 = Vector256.WidenLower(Vector256.Create(w1, Vector128<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector128<ushort> w2 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector256<int> s2 = Vector256.WidenLower(Vector256.Create(w2, Vector128<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector128<ushort> w3 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64<byte>.Zero)); |
||||
|
|
||||
|
Vector256<int> s3 = Vector256.WidenLower(Vector256.Create(w3, Vector128<ushort>.Zero)).AsInt32(); |
||||
|
|
||||
|
Vector256<int> values = TOperator.Interpolate(s0, s1, s2, s3, 255); |
||||
|
Vector128<ushort> halfWords = Vector256.Narrow(values, Vector256<int>.Zero).GetLower().AsUInt16(); |
||||
|
Vector128<byte> halfSamples = Vector128.Narrow(halfWords, Vector128<ushort>.Zero); |
||||
|
Vector128<byte> originals = Vector128.Create(Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64<byte>.Zero); |
||||
|
Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector128.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector128<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector128<int>.Count) |
||||
|
{ |
||||
|
Vector128<byte> b0 = Vector128.CreateScalar(Unsafe.As<byte, uint>(ref Unsafe.Add(ref source, i + 0))).AsByte(); |
||||
|
Vector128<ushort> w0 = Vector128.WidenLower(b0); |
||||
|
Vector128<int> s0 = Vector128.WidenLower(w0).AsInt32(); |
||||
|
|
||||
|
Vector128<byte> b1 = Vector128.CreateScalar(Unsafe.As<byte, uint>(ref Unsafe.Add(ref source, i + 1))).AsByte(); |
||||
|
Vector128<ushort> w1 = Vector128.WidenLower(b1); |
||||
|
Vector128<int> s1 = Vector128.WidenLower(w1).AsInt32(); |
||||
|
|
||||
|
Vector128<byte> b2 = Vector128.CreateScalar(Unsafe.As<byte, uint>(ref Unsafe.Add(ref source, i + 2))).AsByte(); |
||||
|
Vector128<ushort> w2 = Vector128.WidenLower(b2); |
||||
|
Vector128<int> s2 = Vector128.WidenLower(w2).AsInt32(); |
||||
|
|
||||
|
Vector128<byte> b3 = Vector128.CreateScalar(Unsafe.As<byte, uint>(ref Unsafe.Add(ref source, i + 3))).AsByte(); |
||||
|
Vector128<ushort> w3 = Vector128.WidenLower(b3); |
||||
|
Vector128<int> s3 = Vector128.WidenLower(w3).AsInt32(); |
||||
|
|
||||
|
Vector128<int> values = TOperator.Interpolate(s0, s1, s2, s3, 255); |
||||
|
Vector128<byte> halfSamples = Vector128.Narrow( |
||||
|
Vector128.Narrow(values, Vector128<int>.Zero).AsUInt16(), Vector128<ushort>.Zero); |
||||
|
|
||||
|
Vector128<byte> originals = Vector128.CreateScalar(Unsafe.As<byte, uint>(ref Unsafe.Add(ref source, i + 2))).AsByte(); |
||||
|
Vector128_.UnpackLow(halfSamples, originals).GetLower().StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for (; i < count; i++) |
||||
|
{ |
||||
|
int value = TOperator.Interpolate( |
||||
|
Unsafe.Add(ref source, i), |
||||
|
Unsafe.Add(ref source, i + 1), |
||||
|
Unsafe.Add(ref source, i + 2), |
||||
|
Unsafe.Add(ref source, i + 3), |
||||
|
255); |
||||
|
|
||||
|
Unsafe.Add(ref destination, (2 * i) - 1) = (byte)value; |
||||
|
Unsafe.Add(ref destination, 2 * i) = Unsafe.Add(ref source, i + 2); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Inserts half samples using the original edge values and repeated endpoints.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The edge with prefix and doubled output capacity.</param>
|
||||
|
/// <param name="count">The original sample count.</param>
|
||||
|
/// <param name="maximum">The maximum coded sample value.</param>
|
||||
|
/// <param name="scratch">The reusable original-sample workspace.</param>
|
||||
|
public static void Apply(Span<short> edge, int count, int maximum, Span<short> scratch) |
||||
|
{ |
||||
|
ref short destination = ref MemoryMarshal.GetReference(edge); |
||||
|
ref short source = ref MemoryMarshal.GetReference(scratch); |
||||
|
|
||||
|
// Preserve the corner twice and the final sample once. Every SIMD load below covers exactly its
|
||||
|
// input lanes, so the native 16+3-sample workspace also suffices for the widest interpolation.
|
||||
|
source = Unsafe.Subtract(ref destination, 1); |
||||
|
Unsafe.Add(ref source, 1) = source; |
||||
|
edge[..count].CopyTo(scratch[2..]); |
||||
|
Unsafe.Add(ref source, count + 2) = edge[count - 1]; |
||||
|
Unsafe.Subtract(ref destination, 2) = source; |
||||
|
ref short firstOutput = ref Unsafe.Subtract(ref destination, 1); |
||||
|
int i = 0; |
||||
|
|
||||
|
if (Vector512.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector512<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector512<int>.Count) |
||||
|
{ |
||||
|
Vector512<int> s0 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 0)), Vector256<short>.Zero)); |
||||
|
|
||||
|
Vector512<int> s1 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 1)), Vector256<short>.Zero)); |
||||
|
|
||||
|
Vector512<int> s2 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 2)), Vector256<short>.Zero)); |
||||
|
|
||||
|
Vector512<int> s3 = Vector512.WidenLower(Vector512.Create( |
||||
|
Vector256.LoadUnsafe(ref source, (nuint)(i + 3)), Vector256<short>.Zero)); |
||||
|
|
||||
|
Vector512<int> values = TOperator.Interpolate(s0, s1, s2, s3, maximum); |
||||
|
Vector256<short> halfSamples = Vector512.Narrow(values, Vector512<int>.Zero).GetLower(); |
||||
|
Vector256<short> originals = Vector256.LoadUnsafe(ref source, (nuint)(i + 2)); |
||||
|
|
||||
|
// Four contiguous groups of four pairs avoid treating lane-local unpack order as one
|
||||
|
// linear 256-bit edge. Each store writes only prepared half samples and their originals.
|
||||
|
Vector128_.UnpackLow(halfSamples.GetLower(), originals.GetLower()) |
||||
|
.StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
|
||||
|
Vector128_.UnpackHigh(halfSamples.GetLower(), originals.GetLower()) |
||||
|
.StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 8)); |
||||
|
|
||||
|
Vector128_.UnpackLow(halfSamples.GetUpper(), originals.GetUpper()) |
||||
|
.StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 16)); |
||||
|
|
||||
|
Vector128_.UnpackHigh(halfSamples.GetUpper(), originals.GetUpper()) |
||||
|
.StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 24)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector256.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector256<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector256<int>.Count) |
||||
|
{ |
||||
|
Vector256<int> s0 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 0)), Vector128<short>.Zero)); |
||||
|
|
||||
|
Vector256<int> s1 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 1)), Vector128<short>.Zero)); |
||||
|
|
||||
|
Vector256<int> s2 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 2)), Vector128<short>.Zero)); |
||||
|
|
||||
|
Vector256<int> s3 = Vector256.WidenLower(Vector256.Create( |
||||
|
Vector128.LoadUnsafe(ref source, (nuint)(i + 3)), Vector128<short>.Zero)); |
||||
|
|
||||
|
Vector256<int> values = TOperator.Interpolate(s0, s1, s2, s3, maximum); |
||||
|
Vector128<short> halfSamples = Vector256.Narrow(values, Vector256<int>.Zero).GetLower(); |
||||
|
Vector128<short> originals = Vector128.LoadUnsafe(ref source, (nuint)(i + 2)); |
||||
|
Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
Vector128_.UnpackHigh(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)((2 * i) + 8)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
if (Vector128.IsHardwareAccelerated) |
||||
|
{ |
||||
|
int vectorEnd = count - Vector128<int>.Count; |
||||
|
for (; i <= vectorEnd; i += Vector128<int>.Count) |
||||
|
{ |
||||
|
Vector128<int> s0 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 0)), Vector64<short>.Zero)); |
||||
|
|
||||
|
Vector128<int> s1 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 1)), Vector64<short>.Zero)); |
||||
|
|
||||
|
Vector128<int> s2 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64<short>.Zero)); |
||||
|
|
||||
|
Vector128<int> s3 = Vector128.WidenLower(Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 3)), Vector64<short>.Zero)); |
||||
|
|
||||
|
Vector128<int> values = TOperator.Interpolate(s0, s1, s2, s3, maximum); |
||||
|
Vector128<short> halfSamples = Vector128.Narrow(values, Vector128<int>.Zero); |
||||
|
Vector128<short> originals = Vector128.Create( |
||||
|
Vector64.LoadUnsafe(ref source, (nuint)(i + 2)), Vector64<short>.Zero); |
||||
|
|
||||
|
Vector128_.UnpackLow(halfSamples, originals).StoreUnsafe(ref firstOutput, (nuint)(2 * i)); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
for (; i < count; i++) |
||||
|
{ |
||||
|
int value = TOperator.Interpolate( |
||||
|
Unsafe.Add(ref source, i), |
||||
|
Unsafe.Add(ref source, i + 1), |
||||
|
Unsafe.Add(ref source, i + 2), |
||||
|
Unsafe.Add(ref source, i + 3), |
||||
|
maximum); |
||||
|
|
||||
|
Unsafe.Add(ref destination, (2 * i) - 1) = (short)value; |
||||
|
Unsafe.Add(ref destination, 2 * i) = Unsafe.Add(ref source, i + 2); |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,91 @@ |
|||||
|
// Copyright (c) Six Labors.
|
||||
|
// Licensed under the Six Labors Split License.
|
||||
|
|
||||
|
using System.Runtime.Intrinsics; |
||||
|
|
||||
|
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Inserts clipped half-sample positions into AV1 intra-reference edges.
|
||||
|
/// </summary>
|
||||
|
internal static partial class Av1IntraEdgeUpsampler |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// The maximum number of original samples permitted in an upsampled edge.
|
||||
|
/// </summary>
|
||||
|
public const int MaximumCount = 16; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// The sample count required for the original edge, corner, and repeated endpoints.
|
||||
|
/// </summary>
|
||||
|
public const int ScratchLength = MaximumCount + 3; |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Defines signed four-tap interpolation before sample narrowing.
|
||||
|
/// </summary>
|
||||
|
internal interface IEdgeUpsamplingOperator |
||||
|
{ |
||||
|
/// <summary>
|
||||
|
/// Interpolates half samples and clamps them to the coded range.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The preceding samples.</param>
|
||||
|
/// <param name="b">The first central samples.</param>
|
||||
|
/// <param name="c">The second central samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="maximum">The maximum coded sample.</param>
|
||||
|
/// <returns>The rounded and clipped half samples.</returns>
|
||||
|
public static abstract int Interpolate(int a, int b, int c, int d, int maximum); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Interpolates half samples and clamps them to the coded range.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The preceding samples.</param>
|
||||
|
/// <param name="b">The first central samples.</param>
|
||||
|
/// <param name="c">The second central samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="maximum">The maximum coded sample.</param>
|
||||
|
/// <returns>The rounded and clipped half samples.</returns>
|
||||
|
public static abstract Vector128<int> Interpolate(Vector128<int> a, Vector128<int> b, Vector128<int> c, Vector128<int> d, int maximum); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Interpolates half samples and clamps them to the coded range.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The preceding samples.</param>
|
||||
|
/// <param name="b">The first central samples.</param>
|
||||
|
/// <param name="c">The second central samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="maximum">The maximum coded sample.</param>
|
||||
|
/// <returns>The rounded and clipped half samples.</returns>
|
||||
|
public static abstract Vector256<int> Interpolate(Vector256<int> a, Vector256<int> b, Vector256<int> c, Vector256<int> d, int maximum); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Interpolates half samples and clamps them to the coded range.
|
||||
|
/// </summary>
|
||||
|
/// <param name="a">The preceding samples.</param>
|
||||
|
/// <param name="b">The first central samples.</param>
|
||||
|
/// <param name="c">The second central samples.</param>
|
||||
|
/// <param name="d">The following samples.</param>
|
||||
|
/// <param name="maximum">The maximum coded sample.</param>
|
||||
|
/// <returns>The rounded and clipped half samples.</returns>
|
||||
|
public static abstract Vector512<int> Interpolate(Vector512<int> a, Vector512<int> b, Vector512<int> c, Vector512<int> d, int maximum); |
||||
|
} |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Inserts half samples before each original edge sample.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The edge with writable prefix samples at -2 and -1 and room for the doubled extent.</param>
|
||||
|
/// <param name="count">The number of original edge samples, at most <see cref="MaximumCount"/>.</param>
|
||||
|
/// <param name="scratch">The original-sample workspace with at least <see cref="ScratchLength"/> samples.</param>
|
||||
|
public static void Apply(Span<byte> edge, int count, Span<byte> scratch) |
||||
|
=> Upsampler<FourTapOperator>.Apply(edge, count, scratch); |
||||
|
|
||||
|
/// <summary>
|
||||
|
/// Inserts half samples before each original edge sample.
|
||||
|
/// </summary>
|
||||
|
/// <param name="edge">The edge with writable prefix samples at -2 and -1 and room for the doubled extent.</param>
|
||||
|
/// <param name="count">The number of original edge samples, at most <see cref="MaximumCount"/>.</param>
|
||||
|
/// <param name="bitDepth">The coded precision used to clamp interpolation.</param>
|
||||
|
/// <param name="scratch">The original-sample workspace with at least <see cref="ScratchLength"/> samples.</param>
|
||||
|
public static void Apply(Span<short> edge, int count, int bitDepth, Span<short> scratch) |
||||
|
=> Upsampler<FourTapOperator>.Apply(edge, count, (1 << bitDepth) - 1, scratch); |
||||
|
} |
||||
Loading…
Reference in new issue