mirror of https://github.com/SixLabors/ImageSharp
James Jackson-South
2 months ago
10 changed files with 1183 additions and 158 deletions
@ -0,0 +1,429 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
using System.Runtime.Intrinsics.Arm; |
|||
using System.Runtime.Intrinsics.X86; |
|||
using SixLabors.ImageSharp.Common.Helpers; |
|||
using SixLabors.ImageSharp.Formats.Heif.Av1.Transform; |
|||
|
|||
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; |
|||
|
|||
/// <content>
|
|||
/// Provides packed luma subsampling and mean-subtraction operations for chroma-from-luma prediction.
|
|||
/// </content>
|
|||
internal partial class Av1ChromaFromLumaContext |
|||
{ |
|||
/// <summary>
|
|||
/// Stores 8-bit reconstructed luma samples in the Q3 predictor surface.
|
|||
/// </summary>
|
|||
/// <param name="input">The reconstructed luma samples.</param>
|
|||
/// <param name="inputStride">The distance, in samples, between input rows.</param>
|
|||
/// <param name="outputOffset">The first destination sample in the fixed-stride predictor buffer.</param>
|
|||
/// <param name="width">The luma width in samples.</param>
|
|||
/// <param name="height">The luma height in samples.</param>
|
|||
private void StoreSamples(ReadOnlySpan<byte> input, int inputStride, int outputOffset, int width, int height) |
|||
{ |
|||
ref byte inputBase = ref MemoryMarshal.GetReference(input); |
|||
ref short outputBase = ref MemoryMarshal.GetArrayDataReference(this.Q3Buffer); |
|||
|
|||
if (!this.subX) |
|||
{ |
|||
// One luma sample maps directly to one chroma sample, so multiplying by eight converts it to Q3.
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
ref byte inputRow = ref Unsafe.Add(ref inputBase, row * inputStride); |
|||
ref short outputRow = ref Unsafe.Add(ref outputBase, outputOffset + (row * BufferLine)); |
|||
int column = 0; |
|||
|
|||
if (Vector256.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector256<byte>.Count; column += Vector256<byte>.Count) |
|||
{ |
|||
(Vector256<ushort> lower, Vector256<ushort> upper) = Vector256.Widen(Vector256.LoadUnsafe(ref inputRow, (nuint)column)); |
|||
|
|||
(lower << 3).AsInt16().StoreUnsafe(ref outputRow, (nuint)column); |
|||
(upper << 3).AsInt16().StoreUnsafe(ref outputRow, (nuint)(column + Vector256<ushort>.Count)); |
|||
} |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector128<byte>.Count; column += Vector128<byte>.Count) |
|||
{ |
|||
(Vector128<ushort> lower, Vector128<ushort> upper) = Vector128.Widen(Vector128.LoadUnsafe(ref inputRow, (nuint)column)); |
|||
|
|||
(lower << 3).AsInt16().StoreUnsafe(ref outputRow, (nuint)column); |
|||
(upper << 3).AsInt16().StoreUnsafe(ref outputRow, (nuint)(column + Vector128<ushort>.Count)); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
ulong packed = width - column == 4 |
|||
? Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref inputRow, column)) |
|||
: Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref inputRow, column)); |
|||
Vector128<short> q3 = (Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte()) << 3).AsInt16(); |
|||
|
|||
if (width - column == 4) |
|||
{ |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref outputRow, column)), q3.AsUInt64().ToScalar()); |
|||
} |
|||
else |
|||
{ |
|||
q3.StoreUnsafe(ref outputRow, (nuint)column); |
|||
} |
|||
|
|||
column = width; |
|||
} |
|||
} |
|||
|
|||
for (; column < width; column++) |
|||
{ |
|||
Unsafe.Add(ref outputRow, column) = (short)(Unsafe.Add(ref inputRow, column) << 3); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
Vector256<sbyte> ones256 = Vector256.Create((sbyte)1); |
|||
Vector128<sbyte> ones128 = Vector128.Create((sbyte)1); |
|||
int rowStep = this.subY ? 2 : 1; |
|||
int outputShift = this.subY ? 1 : 2; |
|||
for (int row = 0; row < height; row += rowStep) |
|||
{ |
|||
ref byte inputRow = ref Unsafe.Add(ref inputBase, row * inputStride); |
|||
ref byte nextInputRow = ref Unsafe.Add(ref inputRow, this.subY ? inputStride : 0); |
|||
ref short outputRow = ref Unsafe.Add(ref outputBase, outputOffset + ((row >> (this.subY ? 1 : 0)) * BufferLine)); |
|||
int column = 0; |
|||
|
|||
if (Avx2.IsSupported) |
|||
{ |
|||
for (; column <= width - Vector256<byte>.Count; column += Vector256<byte>.Count) |
|||
{ |
|||
Vector256<short> sum = Avx2.MultiplyAddAdjacent(Vector256.LoadUnsafe(ref inputRow, (nuint)column), ones256); |
|||
if (this.subY) |
|||
{ |
|||
sum += Avx2.MultiplyAddAdjacent(Vector256.LoadUnsafe(ref nextInputRow, (nuint)column), ones256); |
|||
} |
|||
|
|||
(sum << outputShift).StoreUnsafe(ref outputRow, (nuint)(column >> 1)); |
|||
} |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector128<byte>.Count; column += Vector128<byte>.Count) |
|||
{ |
|||
Vector128<short> sum = PairSum(Vector128.LoadUnsafe(ref inputRow, (nuint)column), ones128); |
|||
if (this.subY) |
|||
{ |
|||
sum += PairSum(Vector128.LoadUnsafe(ref nextInputRow, (nuint)column), ones128); |
|||
} |
|||
|
|||
(sum << outputShift).StoreUnsafe(ref outputRow, (nuint)(column >> 1)); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
int remaining = width - column; |
|||
ulong packed = remaining == 4 |
|||
? Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref inputRow, column)) |
|||
: Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref inputRow, column)); |
|||
Vector128<short> sum = PairSum(Vector128.CreateScalarUnsafe(packed).AsByte(), ones128); |
|||
if (this.subY) |
|||
{ |
|||
packed = remaining == 4 |
|||
? Unsafe.ReadUnaligned<uint>(ref Unsafe.Add(ref nextInputRow, column)) |
|||
: Unsafe.ReadUnaligned<ulong>(ref Unsafe.Add(ref nextInputRow, column)); |
|||
sum += PairSum(Vector128.CreateScalarUnsafe(packed).AsByte(), ones128); |
|||
} |
|||
|
|||
Vector128<short> q3 = sum << outputShift; |
|||
if (remaining == 4) |
|||
{ |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref outputRow, column >> 1)), q3.AsUInt32().ToScalar()); |
|||
} |
|||
else |
|||
{ |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref outputRow, column >> 1)), q3.AsUInt64().ToScalar()); |
|||
} |
|||
|
|||
column = width; |
|||
} |
|||
} |
|||
|
|||
for (; column < width; column += 2) |
|||
{ |
|||
int sum = Unsafe.Add(ref inputRow, column) + Unsafe.Add(ref inputRow, column + 1); |
|||
if (this.subY) |
|||
{ |
|||
sum += Unsafe.Add(ref nextInputRow, column) + Unsafe.Add(ref nextInputRow, column + 1); |
|||
} |
|||
|
|||
Unsafe.Add(ref outputRow, column >> 1) = (short)(sum << outputShift); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Stores high-bit-depth reconstructed luma samples in the Q3 predictor surface.
|
|||
/// </summary>
|
|||
/// <param name="input">The reconstructed luma samples.</param>
|
|||
/// <param name="inputStride">The distance, in samples, between input rows.</param>
|
|||
/// <param name="outputOffset">The first destination sample in the fixed-stride predictor buffer.</param>
|
|||
/// <param name="width">The luma width in samples.</param>
|
|||
/// <param name="height">The luma height in samples.</param>
|
|||
private void StoreSamples(ReadOnlySpan<short> input, int inputStride, int outputOffset, int width, int height) |
|||
{ |
|||
ref short inputBase = ref MemoryMarshal.GetReference(input); |
|||
ref short outputBase = ref MemoryMarshal.GetArrayDataReference(this.Q3Buffer); |
|||
|
|||
if (!this.subX) |
|||
{ |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
ref short inputRow = ref Unsafe.Add(ref inputBase, row * inputStride); |
|||
ref short outputRow = ref Unsafe.Add(ref outputBase, outputOffset + (row * BufferLine)); |
|||
int column = 0; |
|||
|
|||
if (Vector256.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector256<short>.Count; column += Vector256<short>.Count) |
|||
{ |
|||
(Vector256.LoadUnsafe(ref inputRow, (nuint)column) << 3).StoreUnsafe(ref outputRow, (nuint)column); |
|||
} |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
(Vector128.LoadUnsafe(ref inputRow, (nuint)column) << 3).StoreUnsafe(ref outputRow, (nuint)column); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref inputRow, column))); |
|||
Vector128<short> q3 = Vector128.CreateScalarUnsafe(packed).AsInt16() << 3; |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref outputRow, column)), q3.AsUInt64().ToScalar()); |
|||
column += 4; |
|||
} |
|||
} |
|||
|
|||
for (; column < width; column++) |
|||
{ |
|||
Unsafe.Add(ref outputRow, column) = (short)(Unsafe.Add(ref inputRow, column) << 3); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
Vector256<short> ones256 = Vector256.Create((short)1); |
|||
Vector128<short> ones128 = Vector128.Create((short)1); |
|||
int rowStep = this.subY ? 2 : 1; |
|||
int outputShift = this.subY ? 1 : 2; |
|||
for (int row = 0; row < height; row += rowStep) |
|||
{ |
|||
ref short inputRow = ref Unsafe.Add(ref inputBase, row * inputStride); |
|||
ref short nextInputRow = ref Unsafe.Add(ref inputRow, this.subY ? inputStride : 0); |
|||
ref short outputRow = ref Unsafe.Add(ref outputBase, outputOffset + ((row >> (this.subY ? 1 : 0)) * BufferLine)); |
|||
int column = 0; |
|||
|
|||
if (Vector256.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector256<short>.Count; column += Vector256<short>.Count) |
|||
{ |
|||
Vector256<int> sum = Vector256_.MultiplyAddAdjacent(Vector256.LoadUnsafe(ref inputRow, (nuint)column), ones256); |
|||
if (this.subY) |
|||
{ |
|||
sum += Vector256_.MultiplyAddAdjacent(Vector256.LoadUnsafe(ref nextInputRow, (nuint)column), ones256); |
|||
} |
|||
|
|||
Vector256.Narrow(sum << outputShift, Vector256<int>.Zero).GetLower().StoreUnsafe(ref outputRow, (nuint)(column >> 1)); |
|||
} |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
Vector128<int> sum = Vector128_.MultiplyAddAdjacent(Vector128.LoadUnsafe(ref inputRow, (nuint)column), ones128); |
|||
if (this.subY) |
|||
{ |
|||
sum += Vector128_.MultiplyAddAdjacent(Vector128.LoadUnsafe(ref nextInputRow, (nuint)column), ones128); |
|||
} |
|||
|
|||
Vector128.Narrow(sum << outputShift, Vector128<int>.Zero).GetLower().StoreUnsafe(ref outputRow, (nuint)(column >> 1)); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
Vector128<short> samples = Vector128.CreateScalarUnsafe( |
|||
Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref inputRow, column)))).AsInt16(); |
|||
Vector128<int> sum = Vector128_.MultiplyAddAdjacent(samples, ones128); |
|||
if (this.subY) |
|||
{ |
|||
samples = Vector128.CreateScalarUnsafe( |
|||
Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref nextInputRow, column)))).AsInt16(); |
|||
sum += Vector128_.MultiplyAddAdjacent(samples, ones128); |
|||
} |
|||
|
|||
Vector64<short> q3 = Vector128.Narrow(sum << outputShift, Vector128<int>.Zero).GetLower(); |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref outputRow, column >> 1)), q3.AsUInt32().ToScalar()); |
|||
column += 4; |
|||
} |
|||
} |
|||
|
|||
for (; column < width; column += 2) |
|||
{ |
|||
int sum = Unsafe.Add(ref inputRow, column) + Unsafe.Add(ref inputRow, column + 1); |
|||
if (this.subY) |
|||
{ |
|||
sum += Unsafe.Add(ref nextInputRow, column) + Unsafe.Add(ref nextInputRow, column + 1); |
|||
} |
|||
|
|||
Unsafe.Add(ref outputRow, column >> 1) = (short)(sum << outputShift); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Subtracts the rounded Q3 average from each predictor sample, leaving the AC contribution used by CfL.
|
|||
/// </summary>
|
|||
/// <param name="transformSize">The populated predictor dimensions.</param>
|
|||
private void SubtractAverage(Av1TransformSize transformSize) |
|||
{ |
|||
int width = transformSize.GetWidth(); |
|||
int height = transformSize.GetHeight(); |
|||
|
|||
// Transform dimensions are powers of two, so division by the sample count is an exact right shift. Half
|
|||
// the sample count is accumulated first to implement the normative nearest-integer rounding.
|
|||
int sumQ3 = (width * height) >> 1; |
|||
ref short bufferBase = ref MemoryMarshal.GetArrayDataReference(this.Q3Buffer); |
|||
|
|||
if (Vector256.IsHardwareAccelerated && width >= Vector256<short>.Count) |
|||
{ |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
for (int column = 0; column < width; column += Vector256<short>.Count) |
|||
{ |
|||
(Vector256<int> lower, Vector256<int> upper) = Vector256.Widen(Vector256.LoadUnsafe(ref bufferBase, (nuint)(rowOffset + column))); |
|||
|
|||
sumQ3 += Vector256.Sum(lower) + Vector256.Sum(upper); |
|||
} |
|||
} |
|||
} |
|||
else if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
int column = 0; |
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
(Vector128<int> lower, Vector128<int> upper) = Vector128.Widen(Vector128.LoadUnsafe(ref bufferBase, (nuint)(rowOffset + column))); |
|||
|
|||
sumQ3 += Vector128.Sum(lower) + Vector128.Sum(upper); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref bufferBase, rowOffset + column))); |
|||
sumQ3 += Vector128.Sum(Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsInt16())); |
|||
} |
|||
} |
|||
} |
|||
else |
|||
{ |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
for (int column = 0; column < width; column++) |
|||
{ |
|||
sumQ3 += Unsafe.Add(ref bufferBase, rowOffset + column); |
|||
} |
|||
} |
|||
} |
|||
|
|||
int pelCountLog2 = transformSize.GetBlockWidthLog2() + transformSize.GetBlockHeightLog2(); |
|||
short averageQ3 = (short)(sumQ3 >> pelCountLog2); |
|||
|
|||
if (Vector256.IsHardwareAccelerated && width >= Vector256<short>.Count) |
|||
{ |
|||
Vector256<short> average = Vector256.Create(averageQ3); |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
for (int column = 0; column < width; column += Vector256<short>.Count) |
|||
{ |
|||
(Vector256.LoadUnsafe(ref bufferBase, (nuint)(rowOffset + column)) - average).StoreUnsafe(ref bufferBase, (nuint)(rowOffset + column)); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
Vector128<short> average = Vector128.Create(averageQ3); |
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
int column = 0; |
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
(Vector128.LoadUnsafe(ref bufferBase, (nuint)(rowOffset + column)) - average).StoreUnsafe(ref bufferBase, (nuint)(rowOffset + column)); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
ulong packed = Unsafe.ReadUnaligned<ulong>(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref bufferBase, rowOffset + column))); |
|||
Vector128<short> result = Vector128.CreateScalarUnsafe(packed).AsInt16() - average; |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref bufferBase, rowOffset + column)), result.AsUInt64().ToScalar()); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int rowOffset = row * BufferLine; |
|||
for (int column = 0; column < width; column++) |
|||
{ |
|||
Unsafe.Add(ref bufferBase, rowOffset + column) -= averageQ3; |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Adds adjacent 8-bit samples into eight 16-bit lanes.
|
|||
/// </summary>
|
|||
/// <param name="samples">The packed source samples.</param>
|
|||
/// <param name="ones">The multiplier used by the x86 adjacent multiply-add instruction.</param>
|
|||
/// <returns>The adjacent pair sums.</returns>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<short> PairSum(Vector128<byte> samples, Vector128<sbyte> ones) |
|||
{ |
|||
if (Ssse3.IsSupported) |
|||
{ |
|||
return Ssse3.MultiplyAddAdjacent(samples, ones); |
|||
} |
|||
|
|||
if (AdvSimd.IsSupported) |
|||
{ |
|||
return AdvSimd.AddPairwiseWidening(samples).AsInt16(); |
|||
} |
|||
|
|||
// WebAssembly has byte shuffles but no pairwise-widening intrinsic. Grouping even and odd bytes before
|
|||
// widening keeps all eight pair additions vectorized without adding a general-purpose utility wrapper.
|
|||
Vector128<byte> even = Vector128.Shuffle(samples, Vector128.Create((byte)0, 2, 4, 6, 8, 10, 12, 14, 255, 255, 255, 255, 255, 255, 255, 255)); |
|||
Vector128<byte> odd = Vector128.Shuffle(samples, Vector128.Create((byte)1, 3, 5, 7, 9, 11, 13, 15, 255, 255, 255, 255, 255, 255, 255, 255)); |
|||
return (Vector128.WidenLower(even) + Vector128.WidenLower(odd)).AsInt16(); |
|||
} |
|||
} |
|||
@ -0,0 +1,231 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Runtime.Intrinsics; |
|||
using System.Runtime.Intrinsics.Arm; |
|||
using System.Runtime.Intrinsics.X86; |
|||
|
|||
namespace SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; |
|||
|
|||
/// <summary>
|
|||
/// Applies an AV1 chroma-from-luma residual to a DC-predicted chroma block.
|
|||
/// </summary>
|
|||
internal static class Av1ChromaFromLumaPredictor |
|||
{ |
|||
/// <summary>
|
|||
/// The fixed row stride of the AV1 chroma-from-luma scratch buffer.
|
|||
/// </summary>
|
|||
private const int BufferLine = 32; |
|||
|
|||
/// <summary>
|
|||
/// Applies chroma-from-luma prediction to an 8-bit chroma block.
|
|||
/// </summary>
|
|||
/// <param name="lumaQ3">The zero-mean Q3 luma surface.</param>
|
|||
/// <param name="destination">The DC-predicted chroma block that receives the luma adjustment.</param>
|
|||
/// <param name="destinationStride">The distance, in samples, between destination rows.</param>
|
|||
/// <param name="alphaQ3">The signed Q3 chroma scaling factor.</param>
|
|||
/// <param name="width">The block width in samples.</param>
|
|||
/// <param name="height">The block height in samples.</param>
|
|||
public static void Predict(ReadOnlySpan<short> lumaQ3, Span<byte> destination, int destinationStride, int alphaQ3, int width, int height) |
|||
{ |
|||
ref short lumaBase = ref MemoryMarshal.GetReference(lumaQ3); |
|||
ref byte destinationBase = ref MemoryMarshal.GetReference(destination); |
|||
short dc = destinationBase; |
|||
|
|||
// CfL always follows DC prediction, so the first sample is the common base value for every lane. This
|
|||
// mirrors libaom and avoids loading a block that is known to contain a single repeated prediction value.
|
|||
if (Avx2.IsSupported && width >= Vector256<short>.Count) |
|||
{ |
|||
Vector256<short> alphaSign = Vector256.Create((short)alphaQ3); |
|||
Vector256<short> alphaQ12 = Vector256.Create((short)(Math.Abs(alphaQ3) << 9)); |
|||
Vector256<short> dcVector = Vector256.Create(dc); |
|||
Vector256<short> maximum = Vector256.Create((short)byte.MaxValue); |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
|
|||
for (int column = 0; column < width; column += Vector256<short>.Count) |
|||
{ |
|||
Vector256<short> prediction = Predict(Vector256.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector); |
|||
prediction = Vector256.Clamp(prediction, Vector256<short>.Zero, maximum); |
|||
Vector256.Narrow(prediction.AsUInt16(), Vector256<ushort>.Zero).GetLower().StoreUnsafe(ref destinationBase, (nuint)(destinationRowOffset + column)); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
Vector128<short> alphaSign = Vector128.Create((short)alphaQ3); |
|||
Vector128<short> alphaQ12 = Vector128.Create((short)(Math.Abs(alphaQ3) << 9)); |
|||
Vector128<short> dcVector = Vector128.Create(dc); |
|||
Vector128<short> maximum = Vector128.Create((short)byte.MaxValue); |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
int column = 0; |
|||
|
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
Vector128<short> prediction = Predict(Vector128.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector, alphaQ3); |
|||
prediction = Vector128.Clamp(prediction, Vector128<short>.Zero, maximum); |
|||
Vector128.Narrow(prediction.AsUInt16(), Vector128<ushort>.Zero).GetLower().StoreUnsafe(ref destinationBase, (nuint)(destinationRowOffset + column)); |
|||
} |
|||
|
|||
// Four-wide CfL blocks still have a complete padded scratch row, so reading eight residuals is
|
|||
// valid. Only the four active predictions are stored to the image buffer.
|
|||
if (column < width) |
|||
{ |
|||
Vector128<short> prediction = Predict(Vector128.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector, alphaQ3); |
|||
prediction = Vector128.Clamp(prediction, Vector128<short>.Zero, maximum); |
|||
Vector64<byte> packed = Vector128.Narrow(prediction.AsUInt16(), Vector128<ushort>.Zero).GetLower(); |
|||
Unsafe.WriteUnaligned(ref Unsafe.Add(ref destinationBase, destinationRowOffset + column), packed.AsUInt32().ToScalar()); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
for (int column = 0; column < width; column++) |
|||
{ |
|||
int scaledLumaQ0 = Av1Math.RoundPowerOf2Signed(alphaQ3 * Unsafe.Add(ref lumaBase, lumaRowOffset + column), 6); |
|||
Unsafe.Add(ref destinationBase, destinationRowOffset + column) = (byte)Math.Clamp(dc + scaledLumaQ0, byte.MinValue, byte.MaxValue); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Applies chroma-from-luma prediction to a high-bit-depth chroma block.
|
|||
/// </summary>
|
|||
/// <param name="lumaQ3">The zero-mean Q3 luma surface.</param>
|
|||
/// <param name="destination">The DC-predicted chroma block that receives the luma adjustment.</param>
|
|||
/// <param name="destinationStride">The distance, in samples, between destination rows.</param>
|
|||
/// <param name="alphaQ3">The signed Q3 chroma scaling factor.</param>
|
|||
/// <param name="bitDepth">The number of bits used to represent each sample.</param>
|
|||
/// <param name="width">The block width in samples.</param>
|
|||
/// <param name="height">The block height in samples.</param>
|
|||
public static void Predict(ReadOnlySpan<short> lumaQ3, Span<short> destination, int destinationStride, int alphaQ3, int bitDepth, int width, int height) |
|||
{ |
|||
ref short lumaBase = ref MemoryMarshal.GetReference(lumaQ3); |
|||
ref short destinationBase = ref MemoryMarshal.GetReference(destination); |
|||
short dc = destinationBase; |
|||
short maximum = (short)((1 << bitDepth) - 1); |
|||
|
|||
if (Avx2.IsSupported && width >= Vector256<short>.Count) |
|||
{ |
|||
Vector256<short> alphaSign = Vector256.Create((short)alphaQ3); |
|||
Vector256<short> alphaQ12 = Vector256.Create((short)(Math.Abs(alphaQ3) << 9)); |
|||
Vector256<short> dcVector = Vector256.Create(dc); |
|||
Vector256<short> maximumVector = Vector256.Create(maximum); |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
|
|||
for (int column = 0; column < width; column += Vector256<short>.Count) |
|||
{ |
|||
Vector256<short> prediction = Predict(Vector256.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector); |
|||
Vector256.Clamp(prediction, Vector256<short>.Zero, maximumVector).StoreUnsafe(ref destinationBase, (nuint)(destinationRowOffset + column)); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
if (Vector128.IsHardwareAccelerated) |
|||
{ |
|||
Vector128<short> alphaSign = Vector128.Create((short)alphaQ3); |
|||
Vector128<short> alphaQ12 = Vector128.Create((short)(Math.Abs(alphaQ3) << 9)); |
|||
Vector128<short> dcVector = Vector128.Create(dc); |
|||
Vector128<short> maximumVector = Vector128.Create(maximum); |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
int column = 0; |
|||
|
|||
for (; column <= width - Vector128<short>.Count; column += Vector128<short>.Count) |
|||
{ |
|||
Vector128<short> prediction = Predict(Vector128.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector, alphaQ3); |
|||
Vector128.Clamp(prediction, Vector128<short>.Zero, maximumVector).StoreUnsafe(ref destinationBase, (nuint)(destinationRowOffset + column)); |
|||
} |
|||
|
|||
if (column < width) |
|||
{ |
|||
Vector128<short> prediction = Predict(Vector128.LoadUnsafe(ref lumaBase, (nuint)(lumaRowOffset + column)), alphaSign, alphaQ12, dcVector, alphaQ3); |
|||
prediction = Vector128.Clamp(prediction, Vector128<short>.Zero, maximumVector); |
|||
Unsafe.WriteUnaligned(ref Unsafe.As<short, byte>(ref Unsafe.Add(ref destinationBase, destinationRowOffset + column)), prediction.AsUInt64().ToScalar()); |
|||
} |
|||
} |
|||
|
|||
return; |
|||
} |
|||
|
|||
for (int row = 0; row < height; row++) |
|||
{ |
|||
int lumaRowOffset = row * BufferLine; |
|||
int destinationRowOffset = row * destinationStride; |
|||
for (int column = 0; column < width; column++) |
|||
{ |
|||
int scaledLumaQ0 = Av1Math.RoundPowerOf2Signed(alphaQ3 * Unsafe.Add(ref lumaBase, lumaRowOffset + column), 6); |
|||
Unsafe.Add(ref destinationBase, destinationRowOffset + column) = (short)Math.Clamp(dc + scaledLumaQ0, 0, maximum); |
|||
} |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Calculates sixteen chroma predictions using the packed Q3 arithmetic defined by AV1.
|
|||
/// </summary>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector256<short> Predict(Vector256<short> lumaQ3, Vector256<short> alphaSign, Vector256<short> alphaQ12, Vector256<short> dc) |
|||
{ |
|||
Vector256<short> scaledLumaQ0 = Avx2.MultiplyHighRoundScale(Avx2.Abs(lumaQ3).AsInt16(), alphaQ12); |
|||
Vector256<short> signMask = (lumaQ3 ^ alphaSign) >> 15; |
|||
return ((scaledLumaQ0 ^ signMask) - signMask) + dc; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Calculates eight chroma predictions using the packed Q3 arithmetic defined by AV1.
|
|||
/// </summary>
|
|||
[MethodImpl(MethodImplOptions.AggressiveInlining)] |
|||
private static Vector128<short> Predict(Vector128<short> lumaQ3, Vector128<short> alphaSign, Vector128<short> alphaQ12, Vector128<short> dc, int alphaQ3) |
|||
{ |
|||
Vector128<short> scaledLumaQ0; |
|||
if (Ssse3.IsSupported) |
|||
{ |
|||
scaledLumaQ0 = Ssse3.MultiplyHighRoundScale(Ssse3.Abs(lumaQ3).AsInt16(), alphaQ12); |
|||
} |
|||
else if (AdvSimd.IsSupported) |
|||
{ |
|||
scaledLumaQ0 = AdvSimd.MultiplyRoundedDoublingSaturateHigh(Vector128.Abs(lumaQ3), alphaQ12); |
|||
} |
|||
else |
|||
{ |
|||
// WebAssembly and other Vector128 targets do not expose packed rounded-high multiply. Widening retains
|
|||
// SIMD traversal while reproducing the same signed nearest-integer result in ordinary integer lanes.
|
|||
(Vector128<int> lower, Vector128<int> upper) = Vector128.Widen(lumaQ3); |
|||
Vector128<int> alpha = Vector128.Create(alphaQ3); |
|||
lower *= alpha; |
|||
upper *= alpha; |
|||
lower = (lower + Vector128.Create(32) + (lower >> 31)) >> 6; |
|||
upper = (upper + Vector128.Create(32) + (upper >> 31)) >> 6; |
|||
return Vector128.Narrow(lower, upper) + dc; |
|||
} |
|||
|
|||
Vector128<short> signMask = (lumaQ3 ^ alphaSign) >> 15; |
|||
return ((scaledLumaQ0 ^ signMask) - signMask) + dc; |
|||
} |
|||
} |
|||
@ -0,0 +1,136 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Six Labors Split License.
|
|||
|
|||
using BenchmarkDotNet.Attributes; |
|||
using BenchmarkDotNet.Columns; |
|||
using BenchmarkDotNet.Configs; |
|||
using BenchmarkDotNet.Jobs; |
|||
using SixLabors.ImageSharp.Formats.Heif.Av1.Prediction.ChromaFromLuma; |
|||
|
|||
namespace SixLabors.ImageSharp.Benchmarks.Codecs.Heif; |
|||
|
|||
/// <summary>
|
|||
/// Measures frame-wide AV1 chroma-from-luma prediction at each available intrinsic tier.
|
|||
/// </summary>
|
|||
[Config(typeof(Configuration))] |
|||
[MemoryDiagnoser(displayGenColumns: false)] |
|||
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] |
|||
[CategoriesColumn] |
|||
public class Av1ChromaFromLumaBenchmarks |
|||
{ |
|||
/// <summary>
|
|||
/// The coded frame width, which is an exact multiple of the maximum CfL block side.
|
|||
/// </summary>
|
|||
private const int Width = 1920; |
|||
|
|||
/// <summary>
|
|||
/// The coded frame height including the final padded coding-tree row for a 1080-line presentation.
|
|||
/// </summary>
|
|||
private const int Height = 1088; |
|||
|
|||
/// <summary>
|
|||
/// The maximum CfL block side in chroma samples.
|
|||
/// </summary>
|
|||
private const int BlockSize = 32; |
|||
|
|||
/// <summary>
|
|||
/// The fixed-stride Q3 luma residual surface reused by each benchmark block.
|
|||
/// </summary>
|
|||
private readonly short[] lumaQ3 = new short[BlockSize * BlockSize]; |
|||
|
|||
/// <summary>
|
|||
/// The frame-wide 8-bit DC prediction surface.
|
|||
/// </summary>
|
|||
private readonly byte[] destination8 = new byte[Width * Height]; |
|||
|
|||
/// <summary>
|
|||
/// The frame-wide 12-bit DC prediction surface.
|
|||
/// </summary>
|
|||
private readonly short[] destination12 = new short[Width * Height]; |
|||
|
|||
/// <summary>
|
|||
/// Populates deterministic Q3 residuals and DC predictions outside the measured traversal.
|
|||
/// </summary>
|
|||
[GlobalSetup] |
|||
public void Setup() |
|||
{ |
|||
for (int index = 0; index < this.lumaQ3.Length; index++) |
|||
{ |
|||
this.lumaQ3[index] = (short)(((index * 4051) % 65521) - 32760); |
|||
} |
|||
|
|||
this.destination8.AsSpan().Fill(137); |
|||
this.destination12.AsSpan().Fill(2101); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Measures frame-wide 8-bit chroma-from-luma prediction.
|
|||
/// </summary>
|
|||
/// <returns>The final reconstructed sample, keeping the frame output observable.</returns>
|
|||
[Benchmark] |
|||
[BenchmarkCategory("8Bit")] |
|||
public byte Predict8BitFrame() |
|||
{ |
|||
for (int row = 0; row < Height; row += BlockSize) |
|||
{ |
|||
for (int column = 0; column < Width; column += BlockSize) |
|||
{ |
|||
Av1ChromaFromLumaPredictor.Predict(this.lumaQ3, this.destination8.AsSpan((row * Width) + column), Width, 11, BlockSize, BlockSize); |
|||
} |
|||
} |
|||
|
|||
return this.destination8[^1]; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Measures frame-wide 12-bit chroma-from-luma prediction.
|
|||
/// </summary>
|
|||
/// <returns>The final reconstructed sample, keeping the frame output observable.</returns>
|
|||
[Benchmark] |
|||
[BenchmarkCategory("12Bit")] |
|||
public short Predict12BitFrame() |
|||
{ |
|||
for (int row = 0; row < Height; row += BlockSize) |
|||
{ |
|||
for (int column = 0; column < Width; column += BlockSize) |
|||
{ |
|||
Av1ChromaFromLumaPredictor.Predict(this.lumaQ3, this.destination12.AsSpan((row * Width) + column), Width, 11, 12, BlockSize, BlockSize); |
|||
} |
|||
} |
|||
|
|||
return this.destination12[^1]; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Configures production-process measurements for the default, AVX2, Vector128, and scalar paths.
|
|||
/// </summary>
|
|||
public sealed class Configuration : ManualConfig |
|||
{ |
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="Configuration"/> class.
|
|||
/// </summary>
|
|||
public Configuration() |
|||
{ |
|||
this.AddJob( |
|||
Job.ShortRun |
|||
.WithId("Hardware") |
|||
.AsBaseline()); |
|||
|
|||
this.AddJob( |
|||
Job.ShortRun |
|||
.WithId("Avx2") |
|||
.WithEnvironmentVariable("DOTNET_EnableAVX512F", "0")); |
|||
|
|||
this.AddJob( |
|||
Job.ShortRun |
|||
.WithId("Vector128") |
|||
.WithEnvironmentVariable("DOTNET_EnableAVX512F", "0") |
|||
.WithEnvironmentVariable("DOTNET_EnableAVX2", "0")); |
|||
|
|||
this.AddJob( |
|||
Job.ShortRun |
|||
.WithId("Scalar") |
|||
.WithEnvironmentVariable("DOTNET_EnableHWIntrinsic", "0")); |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue