@ -65,6 +65,39 @@ internal static partial class Av1CompoundInterPredictor
scratch , scratch ,
useSimd : true ) ; useSimd : true ) ;
/// <summary>
/// Reconstructs one high-bit-depth translational reference into AV1's unsigned compound intermediate format.
/// </summary>
public static void PredictCompound (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
Av1InterpolationFilter horizontalFilter ,
Av1InterpolationFilter verticalFilter ,
int horizontalPhase ,
int verticalPhase ,
int bitDepth ,
Span < short > scratch )
= > PredictCompound < CompoundPredictionOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
horizontalFilter ,
verticalFilter ,
horizontalPhase ,
verticalPhase ,
bitDepth ,
scratch ,
useSimd : true ) ;
/// <summary>
/// <summary>
/// Executes one closed compound-prediction conversion operator.
/// Executes one closed compound-prediction conversion operator.
/// </summary>
/// </summary>
@ -175,6 +208,124 @@ internal static partial class Av1CompoundInterPredictor
useSimd ) ; useSimd ) ;
} }
/// <summary>
/// Executes one closed high-bit-depth compound-prediction conversion operator.
/// </summary>
/// <typeparam name="TOperator">The compound-prediction conversion operator.</typeparam>
private static void PredictCompound < TOperator > (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
Av1InterpolationFilter horizontalFilter ,
Av1InterpolationFilter verticalFilter ,
int horizontalPhase ,
int verticalPhase ,
int bitDepth ,
Span < short > scratch ,
bool useSimd )
where TOperator : struct , IAv1CompoundPredictionOperator
{
ReadOnlySpan < short > horizontalCoefficients = GetCompoundCoefficients ( horizontalFilter , horizontalPhase , width < = 4 ) ;
ReadOnlySpan < short > verticalCoefficients = GetCompoundCoefficients ( verticalFilter , verticalPhase , height < = 4 ) ;
// Libaom increases the first-round shift only for 12-bit input. This keeps the signed horizontal
// intermediate within sixteen bits while preserving the same total Q14 convolution precision.
int intermediateRange = bitDepth + FilterBits - Round0Bits + 2 ;
int round0 = Round0Bits + Math . Max ( intermediateRange - 1 6 , 0 ) ;
int roundBits = ( 2 * FilterBits ) - round0 - CompoundRound1Bits ;
int offsetBits = bitDepth + ( 2 * FilterBits ) - round0 ;
int roundOffset = ( 1 < < ( offsetBits - CompoundRound1Bits ) ) +
( 1 < < ( offsetBits - CompoundRound1Bits - 1 ) ) ;
if ( horizontalPhase = = 0 & & verticalPhase = = 0 )
{
CopyCompound < TOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
roundBits ,
roundOffset ,
useSimd ) ;
return ;
}
if ( verticalPhase = = 0 )
{
GetEffectiveKernel ( horizontalCoefficients , out int firstCoefficient , out int tapCount ) ;
FilterCompoundDirect < TOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
horizontalCoefficients [ firstCoefficient . . ] ,
tapCount ,
firstCoefficient - 3 ,
tapStride : 1 ,
preShift : 0 ,
round : round0 ,
roundOffset ,
useSimd ) ;
return ;
}
if ( horizontalPhase = = 0 )
{
GetEffectiveKernel ( verticalCoefficients , out int firstCoefficient , out int tapCount ) ;
FilterCompoundDirect < TOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
verticalCoefficients [ firstCoefficient . . ] ,
tapCount ,
( firstCoefficient - 3 ) * sourceStride ,
sourceStride ,
FilterBits - round0 ,
CompoundRound1Bits ,
roundOffset ,
useSimd ) ;
return ;
}
GetEffectiveKernel ( horizontalCoefficients , out int firstHorizontalCoefficient , out int horizontalTapCount ) ;
GetEffectiveKernel ( verticalCoefficients , out int firstVerticalCoefficient , out int verticalTapCount ) ;
FilterCompound2D < TOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
horizontalCoefficients [ firstHorizontalCoefficient . . ] ,
horizontalTapCount ,
firstHorizontalCoefficient - 3 ,
verticalCoefficients [ firstVerticalCoefficient . . ] ,
verticalTapCount ,
firstVerticalCoefficient - 3 ,
bitDepth ,
round0 ,
scratch ,
useSimd ) ;
}
/// <summary>
/// <summary>
/// Reconstructs one compound intermediate without explicit hardware intrinsics.
/// Reconstructs one compound intermediate without explicit hardware intrinsics.
/// </summary>
/// </summary>
@ -206,6 +357,39 @@ internal static partial class Av1CompoundInterPredictor
scratch , scratch ,
useSimd : false ) ; useSimd : false ) ;
/// <summary>
/// Reconstructs one high-bit-depth compound intermediate without explicit hardware intrinsics.
/// </summary>
public static void PredictCompoundScalar (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
Av1InterpolationFilter horizontalFilter ,
Av1InterpolationFilter verticalFilter ,
int horizontalPhase ,
int verticalPhase ,
int bitDepth ,
Span < short > scratch )
= > PredictCompound < CompoundPredictionOperator > (
source ,
sourceStride ,
sourceOrigin ,
destination ,
destinationStride ,
width ,
height ,
horizontalFilter ,
verticalFilter ,
horizontalPhase ,
verticalPhase ,
bitDepth ,
scratch ,
useSimd : false ) ;
/// <summary>
/// <summary>
/// Copies integer-position samples through one closed compound-prediction operator.
/// Copies integer-position samples through one closed compound-prediction operator.
/// </summary>
/// </summary>
@ -275,6 +459,75 @@ internal static partial class Av1CompoundInterPredictor
} }
} }
/// <summary>
/// Copies high-bit-depth integer-position samples through one closed compound-prediction operator.
/// </summary>
/// <typeparam name="TOperator">The compound-prediction conversion operator.</typeparam>
private static void CopyCompound < TOperator > (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
int roundBits ,
int roundOffset ,
bool useSimd )
where TOperator : struct , IAv1CompoundPredictionOperator
{
ref ushort sourceBase = ref Unsafe . Add ( ref MemoryMarshal . GetReference ( source ) , sourceOrigin ) ;
ref ushort destinationBase = ref MemoryMarshal . GetReference ( destination ) ;
for ( int row = 0 ; row < height ; row + + )
{
ref ushort sourceRow = ref Unsafe . Add ( ref sourceBase , row * sourceStride ) ;
ref ushort destinationRow = ref Unsafe . Add ( ref destinationBase , row * destinationStride ) ;
int column = 0 ;
if ( useSimd & & Vector512 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector512 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector512 < ushort > . Count )
{
Vector512 < ushort > samples = Vector512 . LoadUnsafe ( ref sourceRow , ( nuint ) column ) ;
TOperator . CopyHighBitDepth ( samples , roundBits , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector256 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector256 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector256 < ushort > . Count )
{
Vector256 < ushort > samples = Vector256 . LoadUnsafe ( ref sourceRow , ( nuint ) column ) ;
TOperator . CopyHighBitDepth ( samples , roundBits , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector128 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector128 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector128 < ushort > . Count )
{
Vector128 < ushort > samples = Vector128 . LoadUnsafe ( ref sourceRow , ( nuint ) column ) ;
TOperator . CopyHighBitDepth ( samples , roundBits , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
for ( ; column < width ; column + + )
{
Unsafe . Add ( ref destinationRow , column ) = TOperator . CopyHighBitDepth (
Unsafe . Add ( ref sourceRow , column ) ,
roundBits ,
roundOffset ) ;
}
}
}
/// <summary>
/// <summary>
/// Applies one compound convolution direction through one closed conversion operator.
/// Applies one compound convolution direction through one closed conversion operator.
/// </summary>
/// </summary>
@ -395,6 +648,112 @@ internal static partial class Av1CompoundInterPredictor
} }
} }
/// <summary>
/// Applies one high-bit-depth compound convolution direction through one closed conversion operator.
/// </summary>
/// <typeparam name="TOperator">The compound-prediction conversion operator.</typeparam>
private static void FilterCompoundDirect < TOperator > (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
ReadOnlySpan < short > coefficients ,
int tapCount ,
int sourceOffset ,
int tapStride ,
int preShift ,
int round ,
int roundOffset ,
bool useSimd )
where TOperator : struct , IAv1CompoundPredictionOperator
{
ref ushort sourceBase = ref Unsafe . Add ( ref MemoryMarshal . GetReference ( source ) , sourceOrigin ) ;
ref ushort destinationBase = ref MemoryMarshal . GetReference ( destination ) ;
ref short coefficientBase = ref MemoryMarshal . GetReference ( coefficients ) ;
for ( int row = 0 ; row < height ; row + + )
{
ref ushort sourceRowUnsigned = ref Unsafe . Add ( ref sourceBase , ( row * sourceStride ) + sourceOffset ) ;
ref short sourceRow = ref Unsafe . As < ushort , short > ( ref sourceRowUnsigned ) ;
ref ushort destinationRow = ref Unsafe . Add ( ref destinationBase , row * destinationStride ) ;
int column = 0 ;
if ( useSimd & & Vector512 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector512 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector512 < ushort > . Count )
{
Convolve (
ref sourceRow ,
tapStride ,
( nuint ) column ,
ref coefficientBase ,
tapCount ,
Vector512 < int > . Zero ,
out Vector512 < int > lower ,
out Vector512 < int > upper ) ;
TOperator . PrepareDirect ( lower , upper , preShift , round , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector256 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector256 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector256 < ushort > . Count )
{
Convolve (
ref sourceRow ,
tapStride ,
( nuint ) column ,
ref coefficientBase ,
tapCount ,
Vector256 < int > . Zero ,
out Vector256 < int > lower ,
out Vector256 < int > upper ) ;
TOperator . PrepareDirect ( lower , upper , preShift , round , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector128 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector128 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector128 < ushort > . Count )
{
Convolve (
ref sourceRow ,
tapStride ,
( nuint ) column ,
ref coefficientBase ,
tapCount ,
Vector128 < int > . Zero ,
out Vector128 < int > lower ,
out Vector128 < int > upper ) ;
TOperator . PrepareDirect ( lower , upper , preShift , round , roundOffset )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
for ( ; column < width ; column + + )
{
int result = ConvolveScalar (
ref Unsafe . Add ( ref sourceRowUnsigned , column ) ,
tapStride ,
ref coefficientBase ,
tapCount ) ;
Unsafe . Add ( ref destinationRow , column ) = TOperator . PrepareDirect ( result , preShift , round , roundOffset ) ;
}
}
}
/// <summary>
/// <summary>
/// Applies separable compound convolution through caller-owned signed scratch.
/// Applies separable compound convolution through caller-owned signed scratch.
/// </summary>
/// </summary>
@ -593,6 +952,205 @@ internal static partial class Av1CompoundInterPredictor
} }
} }
/// <summary>
/// Applies separable high-bit-depth compound convolution through caller-owned signed scratch.
/// </summary>
/// <typeparam name="TOperator">The compound-prediction conversion operator.</typeparam>
private static void FilterCompound2D < TOperator > (
ReadOnlySpan < ushort > source ,
int sourceStride ,
int sourceOrigin ,
Span < ushort > destination ,
int destinationStride ,
int width ,
int height ,
ReadOnlySpan < short > horizontalCoefficients ,
int horizontalTapCount ,
int horizontalSourceOffset ,
ReadOnlySpan < short > verticalCoefficients ,
int verticalTapCount ,
int verticalSourceOffset ,
int bitDepth ,
int round0 ,
Span < short > scratch ,
bool useSimd )
where TOperator : struct , IAv1CompoundPredictionOperator
{
ref ushort sourceBase = ref Unsafe . Add ( ref MemoryMarshal . GetReference ( source ) , sourceOrigin ) ;
ref ushort destinationBase = ref MemoryMarshal . GetReference ( destination ) ;
ref short scratchBase = ref MemoryMarshal . GetReference ( scratch ) ;
ref short horizontalCoefficientBase = ref MemoryMarshal . GetReference ( horizontalCoefficients ) ;
ref short verticalCoefficientBase = ref MemoryMarshal . GetReference ( verticalCoefficients ) ;
int scratchStride = Math . Max ( width , MinimumScratchStride ) ;
int intermediateHeight = height + verticalTapCount - 1 ;
int horizontalBias = 1 < < ( bitDepth + FilterBits - 1 ) ;
int verticalBias = 1 < < ( bitDepth + ( 2 * FilterBits ) - round0 ) ;
// High-bit-depth input is still below short.MaxValue. Reinterpreting the source lets the shared signed
// widening kernels apply negative filter coefficients without copying or allocating a conversion buffer.
for ( int row = 0 ; row < intermediateHeight ; row + + )
{
ref ushort sourceRowUnsigned = ref Unsafe . Add (
ref sourceBase ,
( ( row + verticalSourceOffset ) * sourceStride ) + horizontalSourceOffset ) ;
ref short sourceRow = ref Unsafe . As < ushort , short > ( ref sourceRowUnsigned ) ;
ref short scratchRow = ref Unsafe . Add ( ref scratchBase , row * scratchStride ) ;
int column = 0 ;
if ( useSimd & & Vector512 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector512 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector512 < ushort > . Count )
{
Convolve (
ref sourceRow ,
1 ,
( nuint ) column ,
ref horizontalCoefficientBase ,
horizontalTapCount ,
Vector512 < int > . Zero ,
out Vector512 < int > lower ,
out Vector512 < int > upper ) ;
TOperator . PrepareHighBitDepthHorizontal ( lower , upper , horizontalBias , round0 )
. StoreUnsafe ( ref scratchRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector256 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector256 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector256 < ushort > . Count )
{
Convolve (
ref sourceRow ,
1 ,
( nuint ) column ,
ref horizontalCoefficientBase ,
horizontalTapCount ,
Vector256 < int > . Zero ,
out Vector256 < int > lower ,
out Vector256 < int > upper ) ;
TOperator . PrepareHighBitDepthHorizontal ( lower , upper , horizontalBias , round0 )
. StoreUnsafe ( ref scratchRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector128 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector128 < ushort > . Count ;
for ( ; column < = vectorEnd ; column + = Vector128 < ushort > . Count )
{
Convolve (
ref sourceRow ,
1 ,
( nuint ) column ,
ref horizontalCoefficientBase ,
horizontalTapCount ,
Vector128 < int > . Zero ,
out Vector128 < int > lower ,
out Vector128 < int > upper ) ;
TOperator . PrepareHighBitDepthHorizontal ( lower , upper , horizontalBias , round0 )
. StoreUnsafe ( ref scratchRow , ( nuint ) column ) ;
}
}
for ( ; column < width ; column + + )
{
int result = ConvolveScalar (
ref Unsafe . Add ( ref sourceRowUnsigned , column ) ,
1 ,
ref horizontalCoefficientBase ,
horizontalTapCount ) ;
Unsafe . Add ( ref scratchRow , column ) =
TOperator . PrepareHighBitDepthHorizontal ( result , horizontalBias , round0 ) ;
}
}
for ( int row = 0 ; row < height ; row + + )
{
ref short scratchRow = ref Unsafe . Add ( ref scratchBase , row * scratchStride ) ;
ref ushort destinationRow = ref Unsafe . Add ( ref destinationBase , row * destinationStride ) ;
int column = 0 ;
if ( useSimd & & Vector512 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector512 < short > . Count ;
for ( ; column < = vectorEnd ; column + = Vector512 < short > . Count )
{
Convolve (
ref scratchRow ,
scratchStride ,
( nuint ) column ,
ref verticalCoefficientBase ,
verticalTapCount ,
Vector512 < int > . Zero ,
out Vector512 < int > lower ,
out Vector512 < int > upper ) ;
TOperator . PrepareHighBitDepthVertical ( lower , upper , verticalBias )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector256 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector256 < short > . Count ;
for ( ; column < = vectorEnd ; column + = Vector256 < short > . Count )
{
Convolve (
ref scratchRow ,
scratchStride ,
( nuint ) column ,
ref verticalCoefficientBase ,
verticalTapCount ,
Vector256 < int > . Zero ,
out Vector256 < int > lower ,
out Vector256 < int > upper ) ;
TOperator . PrepareHighBitDepthVertical ( lower , upper , verticalBias )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
if ( useSimd & & Vector128 . IsHardwareAccelerated )
{
int vectorEnd = width - Vector128 < short > . Count ;
for ( ; column < = vectorEnd ; column + = Vector128 < short > . Count )
{
Convolve (
ref scratchRow ,
scratchStride ,
( nuint ) column ,
ref verticalCoefficientBase ,
verticalTapCount ,
Vector128 < int > . Zero ,
out Vector128 < int > lower ,
out Vector128 < int > upper ) ;
TOperator . PrepareHighBitDepthVertical ( lower , upper , verticalBias )
. StoreUnsafe ( ref destinationRow , ( nuint ) column ) ;
}
}
for ( ; column < width ; column + + )
{
int result = ConvolveScalar (
ref Unsafe . Add ( ref scratchRow , column ) ,
scratchStride ,
ref verticalCoefficientBase ,
verticalTapCount ) ;
Unsafe . Add ( ref destinationRow , column ) =
TOperator . PrepareHighBitDepthVertical ( result , verticalBias ) ;
}
}
}
/// <summary>
/// <summary>
/// Gets the selected interpolation kernel for compound traversal.
/// Gets the selected interpolation kernel for compound traversal.
/// </summary>
/// </summary>
James Jackson-South
3 days ago
