diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs
deleted file mode 100644
index 2a51b57c23..0000000000
--- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1ByteInverseTransformOutputOperator.cs
+++ /dev/null
@@ -1,49 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
-
-///
-/// Reconstructs eight-bit AV1 samples from predicted values and inverse-transform residuals.
-///
-internal readonly struct Av1ByteInverseTransformOutputOperator : IAv1InverseTransformOutputOperator
-{
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static byte Add(byte prediction, int residual, int bitDepth)
- {
- _ = bitDepth;
- return (byte)Math.Clamp(prediction + residual, byte.MinValue, byte.MaxValue);
- }
-
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ref byte prediction, ref byte destination, Vector128 residual, int bitDepth)
- {
- uint packed = Unsafe.ReadUnaligned(ref prediction);
- Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
- Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32();
- Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue));
- Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero);
- Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero);
- Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt32().ToScalar());
- _ = bitDepth;
- }
-
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ref byte prediction, ref byte destination, Vector256 residual, int bitDepth)
- {
- ulong packed = Unsafe.ReadUnaligned(ref prediction);
- Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
- Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32();
- Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue));
- Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32());
- Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero);
- Unsafe.WriteUnaligned(ref destination, reconstructed8.AsUInt64().ToScalar());
- _ = bitDepth;
- }
-}
diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs
deleted file mode 100644
index d171133928..0000000000
--- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1HighBitDepthInverseTransformOutputOperator.cs
+++ /dev/null
@@ -1,40 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Six Labors Split License.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.Intrinsics;
-using SixLabors.ImageSharp.Common.Helpers;
-
-namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
-
-///
-/// Reconstructs high-bit-depth AV1 samples from predicted values and inverse-transform residuals.
-///
-internal readonly struct Av1HighBitDepthInverseTransformOutputOperator : IAv1InverseTransformOutputOperator
-{
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static short Add(short prediction, int residual, int bitDepth)
- => (short)Math.Clamp(prediction + residual, 0, (1 << bitDepth) - 1);
-
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ref short prediction, ref short destination, Vector128 residual, int bitDepth)
- {
- ulong packed = Unsafe.ReadUnaligned(ref Unsafe.As(ref prediction));
- Vector128 predicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsInt16());
- Vector128 reconstructed = Vector128.Clamp(predicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1));
- Vector128 narrowed = Vector128.Narrow(reconstructed, Vector128.Zero);
- Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar());
- }
-
- ///
- [MethodImpl(MethodImplOptions.AggressiveInlining)]
- public static void Add(ref short prediction, ref short destination, Vector256 residual, int bitDepth)
- {
- Vector256 predicted = Vector256_.Widen(Vector128.LoadUnsafe(ref prediction));
- Vector256 reconstructed = Vector256.Clamp(predicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1));
- Vector128 narrowed = Vector128.Narrow(reconstructed.GetLower(), reconstructed.GetUpper());
- narrowed.StoreUnsafe(ref destination);
- }
-}
diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs
index ac2e23b1ed..ca54711f83 100644
--- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs
+++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1Inverse2dTransformer.cs
@@ -33,7 +33,7 @@ internal static class Av1Inverse2dTransformer
ref Av1Transform2dFlipConfiguration config,
Span workspace,
int bitDepth)
- => Transform2dAdd(
+ => Transform2dAdd>(
input,
outputForRead,
strideForRead,
@@ -61,7 +61,7 @@ internal static class Av1Inverse2dTransformer
int strideForWrite,
ref Av1Transform2dFlipConfiguration config,
Span workspace)
- => Transform2dAdd(
+ => Transform2dAdd>(
input,
outputForRead,
strideForRead,
diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs
new file mode 100644
index 0000000000..d121ff0fb4
--- /dev/null
+++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseTransformOutputOperator.cs
@@ -0,0 +1,85 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.Intrinsics;
+using SixLabors.ImageSharp.Common.Helpers;
+
+namespace SixLabors.ImageSharp.Formats.Heif.Av1.Transform;
+
+///
+/// Reconstructs AV1 samples from predicted values and inverse-transform residuals.
+///
+/// The decoded sample storage type.
+internal readonly struct Av1InverseTransformOutputOperator : IAv1InverseTransformOutputOperator
+ where TSample : unmanaged
+{
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static TSample Add(TSample prediction, int residual, int bitDepth)
+ {
+ // TSample is fixed by the byte and short decoder entry points. The JIT removes this type test from each
+ // closed transform so storage selection does not introduce a branch in the reconstruction loop.
+ if (typeof(TSample) == typeof(byte))
+ {
+ byte value = (byte)Math.Clamp(Unsafe.As(ref prediction) + residual, byte.MinValue, byte.MaxValue);
+ return Unsafe.As(ref value);
+ }
+
+ short result = (short)Math.Clamp(Unsafe.As(ref prediction) + residual, 0, (1 << bitDepth) - 1);
+ return Unsafe.As(ref result);
+ }
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Add(ref TSample prediction, ref TSample destination, Vector128 residual, int bitDepth)
+ {
+ if (typeof(TSample) == typeof(byte))
+ {
+ ref byte source = ref Unsafe.As(ref prediction);
+ uint packed = Unsafe.ReadUnaligned(ref source);
+ Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
+ Vector128 predicted32 = Vector128.WidenLower(predicted16).AsInt32();
+ Vector128 reconstructed = Vector128.Clamp(predicted32 + residual, Vector128.Zero, Vector128.Create((int)byte.MaxValue));
+ Vector128 reconstructed16 = Vector128.Narrow(reconstructed.AsUInt32(), Vector128.Zero);
+ Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero);
+ Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt32().ToScalar());
+ return;
+ }
+
+ ref short highBitDepthSource = ref Unsafe.As(ref prediction);
+ ulong highBitDepthPacked = Unsafe.ReadUnaligned(ref Unsafe.As(ref highBitDepthSource));
+ Vector128 highBitDepthPredicted = Vector128.WidenLower(Vector128.CreateScalarUnsafe(highBitDepthPacked).AsInt16());
+ Vector128 highBitDepthReconstructed =
+ Vector128.Clamp(highBitDepthPredicted + residual, Vector128.Zero, Vector128.Create((1 << bitDepth) - 1));
+
+ Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed, Vector128.Zero);
+ Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), narrowed.AsUInt64().ToScalar());
+ }
+
+ ///
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static void Add(ref TSample prediction, ref TSample destination, Vector256 residual, int bitDepth)
+ {
+ if (typeof(TSample) == typeof(byte))
+ {
+ ref byte source = ref Unsafe.As(ref prediction);
+ ulong packed = Unsafe.ReadUnaligned(ref source);
+ Vector128 predicted16 = Vector128.WidenLower(Vector128.CreateScalarUnsafe(packed).AsByte());
+ Vector256 predicted32 = Vector256.Create(Vector128.WidenLower(predicted16), Vector128.WidenUpper(predicted16)).AsInt32();
+ Vector256 reconstructed = Vector256.Clamp(predicted32 + residual, Vector256.Zero, Vector256.Create((int)byte.MaxValue));
+ Vector128 reconstructed16 = Vector128.Narrow(reconstructed.GetLower().AsUInt32(), reconstructed.GetUpper().AsUInt32());
+ Vector128 reconstructed8 = Vector128.Narrow(reconstructed16, Vector128.Zero);
+ Unsafe.WriteUnaligned(ref Unsafe.As(ref destination), reconstructed8.AsUInt64().ToScalar());
+ return;
+ }
+
+ ref short highBitDepthSource = ref Unsafe.As(ref prediction);
+ Vector256 highBitDepthPredicted = Vector256_.Widen(Vector128.LoadUnsafe(ref highBitDepthSource));
+ Vector256 highBitDepthReconstructed =
+ Vector256.Clamp(highBitDepthPredicted + residual, Vector256.Zero, Vector256.Create((1 << bitDepth) - 1));
+
+ Vector128 narrowed = Vector128.Narrow(highBitDepthReconstructed.GetLower(), highBitDepthReconstructed.GetUpper());
+ narrowed.StoreUnsafe(ref Unsafe.As(ref destination));
+ }
+}
diff --git a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs
index 05385ef858..4b2306089f 100644
--- a/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs
+++ b/src/ImageSharp/Formats/Heif/Av1/Transform/Av1InverseWalshHadamardTransformer.cs
@@ -35,7 +35,7 @@ internal static class Av1InverseWalshHadamardTransformer
int writeStride,
int coefficientCount,
Span workspace)
- => TransformAdd(
+ => TransformAdd>(
coefficients,
readBuffer,
readStride,
@@ -65,7 +65,7 @@ internal static class Av1InverseWalshHadamardTransformer
int coefficientCount,
Span workspace,
int bitDepth)
- => TransformAdd(
+ => TransformAdd>(
coefficients,
readBuffer,
readStride,
diff --git a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs
index f6e12fd1db..01d535a2b9 100644
--- a/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Heif/Av1/Av1InverseTransformTests.cs
@@ -788,10 +788,10 @@ public class Av1InverseTransformTests
Array.Fill(scalar, byte.MaxValue);
Array.Fill(vector128, byte.MaxValue);
- Av1Inverse2dTransformer.Transform2dScalar(
+ Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth);
- Av1Inverse2dTransformer.Transform2dVector128(
+ Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth);
Assert.Equal(scalar, vector128);
@@ -802,7 +802,7 @@ public class Av1InverseTransformTests
int[] vector256Workspace = new int[workspaceLength];
Array.Fill(vector256, byte.MaxValue);
- Av1Inverse2dTransformer.Transform2dVector256(
+ Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth);
Assert.Equal(scalar, vector256);
@@ -849,10 +849,10 @@ public class Av1InverseTransformTests
Array.Fill(scalar, short.MinValue);
Array.Fill(vector128, short.MinValue);
- Av1Inverse2dTransformer.Transform2dScalar(
+ Av1Inverse2dTransformer.Transform2dScalar, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, scalar, writeStride, ref config, scalarWorkspace, bitDepth);
- Av1Inverse2dTransformer.Transform2dVector128(
+ Av1Inverse2dTransformer.Transform2dVector128, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, vector128, writeStride, ref config, vector128Workspace, bitDepth);
Assert.Equal(scalar, vector128);
@@ -863,7 +863,7 @@ public class Av1InverseTransformTests
int[] vector256Workspace = new int[workspaceLength];
Array.Fill(vector256, short.MinValue);
- Av1Inverse2dTransformer.Transform2dVector256(
+ Av1Inverse2dTransformer.Transform2dVector256, TColumnOperator, TRowOperator>(
coefficients, prediction, readStride, vector256, writeStride, ref config, vector256Workspace, bitDepth);
Assert.Equal(scalar, vector256);