diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
index b1f371e0f..3980739e1 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
@@ -26,15 +26,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         private JpegFrame frame;
         private JpegComponent[] components;
 
-        public JpegFrame Frame
-        {
-            set
-            {
-                frame = value;
-                components = value.Components;
-            }
-        }
-
         // The restart interval.
         private int restartInterval;
         // How many mcu's are left to do.
@@ -72,6 +63,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
         private HuffmanScanBuffer scanBuffer;
 
+        private SpectralConverter spectralConverter;
+
         private CancellationToken cancellationToken;
 
         /// <summary>
@@ -90,10 +83,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <param name="cancellationToken">The token to monitor cancellation.</param>
         public HuffmanScanDecoder(
             BufferedReadStream stream,
+            SpectralConverter converter,
             CancellationToken cancellationToken)
         {
             this.dctZigZag = ZigZag.CreateUnzigTable();
             this.stream = stream;
+            this.spectralConverter = converter;
             this.cancellationToken = cancellationToken;
         }
 
@@ -121,6 +116,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             }
         }
 
+        public void InjectFrameData(JpegFrame frame, IRawJpegData jpegData)
+        {
+            this.frame = frame;
+            this.components = frame.Components;
+
+            this.spectralConverter.InjectFrameData(frame, jpegData);
+        }
+
         private void ParseBaselineData()
         {
             if (this.componentsLength == 1)
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
index 1befbbf82..ae4a90337 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
@@ -15,6 +15,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 {
     internal abstract class SpectralConverter
     {
+        public abstract void InjectFrameData(JpegFrame frame, IRawJpegData jpegData);
+
         public abstract void ConvertStride();
     }
 
@@ -72,7 +74,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             this.cancellationToken = ct;
         }
 
-        public void InjectFrameData(JpegFrame frame, IRawJpegData jpegData)
+        public override void InjectFrameData(JpegFrame frame, IRawJpegData jpegData)
         {
             MemoryAllocator allocator = this.configuration.MemoryAllocator;
 
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 44bd05145..168286071 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -215,13 +215,16 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         public Image<TPixel> Decode<TPixel>(BufferedReadStream stream, CancellationToken cancellationToken)
             where TPixel : unmanaged, IPixel<TPixel>
         {
-            this.scanDecoder = new HuffmanScanDecoder(stream, cancellationToken);
+            SpectralConverter<TPixel> spectralConverter = new SpectralConverter<TPixel>(this.Configuration, cancellationToken);
+
+            this.scanDecoder = new HuffmanScanDecoder(stream, spectralConverter, cancellationToken);
 
             this.ParseStream(stream, cancellationToken: cancellationToken);
             this.InitExifProfile();
             this.InitIccProfile();
             this.InitIptcProfile();
             this.InitDerivedMetadataProperties();
+
             return this.PostProcessIntoImage<TPixel>(cancellationToken);
         }