diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
index e1faf93f49..f1335eda9e 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
@@ -113,16 +113,24 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// Decodes the entropy coded data.
         /// </summary>
         /// <param name="scanComponentCount">Component count in the current scan.</param>
-        public void ParseEntropyCodedData(int scanComponentCount)
+        public void ParseEntropyCodedData(int scanComponentCount, JpegFrame frame, IRawJpegData jpegData)
         {
             this.cancellationToken.ThrowIfCancellationRequested();
 
             this.scanComponentCount = scanComponentCount;
-
             this.scanBuffer = new HuffmanScanBuffer(this.stream);
 
-            bool fullScan = this.frame.Progressive || this.frame.MultiScan;
-            this.frame.AllocateComponents(fullScan);
+            // Decoder can encounter markers which would alter parameters
+            // needed for spectral buffers allocation and for spectral
+            // converter allocation
+            if (this.frame == null)
+            {
+                frame.AllocateComponents();
+
+                this.frame = frame;
+                this.components = frame.Components;
+                this.spectralConverter.InjectFrameData(frame, jpegData);
+            }
 
             if (!this.frame.Progressive)
             {
@@ -139,14 +147,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             }
         }
 
-        public void InjectFrameData(JpegFrame frame, IRawJpegData jpegData)
-        {
-            this.frame = frame;
-            this.components = frame.Components;
-
-            this.spectralConverter.InjectFrameData(frame, jpegData);
-        }
-
         private void ParseBaselineData()
         {
             if (this.scanComponentCount != 1)
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
index 3804e1c6c0..94f4a38a59 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
@@ -121,12 +121,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
         public void AllocateSpectral(bool fullScan)
         {
-            if (this.SpectralBlocks != null)
-            {
-                // this method will be called each scan marker so we need to allocate only once
-                return;
-            }
-
             int spectralAllocWidth = this.SizeInBlocks.Width;
             int spectralAllocHeight = fullScan ? this.SizeInBlocks.Height : this.VerticalSamplingFactor;
 
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
index fc109be261..d8f8147068 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
@@ -139,8 +139,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             }
         }
 
-        public void AllocateComponents(bool fullScan)
+        public void AllocateComponents()
         {
+            bool fullScan = this.Progressive || this.MultiScan;
             for (int i = 0; i < this.ComponentCount; i++)
             {
                 JpegComponent component = this.Components[i];
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 8ae4dbd917..daacd862b9 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -1160,7 +1160,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             if (!metadataOnly)
             {
                 this.Frame.Init(maxH, maxV);
-                this.scanDecoder.InjectFrameData(this.Frame, this);
             }
         }
 
@@ -1336,7 +1335,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             this.scanDecoder.SuccessiveHigh = successiveApproximation >> 4;
             this.scanDecoder.SuccessiveLow = successiveApproximation & 15;
 
-            this.scanDecoder.ParseEntropyCodedData(selectorsCount);
+            this.scanDecoder.ParseEntropyCodedData(selectorsCount, this.Frame, this);
         }
 
         /// <summary>