diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
index d212340c1..b1f371e0f 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
@@ -94,7 +94,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         {
             this.dctZigZag = ZigZag.CreateUnzigTable();
             this.stream = stream;
-            this.scanBuffer = new HuffmanScanBuffer(stream);
             this.cancellationToken = cancellationToken;
         }
 
@@ -105,6 +104,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         {
             this.cancellationToken.ThrowIfCancellationRequested();
 
+            this.scanBuffer = new HuffmanScanBuffer(this.stream);
+
             if (!this.frame.Progressive)
             {
                 this.ParseBaselineData();
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index dda4e96ea..a52ce3f9c 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -172,6 +172,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// <inheritdoc/>
         public Block8x8F[] QuantizationTables { get; private set; }
 
+        private HuffmanScanDecoder scanDecoder;
+
         /// <summary>
         /// Finds the next file marker within the byte stream.
         /// </summary>
@@ -213,6 +215,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         public Image<TPixel> Decode<TPixel>(BufferedReadStream stream, CancellationToken cancellationToken)
             where TPixel : unmanaged, IPixel<TPixel>
         {
+            this.scanDecoder = new HuffmanScanDecoder(stream, cancellationToken);
+
             this.ParseStream(stream, cancellationToken: cancellationToken);
             this.InitExifProfile();
             this.InitIccProfile();
@@ -1049,26 +1053,17 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             int spectralEnd = this.temp[1];
             int successiveApproximation = this.temp[2];
 
-            var sd = new HuffmanScanDecoder(
-                stream,
-                cancellationToken)
-            {
-                Frame = this.Frame,
-
-                dcHuffmanTables = this.dcHuffmanTables,
-                acHuffmanTables = this.acHuffmanTables,
-
-                ResetInterval = this.resetInterval,
-
-                componentsLength = selectorsCount,
-
-                spectralStart = spectralStart,
-                spectralEnd = spectralEnd,
-                successiveHigh = successiveApproximation >> 4,
-                successiveLow = successiveApproximation & 15
-            };
-
-            sd.ParseEntropyCodedData();
+            this.scanDecoder.Frame = this.Frame;
+            this.scanDecoder.dcHuffmanTables = this.dcHuffmanTables;
+            this.scanDecoder.acHuffmanTables = this.acHuffmanTables;
+            this.scanDecoder.ResetInterval = this.resetInterval;
+            this.scanDecoder.componentsLength = selectorsCount;
+            this.scanDecoder.spectralStart = spectralStart;
+            this.scanDecoder.spectralEnd = spectralEnd;
+            this.scanDecoder.successiveHigh = successiveApproximation >> 4;
+            this.scanDecoder.successiveLow = successiveApproximation & 15;
+
+            this.scanDecoder.ParseEntropyCodedData();
         }
 
         /// <summary>