diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
index 83406e0d6..9eafaea0a 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
@@ -71,16 +71,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <summary>
         /// Invoke <see cref="JpegBlockPostProcessor"/> for <see cref="BlockRowsPerStep"/> block rows, copy the result into <see cref="ColorBuffer"/>.
         /// </summary>
-        public void CopyBlocksToColorBuffer()
+        public void CopyBlocksToColorBuffer(int step)
         {
             var blockPp = new JpegBlockPostProcessor(this.RawJpeg, this.Component);
             float maximumValue = MathF.Pow(2, this.RawJpeg.Precision) - 1;
 
             int destAreaStride = this.ColorBuffer.Width;
 
+            int yBlockStart = step * this.BlockRowsPerStep;
+
             for (int y = 0; y < this.BlockRowsPerStep; y++)
             {
-                int yBlock = this.currentComponentRowInBlocks + y;
+                int yBlock = yBlockStart + y;
 
                 if (yBlock >= this.SizeInBlocks.Height)
                 {
@@ -104,7 +106,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                     blockPp.ProcessBlockColorsInto(ref block, ref destAreaOrigin, destAreaStride, maximumValue);
                 }
             }
+        }
 
+        public void CopyBlocksToColorBuffer()
+        {
+            this.CopyBlocksToColorBuffer(this.currentComponentRowInBlocks);
             this.currentComponentRowInBlocks += this.BlockRowsPerStep;
         }
     }
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
index ec1c057b2..16a55e95e 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
@@ -17,7 +17,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
     {
         public abstract void InjectFrameData(JpegFrame frame, IRawJpegData jpegData);
 
-        public abstract void ConvertStride();
+        public abstract void ConvertStride(int step);
 
         public abstract void Dispose();
     }
@@ -44,8 +44,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
         private int PixelRowsPerStep;
 
-        private int PixelRowCounter;
-
 
         private bool converted;
 
@@ -55,10 +53,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             {
                 if (!this.converted)
                 {
-                    while (this.PixelRowCounter < this.pixelBuffer.Height)
+                    int steps = (int)Math.Ceiling(this.pixelBuffer.Height / (float)this.PixelRowsPerStep);
+
+                    for (int i = 0; i < steps; i++)
                     {
                         this.cancellationToken.ThrowIfCancellationRequested();
-                        this.ConvertStride();
+                        this.ConvertStride(i);
                     }
 
                     this.converted = true;
@@ -103,20 +103,22 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             this.colorConverter = JpegColorConverter.GetConverter(jpegData.ColorSpace, frame.Precision);
         }
 
-        public override void ConvertStride()
+        public override void ConvertStride(int step)
         {
-            int maxY = Math.Min(this.pixelBuffer.Height, this.PixelRowCounter + this.PixelRowsPerStep);
+            int pixelRowStart = this.PixelRowsPerStep * step;
+
+            int maxY = Math.Min(this.pixelBuffer.Height, pixelRowStart + this.PixelRowsPerStep);
 
             var buffers = new Buffer2D<float>[this.componentProcessors.Length];
             for (int i = 0; i < this.componentProcessors.Length; i++)
             {
-                this.componentProcessors[i].CopyBlocksToColorBuffer();
+                this.componentProcessors[i].CopyBlocksToColorBuffer(step);
                 buffers[i] = this.componentProcessors[i].ColorBuffer;
             }
 
-            for (int yy = this.PixelRowCounter; yy < maxY; yy++)
+            for (int yy = pixelRowStart; yy < maxY; yy++)
             {
-                int y = yy - this.PixelRowCounter;
+                int y = yy - pixelRowStart;
 
                 var values = new JpegColorConverter.ComponentValues(buffers, y);
                 this.colorConverter.ConvertToRgba(values, this.rgbaBuffer.GetSpan());
@@ -126,8 +128,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 // TODO: Investigate if slicing is actually necessary
                 PixelOperations<TPixel>.Instance.FromVector4Destructive(this.configuration, this.rgbaBuffer.GetSpan().Slice(0, destRow.Length), destRow);
             }
-
-            this.PixelRowCounter += this.PixelRowsPerStep;
         }
 
         public override void Dispose()