diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
index 58c34ecf02..614e96e54a 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponent.cs
@@ -138,9 +138,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             int spectralAllocWidth = this.SizeInBlocks.Width;
             int spectralAllocHeight = fullScan ? this.SizeInBlocks.Height : this.VerticalSamplingFactor;
 
-            // We don't need to clear buffer for stride-by-stride approach
-            AllocationOptions allocOptions = fullScan ? AllocationOptions.Clean : AllocationOptions.None;
-            this.SpectralBlocks = this.memoryAllocator.Allocate2D<Block8x8>(spectralAllocWidth, spectralAllocHeight, allocOptions);
+            this.SpectralBlocks = this.memoryAllocator.Allocate2D<Block8x8>(spectralAllocWidth, spectralAllocHeight, AllocationOptions.Clean);
         }
     }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
index 6ad2bf00c1..61f521c2f4 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
@@ -90,6 +90,10 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
         public override void ConvertStrideBaseline()
         {
+            // Convert next pixel stride using single spectral `stride'
+            // Note that zero passing eliminates the need of virtual call from JpegComponentPostProcessor
+            this.ConvertNextStride(spectralStep: 0);
+
             // Clear spectral stride - this is VERY important as jpeg possibly won't fill entire buffer each stride
             // Which leads to decoding artifacts
             // Note that this code clears all buffers of the post processors, it's their responsibility to allocate only single stride
@@ -97,10 +101,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             {
                 cpp.ClearSpectralBuffers();
             }
-
-            // Convert next pixel stride using single spectral `stride'
-            // Note that zero passing eliminates the need of virtual call from JpegComponentPostProcessor
-            this.ConvertNextStride(spectralStep: 0);
         }
 
         public override void Dispose()