diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
index 6e81e3a9a6..d271287bca 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
@@ -138,10 +138,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         public void EncodeScanBaselineInterleaved<TPixel>(JpegFrame frame, SpectralConverter<TPixel> converter, CancellationToken cancellationToken)
             where TPixel : unmanaged, IPixel<TPixel>
         {
-            // DEBUG INITIALIZATION SETUP
-            frame.AllocateComponents(fullScan: false);
-
-            // DEBUG ENCODING SETUP
             int mcu = 0;
             int mcusPerColumn = frame.McusPerColumn;
             int mcusPerLine = frame.McusPerLine;
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/JpegComponent.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/JpegComponent.cs
index 65425c05c1..49c7d4257e 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/JpegComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/JpegComponent.cs
@@ -107,7 +107,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
             int spectralAllocWidth = this.SizeInBlocks.Width;
             int spectralAllocHeight = fullScan ? this.SizeInBlocks.Height : this.VerticalSamplingFactor;
 
-            this.SpectralBlocks = this.memoryAllocator.Allocate2D<Block8x8>(spectralAllocWidth, spectralAllocHeight, AllocationOptions.Clean);
+            this.SpectralBlocks = this.memoryAllocator.Allocate2D<Block8x8>(spectralAllocWidth, spectralAllocHeight);
         }
     }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
index ccdac68f18..4318f9e09c 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
@@ -126,6 +126,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
             var spectralConverter = new SpectralConverter<TPixel>(frame, image, this.QuantizationTables, Configuration.Default);
 
+            // TODO: change this for non-interleaved scans
+            frame.AllocateComponents(fullScan: false);
             if (frame.ComponentCount > 1)
             {
                 this.scanEncoder.EncodeScanBaselineInterleaved(frame, spectralConverter, cancellationToken);