diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
index 0a0a2cd9e..83406e0d6 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
@@ -27,23 +27,20 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <summary>
         /// Initializes a new instance of the <see cref="JpegComponentPostProcessor"/> class.
         /// </summary>
-        public JpegComponentPostProcessor(MemoryAllocator memoryAllocator, JpegImagePostProcessor imagePostProcessor, IJpegComponent component)
+        public JpegComponentPostProcessor(MemoryAllocator memoryAllocator, IRawJpegData rawJpeg, Size postProcessorBufferSize, IJpegComponent component)
         {
             this.Component = component;
-            this.ImagePostProcessor = imagePostProcessor;
+            this.RawJpeg = rawJpeg;
             this.blockAreaSize = this.Component.SubSamplingDivisors * 8;
             this.ColorBuffer = memoryAllocator.Allocate2DOveraligned<float>(
-                imagePostProcessor.PostProcessorBufferSize.Width,
-                imagePostProcessor.PostProcessorBufferSize.Height,
+                postProcessorBufferSize.Width,
+                postProcessorBufferSize.Height,
                 this.blockAreaSize.Height);
 
-            this.BlockRowsPerStep = imagePostProcessor.BlockRowsPerStep / this.Component.SubSamplingDivisors.Height;
+            this.BlockRowsPerStep = postProcessorBufferSize.Height / 8 / this.Component.SubSamplingDivisors.Height;
         }
 
-        /// <summary>
-        /// Gets the <see cref="JpegImagePostProcessor"/>
-        /// </summary>
-        public JpegImagePostProcessor ImagePostProcessor { get; }
+        public IRawJpegData RawJpeg { get; }
 
         /// <summary>
         /// Gets the <see cref="Component"/>
@@ -76,8 +73,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         public void CopyBlocksToColorBuffer()
         {
-            var blockPp = new JpegBlockPostProcessor(this.ImagePostProcessor.RawJpeg, this.Component);
-            float maximumValue = MathF.Pow(2, this.ImagePostProcessor.RawJpeg.Precision) - 1;
+            var blockPp = new JpegBlockPostProcessor(this.RawJpeg, this.Component);
+            float maximumValue = MathF.Pow(2, this.RawJpeg.Precision) - 1;
 
             int destAreaStride = this.ColorBuffer.Width;
 
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegImagePostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegImagePostProcessor.cs
index 5f3389e17..18b2bc6e8 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegImagePostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegImagePostProcessor.cs
@@ -62,14 +62,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
             this.NumberOfPostProcessorSteps = c0.SizeInBlocks.Height / this.BlockRowsPerStep;
 
-            this.PostProcessorBufferSize = new Size(c0.SizeInBlocks.Width * 8, this.PixelRowsPerStep);
-
+            var postProcessorBufferSize = new Size(c0.SizeInBlocks.Width * 8, this.PixelRowsPerStep);
             MemoryAllocator memoryAllocator = configuration.MemoryAllocator;
-
             this.ComponentProcessors = new JpegComponentPostProcessor[rawJpeg.Components.Length];
             for (int i = 0; i < rawJpeg.Components.Length; i++)
             {
-                this.ComponentProcessors[i] = new JpegComponentPostProcessor(memoryAllocator, this, rawJpeg.Components[i]);
+                this.ComponentProcessors[i] = new JpegComponentPostProcessor(memoryAllocator, this.RawJpeg, postProcessorBufferSize, rawJpeg.Components[i]);
             }
 
             this.rgbaBuffer = memoryAllocator.Allocate<Vector4>(rawJpeg.ImageSizeInPixels.Width);
@@ -91,11 +89,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         public int NumberOfPostProcessorSteps { get; }
 
-        /// <summary>
-        /// Gets the size of the temporary buffers we need to allocate into <see cref="JpegComponentPostProcessor.ColorBuffer"/>.
-        /// </summary>
-        public Size PostProcessorBufferSize { get; }
-
         /// <summary>
         /// Gets the value of the counter that grows by each step by <see cref="PixelRowsPerStep"/>.
         /// </summary>