diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegBlockPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegBlockPostProcessor.cs
deleted file mode 100644
index 15f212b40..000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegBlockPostProcessor.cs
+++ /dev/null
@@ -1,82 +0,0 @@
-// Copyright (c) Six Labors.
-// Licensed under the Apache License, Version 2.0.
-
-using System;
-using System.Runtime.InteropServices;
-using SixLabors.ImageSharp.Memory;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    /// <summary>
-    /// Encapsulates the implementation of processing "raw" jpeg buffers into Jpeg image channels.
-    /// </summary>
-    [StructLayout(LayoutKind.Sequential)]
-    internal struct JpegBlockPostProcessor
-    {
-        /// <summary>
-        /// Source block
-        /// </summary>
-        public Block8x8F SourceBlock;
-
-        /// <summary>
-        /// The quantization table as <see cref="Block8x8F"/>.
-        /// </summary>
-        public Block8x8F DequantiazationTable;
-
-        /// <summary>
-        /// Defines the horizontal and vertical scale we need to apply to the 8x8 sized block.
-        /// </summary>
-        private Size subSamplingDivisors;
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="JpegBlockPostProcessor"/> struct.
-        /// </summary>
-        /// <param name="decoder">The raw jpeg data.</param>
-        /// <param name="component">The raw component.</param>
-        public JpegBlockPostProcessor(IRawJpegData decoder, IJpegComponent component)
-        {
-            int qtIndex = component.QuantizationTableIndex;
-            this.DequantiazationTable = decoder.QuantizationTables[qtIndex];
-            this.subSamplingDivisors = component.SubSamplingDivisors;
-
-            this.SourceBlock = default;
-        }
-
-        /// <summary>
-        /// Processes 'sourceBlock' producing Jpeg color channel values from spectral values:
-        /// - Dequantize
-        /// - Applying IDCT
-        /// - Level shift by +maximumValue/2, clip to [0, maximumValue]
-        /// - Copy the resulting color values into 'destArea' scaling up the block by amount defined in <see cref="subSamplingDivisors"/>.
-        /// </summary>
-        /// <param name="sourceBlock">The source block.</param>
-        /// <param name="destAreaOrigin">Reference to the origin of the destination pixel area.</param>
-        /// <param name="destAreaStride">The width of the destination pixel buffer.</param>
-        /// <param name="maximumValue">The maximum value derived from the bitdepth.</param>
-        public void ProcessBlockColorsInto(
-            ref Block8x8 sourceBlock,
-            ref float destAreaOrigin,
-            int destAreaStride,
-            float maximumValue)
-        {
-            ref Block8x8F block = ref this.SourceBlock;
-            block.LoadFrom(ref sourceBlock);
-
-            // Dequantize:
-            block.MultiplyInPlace(ref this.DequantiazationTable);
-
-            FastFloatingPointDCT.TransformIDCT(ref block);
-
-            // To conform better to libjpeg we actually NEED TO loose precision here.
-            // This is because they store blocks as Int16 between all the operations.
-            // To be "more accurate", we need to emulate this by rounding!
-            block.NormalizeColorsAndRoundInPlace(maximumValue);
-
-            block.ScaledCopyTo(
-                ref destAreaOrigin,
-                destAreaStride,
-                this.subSamplingDivisors.Width,
-                this.subSamplingDivisors.Height);
-        }
-    }
-}
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
index 9a659d621..b26aae8d8 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
@@ -76,14 +76,17 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         {
             Buffer2D<Block8x8> spectralBuffer = this.Component.SpectralBlocks;
 
-            var blockPp = new JpegBlockPostProcessor(this.RawJpeg, this.Component);
-
             float maximumValue = this.frame.MaxColorChannelValue;
 
             int destAreaStride = this.ColorBuffer.Width;
 
             int yBlockStart = step * this.BlockRowsPerStep;
 
+            Size subSamplingDivisors = this.Component.SubSamplingDivisors;
+
+            Block8x8F dequantTable = this.RawJpeg.QuantizationTables[this.Component.QuantizationTableIndex];
+            Block8x8F workspaceBlock = default;
+
             for (int y = 0; y < this.BlockRowsPerStep; y++)
             {
                 int yBlock = yBlockStart + y;
@@ -103,11 +106,28 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
                 for (int xBlock = 0; xBlock < widthInBlocks; xBlock++)
                 {
-                    ref Block8x8 block = ref blockRow[xBlock];
                     int xBuffer = xBlock * this.blockAreaSize.Width;
                     ref float destAreaOrigin = ref colorBufferRow[xBuffer];
 
-                    blockPp.ProcessBlockColorsInto(ref block, ref destAreaOrigin, destAreaStride, maximumValue);
+                    workspaceBlock.LoadFrom(ref blockRow[xBlock]);
+
+                    // Dequantize
+                    workspaceBlock.MultiplyInPlace(ref dequantTable);
+
+                    // Convert from spectral to color
+                    FastFloatingPointDCT.TransformIDCT(ref workspaceBlock);
+
+                    // To conform better to libjpeg we actually NEED TO loose precision here.
+                    // This is because they store blocks as Int16 between all the operations.
+                    // To be "more accurate", we need to emulate this by rounding!
+                    workspaceBlock.NormalizeColorsAndRoundInPlace(maximumValue);
+
+                    // Write to color buffer acording to sampling factors
+                    workspaceBlock.ScaledCopyTo(
+                        ref destAreaOrigin,
+                        destAreaStride,
+                        subSamplingDivisors.Width,
+                        subSamplingDivisors.Height);
                 }
             }
         }