diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs
index f5d9c1bf8..b564fc0d7 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs
@@ -12,32 +12,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
     /// </summary>
     internal struct DecodedBlock
     {
-        /// <summary>
-        /// X coordinate of the current block, in units of 8x8. (The third block in the first row has (bx, by) = (2, 0))
-        /// </summary>
-        public int Bx;
-
-        /// <summary>
-        /// Y coordinate of the current block, in units of 8x8. (The third block in the first row has (bx, by) = (2, 0))
-        /// </summary>
-        public int By;
-
         /// <summary>
         /// The <see cref="Block8x8F"/>
         /// </summary>
         public Block8x8F Block;
-
-        /// <summary>
-        /// Store the block data into a <see cref="DecodedBlock"/>
-        /// </summary>
-        /// <param name="bx">X coordinate of the block</param>
-        /// <param name="by">Y coordinate of the block</param>
-        /// <param name="block">The <see cref="Block8x8F"/></param>
-        public void SaveBlock(int bx, int by, ref Block8x8F block)
-        {
-            this.Bx = bx;
-            this.By = by;
-            this.Block = block;
-        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
index 3a98fd3e7..f1d97adb2 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
@@ -47,10 +47,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// <param name="decoder">The <see cref="OldJpegDecoderCore"/> instance</param>
         public void ProcessAllBlocks(OldJpegDecoderCore decoder)
         {
-            Buffer<DecodedBlock> blockArray = decoder.Components[this.componentIndex].DecodedBlocks;
-            for (int i = 0; i < blockArray.Length; i++)
+            OldComponent component = decoder.Components[this.componentIndex];
+
+            for (int by = 0; by < component.BlockCountY; by++)
             {
-                this.ProcessBlockColors(decoder, ref blockArray[i]);
+                for (int bx = 0; bx < component.BlockCountX; bx++)
+                {
+                    this.ProcessBlockColors(decoder, component, bx, by);
+                }
             }
         }
 
@@ -58,10 +62,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// Dequantize, perform the inverse DCT and store decodedBlock.Block to the into the corresponding <see cref="OldJpegPixelArea"/> instance.
         /// </summary>
         /// <param name="decoder">The <see cref="OldJpegDecoderCore"/></param>
-        /// <param name="decodedBlock">The <see cref="DecodedBlock"/></param>
-        private void ProcessBlockColors(OldJpegDecoderCore decoder, ref DecodedBlock decodedBlock)
+        /// <param name="component">The <see cref="OldComponent"/></param>
+        /// <param name="bx">The x index of the block in <see cref="OldComponent.SpectralBlocks"/></param>
+        /// <param name="by">The y index of the block in <see cref="OldComponent.SpectralBlocks"/></param>
+        private void ProcessBlockColors(OldJpegDecoderCore decoder, OldComponent component, int bx, int by)
         {
-            this.data.Block = decodedBlock.Block;
+            this.data.Block = component.GetBlockReference(bx, by).Block;
             int qtIndex = decoder.Components[this.componentIndex].Selector;
             this.data.QuantiazationTable = decoder.QuantizationTables[qtIndex];
 
@@ -72,7 +78,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             DCT.TransformIDCT(ref *b, ref *this.pointers.Temp1, ref *this.pointers.Temp2);
 
             OldJpegPixelArea destChannel = decoder.GetDestinationChannel(this.componentIndex);
-            OldJpegPixelArea destArea = destChannel.GetOffsetedSubAreaForBlock(decodedBlock.Bx, decodedBlock.By);
+            OldJpegPixelArea destArea = destChannel.GetOffsetedSubAreaForBlock(bx, by);
             destArea.LoadColorsFrom(this.pointers.Temp1, this.pointers.Temp2);
         }
 
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
index e6669483a..b0a034cb4 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
@@ -47,23 +47,38 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// Gets the <see cref="Buffer{T}"/> storing the "raw" frequency-domain decoded blocks.
         /// We need to apply IDCT, dequantiazition and unzigging to transform them into color-space blocks.
         /// This is done by <see cref="OldJpegDecoderCore.ProcessBlocksIntoJpegImageChannels{TPixel}"/>.
-        /// When <see cref="OldJpegDecoderCore.IsProgressive"/>==true, we are touching these blocks multiple times - each time we process a Scan.
+        /// When <see cref="OldJpegDecoderCore.IsProgressive"/> us true, we are touching these blocks multiple times - each time we process a Scan.
         /// </summary>
-        public Buffer<DecodedBlock> DecodedBlocks { get; private set; }
+        public Buffer2D<DecodedBlock> SpectralBlocks { get; private set; }
 
         /// <summary>
-        /// Initializes <see cref="DecodedBlocks"/>
+        /// Gets the number of blocks for this component along the X axis
+        /// </summary>
+        public int BlockCountX { get; private set; }
+
+        /// <summary>
+        /// Gets the number of blocks for this component along the Y axis
+        /// </summary>
+        public int BlockCountY { get; private set; }
+
+        public ref DecodedBlock GetBlockReference(int bx, int by)
+        {
+            return ref this.SpectralBlocks[bx, by];
+        }
+
+        /// <summary>
+        /// Initializes <see cref="SpectralBlocks"/>
         /// </summary>
         /// <param name="decoder">The <see cref="OldJpegDecoderCore"/> instance</param>
         public void InitializeBlocks(OldJpegDecoderCore decoder)
         {
-            // TODO: count could be component and JpegSubsample specific:
-            int count = decoder.TotalMCUCount * this.HorizontalFactor * this.VerticalFactor;
-            this.DecodedBlocks = Buffer<DecodedBlock>.CreateClean(count);
+            this.BlockCountX = decoder.MCUCountX * this.HorizontalFactor;
+            this.BlockCountY = decoder.MCUCountY * this.VerticalFactor;
+            this.SpectralBlocks = Buffer2D<DecodedBlock>.CreateClean(this.BlockCountX, this.BlockCountY);
         }
 
         /// <summary>
-        /// Initializes all component data except <see cref="DecodedBlocks"/>.
+        /// Initializes all component data except <see cref="SpectralBlocks"/>.
         /// </summary>
         /// <param name="decoder">The <see cref="OldJpegDecoderCore"/> instance</param>
         public void InitializeData(OldJpegDecoderCore decoder)
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
index c90ab882c..14004b1ea 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
@@ -169,12 +169,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                                 }
                             }
 
-                            // Take an existing block (required when progressive):
-                            int blockIndex = this.GetBlockIndex(decoder);
+                            // Take a block from the components buffer:
+                            OldComponent component = decoder.Components[this.ComponentIndex];
+                            ref DecodedBlock blockRefOnHeap = ref component.GetBlockReference(this.bx, this.by);
 
-                            Buffer<DecodedBlock> blocks = decoder.Components[this.ComponentIndex].DecodedBlocks;
-
-                            this.data.Block = blocks[blockIndex].Block;
+                            this.data.Block = blockRefOnHeap.Block;
 
                             if (!decoder.InputProcessor.UnexpectedEndOfStreamReached)
                             {
@@ -182,7 +181,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                             }
 
                             // Store the decoded block
-                            blocks[blockIndex].SaveBlock(this.bx, this.by, ref this.data.Block);
+                            blockRefOnHeap.Block = this.data.Block;
                         }
 
                         // for j
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
index a9027c7e9..a1a24d4e3 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
@@ -212,7 +212,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
 
             if (this.Components != null)
             {
-                foreach (Buffer<DecodedBlock> blockArray in this.Components.Select(c => c.DecodedBlocks))
+                foreach (Buffer<DecodedBlock> blockArray in this.Components.Select(c => c.SpectralBlocks))
                 {
                     blockArray?.Dispose();
                 }