diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs
deleted file mode 100644
index b564fc0d7..000000000
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/DecodedBlock.cs
+++ /dev/null
@@ -1,20 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using Block8x8F = SixLabors.ImageSharp.Formats.Jpeg.Common.Block8x8F;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
-{
-    /// <summary>
-    /// A structure to store unprocessed <see cref="Block8x8F"/> instances and their coordinates while scanning the image.
-    /// The <see cref="Block"/> is present in a "raw" decoded frequency-domain form.
-    /// We need to apply IDCT and unzigging to transform them into color-space blocks.
-    /// </summary>
-    internal struct DecodedBlock
-    {
-        /// <summary>
-        /// The <see cref="Block8x8F"/>
-        /// </summary>
-        public Block8x8F Block;
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
index f1d97adb2..de28b3a7d 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/JpegBlockProcessor.cs
@@ -67,7 +67,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// <param name="by">The y index of the block in <see cref="OldComponent.SpectralBlocks"/></param>
         private void ProcessBlockColors(OldJpegDecoderCore decoder, OldComponent component, int bx, int by)
         {
-            this.data.Block = component.GetBlockReference(bx, by).Block;
+            this.data.Block = component.GetBlockReference(bx, by);
             int qtIndex = decoder.Components[this.componentIndex].Selector;
             this.data.QuantiazationTable = decoder.QuantizationTables[qtIndex];
 
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
index b0a034cb4..7b44c012d 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldComponent.cs
@@ -5,12 +5,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
 {
     using System;
 
+    using SixLabors.ImageSharp.Formats.Jpeg.Common;
     using SixLabors.ImageSharp.Memory;
 
     /// <summary>
     /// Represents a single color component
     /// </summary>
-    internal class OldComponent
+    internal class OldComponent : IDisposable
     {
         public OldComponent(byte identifier, int index)
         {
@@ -49,7 +50,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// This is done by <see cref="OldJpegDecoderCore.ProcessBlocksIntoJpegImageChannels{TPixel}"/>.
         /// When <see cref="OldJpegDecoderCore.IsProgressive"/> us true, we are touching these blocks multiple times - each time we process a Scan.
         /// </summary>
-        public Buffer2D<DecodedBlock> SpectralBlocks { get; private set; }
+        public Buffer2D<Block8x8F> SpectralBlocks { get; private set; }
 
         /// <summary>
         /// Gets the number of blocks for this component along the X axis
@@ -61,7 +62,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// </summary>
         public int BlockCountY { get; private set; }
 
-        public ref DecodedBlock GetBlockReference(int bx, int by)
+        public ref Block8x8F GetBlockReference(int bx, int by)
         {
             return ref this.SpectralBlocks[bx, by];
         }
@@ -74,7 +75,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         {
             this.BlockCountX = decoder.MCUCountX * this.HorizontalFactor;
             this.BlockCountY = decoder.MCUCountY * this.VerticalFactor;
-            this.SpectralBlocks = Buffer2D<DecodedBlock>.CreateClean(this.BlockCountX, this.BlockCountY);
+            this.SpectralBlocks = Buffer2D<Block8x8F>.CreateClean(this.BlockCountX, this.BlockCountY);
         }
 
         /// <summary>
@@ -229,5 +230,10 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             this.HorizontalFactor = h;
             this.VerticalFactor = v;
         }
+
+        public void Dispose()
+        {
+            this.SpectralBlocks.Dispose();
+        }
     }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
index 14004b1ea..2f0bf7715 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OldJpegScanDecoder.cs
@@ -169,19 +169,20 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                                 }
                             }
 
-                            // Take a block from the components buffer:
+                            // Find the block at (bx,by) in the component's buffer:
                             OldComponent component = decoder.Components[this.ComponentIndex];
-                            ref DecodedBlock blockRefOnHeap = ref component.GetBlockReference(this.bx, this.by);
+                            ref Block8x8F blockRefOnHeap = ref component.GetBlockReference(this.bx, this.by);
 
-                            this.data.Block = blockRefOnHeap.Block;
+                            // Copy block to stack
+                            this.data.Block = blockRefOnHeap;
 
                             if (!decoder.InputProcessor.UnexpectedEndOfStreamReached)
                             {
                                 this.DecodeBlock(decoder, scanIndex);
                             }
 
-                            // Store the decoded block
-                            blockRefOnHeap.Block = this.data.Block;
+                            // Store the result block:
+                            blockRefOnHeap = this.data.Block;
                         }
 
                         // for j
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
index a1a24d4e3..dbf1bfcfb 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/OldJpegDecoderCore.cs
@@ -200,9 +200,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
             return image;
         }
 
-        /// <summary>
-        /// Dispose
-        /// </summary>
+        /// <inheritdoc />
         public void Dispose()
         {
             for (int i = 0; i < this.HuffmanTrees.Length; i++)
@@ -212,12 +210,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
 
             if (this.Components != null)
             {
-                foreach (Buffer<DecodedBlock> blockArray in this.Components.Select(c => c.SpectralBlocks))
+                foreach (OldComponent component in this.Components)
                 {
-                    blockArray?.Dispose();
+                    component.Dispose();
                 }
             }
-            
 
             this.ycbcrImage?.Dispose();
             this.InputProcessor.Dispose();