diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs
index db31706135..120bfb58d3 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs
@@ -43,6 +43,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         public void Dispose()
         {
             this.Output?.Dispose();
+            this.Output = null;
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
index 0cb9bbb1cb..b386a86f3c 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
@@ -68,6 +68,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         public void Dispose()
         {
             this.BlockData?.Dispose();
+            this.BlockData = null;
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs
deleted file mode 100644
index d716355ad3..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs
+++ /dev/null
@@ -1,54 +0,0 @@
-// <copyright file="HuffmanBranch.cs" company="James Jackson-South">
-// Copyright (c) James Jackson-South and contributors.
-// Licensed under the Apache License, Version 2.0.
-// </copyright>
-
-namespace ImageSharp.Formats.Jpeg.Port.Components
-{
-    using System.Runtime.CompilerServices;
-
-    /// <summary>
-    /// Represents a branch in the huffman tree
-    /// </summary>
-    internal struct HuffmanBranch
-    {
-        /// <summary>
-        /// The index
-        /// </summary>
-        public int Index;
-
-        /// <summary>
-        /// The value
-        /// </summary>
-        public short Value;
-
-        /// <summary>
-        /// The children.
-        /// </summary>
-        public HuffmanBranch[] Children;
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="HuffmanBranch"/> struct.
-        /// </summary>
-        /// <param name="value">The value</param>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public HuffmanBranch(short value)
-        {
-            this.Index = 0;
-            this.Value = value;
-            this.Children = new HuffmanBranch[2];
-        }
-
-        /// <summary>
-        /// Initializes a new instance of the <see cref="HuffmanBranch"/> struct.
-        /// </summary>
-        /// <param name="children">The branch children</param>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public HuffmanBranch(HuffmanBranch[] children)
-        {
-            this.Index = 0;
-            this.Value = -1;
-            this.Children = children;
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
index 995fd550c9..bc31629785 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
@@ -1,32 +1,45 @@
-namespace ImageSharp.Formats.Jpeg.Port.Components
+// <copyright file="HuffmanTable.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
 {
     using System;
     using System.Runtime.CompilerServices;
 
+    using ImageSharp.Memory;
+
     /// <summary>
-    /// Represents a HUffman Table
+    /// Represents a Huffman Table
     /// </summary>
-    internal sealed class HuffmanTable
+    internal struct HuffmanTable : IDisposable
     {
-        private short[] huffcode = new short[257];
-        private short[] huffsize = new short[257];
-        private short[] valOffset = new short[18];
-        private long[] maxcode = new long[18];
+        private Buffer<short> huffcode;
+        private Buffer<short> huffsize;
+        private Buffer<short> valOffset;
+        private Buffer<long> maxcode;
 
-        private byte[] huffval;
-        private byte[] bits;
+        private Buffer<byte> huffval;
+        private Buffer<byte> bits;
 
         /// <summary>
-        /// Initializes a new instance of the <see cref="HuffmanTable"/> class.
+        /// Initializes a new instance of the <see cref="HuffmanTable"/> struct.
         /// </summary>
         /// <param name="lengths">The code lengths</param>
         /// <param name="values">The huffman values</param>
         public HuffmanTable(byte[] lengths, byte[] values)
         {
-            this.huffval = new byte[values.Length];
-            Buffer.BlockCopy(values, 0, this.huffval, 0, values.Length);
-            this.bits = new byte[lengths.Length];
-            Buffer.BlockCopy(lengths, 0, this.bits, 0, lengths.Length);
+            this.huffcode = Buffer<short>.CreateClean(257);
+            this.huffsize = Buffer<short>.CreateClean(257);
+            this.valOffset = Buffer<short>.CreateClean(18);
+            this.maxcode = Buffer<long>.CreateClean(18);
+
+            this.huffval = Buffer<byte>.CreateClean(values.Length);
+            Buffer.BlockCopy(values, 0, this.huffval.Array, 0, values.Length);
+
+            this.bits = Buffer<byte>.CreateClean(lengths.Length);
+            Buffer.BlockCopy(lengths, 0, this.bits.Array, 0, lengths.Length);
 
             this.GenerateSizeTable();
             this.GenerateCodeTable();
@@ -66,6 +79,24 @@
             return this.valOffset[i];
         }
 
+        /// <inheritdoc/>
+        public void Dispose()
+        {
+            this.huffcode?.Dispose();
+            this.huffsize?.Dispose();
+            this.valOffset?.Dispose();
+            this.maxcode?.Dispose();
+            this.huffval?.Dispose();
+            this.bits?.Dispose();
+
+            this.huffcode = null;
+            this.huffsize = null;
+            this.valOffset = null;
+            this.maxcode = null;
+            this.huffval = null;
+            this.bits = null;
+        }
+
         /// <summary>
         /// Figure C.1: make table of Huffman code length for each symbol
         /// </summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
index 8aeafd7db8..6de8c441db 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
@@ -5,13 +5,14 @@
 
 namespace ImageSharp.Formats.Jpeg.Port.Components
 {
+    using System;
     using System.Collections.Generic;
     using System.Runtime.CompilerServices;
 
     /// <summary>
     /// Defines a pair of huffman tables
     /// </summary>
-    internal class HuffmanTables
+    internal sealed class HuffmanTables : IDisposable
     {
         private readonly HuffmanTable[] tables = new HuffmanTable[4];
 
@@ -34,5 +35,14 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
                 this.tables[index] = value;
             }
         }
+
+        /// <inheritdoc/>
+        public void Dispose()
+        {
+            for (int i = 0; i < this.tables.Length; i++)
+            {
+                this.tables[i].Dispose();
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
index 1d588301fe..62b0a82e4c 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
@@ -734,7 +734,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
                 return;
             }
 
-            var componentBlockDataSpan = component.BlockData.Span;
+            Span<short> componentBlockDataSpan = component.BlockData.Span;
             int k = this.specStart;
             int e = this.specEnd;
             while (k <= e)
@@ -773,7 +773,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
             int k = this.specStart;
             int e = this.specEnd;
             int r = 0;
-            var componentBlockDataSpan = component.BlockData.Span;
+            Span<short> componentBlockDataSpan = component.BlockData.Span;
             while (k <= e)
             {
                 byte z = QuantizationTables.DctZigZag[k];
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
index 7bd71048c0..ca93e96dd5 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
@@ -160,12 +160,16 @@ namespace ImageSharp.Formats.Jpeg.Port
             this.frame?.Dispose();
             this.components?.Dispose();
             this.quantizationTables?.Dispose();
+            this.dcHuffmanTables?.Dispose();
+            this.acHuffmanTables?.Dispose();
             this.pixelArea.Dispose();
 
             // Set large fields to null.
             this.frame = null;
             this.components = null;
             this.quantizationTables = null;
+            this.dcHuffmanTables = null;
+            this.acHuffmanTables = null;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -696,7 +700,7 @@ namespace ImageSharp.Formats.Jpeg.Port
                             this.InputStream.Read(huffmanValues.Array, 0, codeLengthSum);
 
                             i += 17 + codeLengthSum;
-                            Debug.WriteLine(huffmanTableSpec >> 4 == 0 ? "this.dcHuffmanTables" : "this.acHuffmanTables");
+
                             this.BuildHuffmanTable(
                                 huffmanTableSpec >> 4 == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
                                 huffmanTableSpec & 15,