diff --git a/src/ImageSharp/Common/Extensions/ListExtensions.cs b/src/ImageSharp/Common/Extensions/ListExtensions.cs
new file mode 100644
index 000000000..752f7ef21
--- /dev/null
+++ b/src/ImageSharp/Common/Extensions/ListExtensions.cs
@@ -0,0 +1,48 @@
+// <copyright file="ListExtensions.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Common.Extensions
+{
+    using System.Collections.Generic;
+
+    /// <summary>
+    /// Encapsulates a series of time saving extension methods to the <see cref="T:System.Collections.Generic.List"/> class.
+    /// </summary>
+    internal static class ListExtensions
+    {
+        /// <summary>
+        /// Inserts an item at the given index automatically expanding the capacity if required.
+        /// </summary>
+        /// <typeparam name="T">The type of object within the list</typeparam>
+        /// <param name="list">The list</param>
+        /// <param name="index">The index</param>
+        /// <param name="item">The item to insert</param>
+        public static void SafeInsert<T>(this List<T> list, int index, T item)
+        {
+            if (index >= list.Count)
+            {
+                list.Add(item);
+            }
+            else
+            {
+                list[index] = item;
+            }
+        }
+
+        /// <summary>
+        /// Removes the last element from a list and returns that element. This method changes the length of the list.
+        /// </summary>
+        /// <typeparam name="T">The type of object within the list</typeparam>
+        /// <param name="list">The list</param>
+        /// <returns>The last element in the specified sequence.</returns>
+        public static T Pop<T>(this List<T> list)
+        {
+            int last = list.Count - 1;
+            T item = list[last];
+            list.RemoveAt(last);
+            return item;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs
new file mode 100644
index 000000000..0f0a9b540
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanBranch.cs
@@ -0,0 +1,55 @@
+// <copyright file="HuffmanBranch.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    using System.Collections.Generic;
+
+    /// <summary>
+    /// Represents a branch in the huffman tree
+    /// </summary>
+    internal struct HuffmanBranch
+    {
+        /// <summary>
+        /// The index
+        /// </summary>
+        public int Index;
+
+        /// <summary>
+        /// The value
+        /// </summary>
+        public short Value;
+
+        /// <summary>
+        /// The children
+        /// </summary>
+        public List<HuffmanBranch> Children;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="HuffmanBranch"/> struct.
+        /// </summary>
+        /// <param name="value">The value</param>
+        public HuffmanBranch(short value)
+            : this(value, new List<HuffmanBranch>())
+        {
+        }
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="HuffmanBranch"/> struct.
+        /// </summary>
+        /// <param name="children">The branch children</param>
+        public HuffmanBranch(List<HuffmanBranch> children)
+            : this((short)0, children)
+        {
+        }
+
+        private HuffmanBranch(short value, List<HuffmanBranch> children)
+        {
+            this.Index = 0;
+            this.Value = value;
+            this.Children = children;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
index 5fcc2007b..08c37bcbc 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
@@ -1,43 +1,50 @@
-namespace ImageSharp.Formats.Jpeg.Port.Components
+// <copyright file="HuffmanTables.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
 {
     using System.Collections.Generic;
-
-    using ImageSharp.Memory;
+    using System.Runtime.CompilerServices;
 
     /// <summary>
     /// Defines a pair of huffman tables
     /// </summary>
     internal class HuffmanTables
     {
-        /// <summary>
-        /// Gets or sets the quantization tables.
-        /// </summary>
-        public Fast2DArray<HuffmanBranch> Tables { get; set; } = new Fast2DArray<HuffmanBranch>(256, 2);
-    }
-
-    internal struct HuffmanBranch
-    {
-        public HuffmanBranch(short value)
-            : this(value, new List<HuffmanBranch>())
-        {
-        }
+        private List<HuffmanBranch> first = new List<HuffmanBranch>();
 
-        public HuffmanBranch(List<HuffmanBranch> children)
-            : this(0, children)
-        {
-        }
+        private List<HuffmanBranch> second = new List<HuffmanBranch>();
 
-        private HuffmanBranch(short value, List<HuffmanBranch> children)
+        /// <summary>
+        /// Gets or sets the table at the given index.
+        /// </summary>
+        /// <param name="index">The index</param>
+        /// <returns>The <see cref="List{HuffmanBranch}"/></returns>
+        public List<HuffmanBranch> this[int index]
         {
-            this.Index = 0;
-            this.Value = value;
-            this.Children = children;
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get
+            {
+                if (index == 0)
+                {
+                    return this.first;
+                }
+
+                return this.second;
+            }
+
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            set
+            {
+                if (index == 0)
+                {
+                    this.first = value;
+                }
+
+                this.second = value;
+            }
         }
-
-        public int Index;
-
-        public short Value;
-
-        public List<HuffmanBranch> Children;
     }
-}
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Huffman.cs b/src/ImageSharp/Formats/Jpeg/Port/Huffman.cs
deleted file mode 100644
index 75b6dc562..000000000
--- a/src/ImageSharp/Formats/Jpeg/Port/Huffman.cs
+++ /dev/null
@@ -1,6 +0,0 @@
-namespace ImageSharp.Formats.Jpeg.Port
-{
-    class Huffman
-    {
-    }
-}
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
index db25ba845..95a83b086 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
@@ -8,8 +8,8 @@ namespace ImageSharp.Formats.Jpeg.Port
     using System;
     using System.Collections.Generic;
     using System.IO;
-    using System.Linq;
 
+    using ImageSharp.Common.Extensions;
     using ImageSharp.Formats.Jpeg.Port.Components;
     using ImageSharp.Memory;
     using ImageSharp.PixelFormats;
@@ -50,11 +50,6 @@ namespace ImageSharp.Formats.Jpeg.Port
         /// </summary>
         private JFif jFif;
 
-        /// <summary>
-        /// Whether the image has a EXIF header
-        /// </summary>
-        private bool isExif;
-
         /// <summary>
         /// Initializes a new instance of the <see cref="JpegDecoderCore" /> class.
         /// </summary>
@@ -345,108 +340,96 @@ namespace ImageSharp.Formats.Jpeg.Port
                 throw new ImageFormatException("DHT has wrong length");
             }
 
-            using (var huffmanData = new Buffer<byte>(remaining))
+            using (var huffmanData = new Buffer<byte>(16))
             {
-                this.InputStream.Skip(1);
-                this.InputStream.Read(huffmanData.Array, 0, remaining);
-
-                for (int i = 0; i < remaining;)
+                for (int i = 2; i < remaining;)
                 {
-                    byte huffmanTableSpec = huffmanData[i];
-                    byte[] codeLengths = new byte[16];
-                    int codeLengthSum = 0;
-
-                    for (int j = 0; j < 16; j++)
-                    {
-                        codeLengthSum += codeLengths[j] = huffmanData[j];
-                    }
+                    byte huffmanTableSpec = (byte)this.InputStream.ReadByte();
+                    this.InputStream.Read(huffmanData.Array, 0, 16);
 
-                    // TODO: Pooling?
-                    short[] huffmanValues = new short[codeLengthSum];
-                    using (var values = new Buffer<byte>(codeLengthSum))
+                    using (var codeLengths = new Buffer<byte>(16))
                     {
-                        this.InputStream.Read(values.Array, 0, codeLengthSum);
+                        int codeLengthSum = 0;
 
-                        for (int j = 0; j < codeLengthSum; j++)
+                        for (int j = 0; j < 16; j++)
                         {
-                            huffmanValues[j] = values[j];
+                            codeLengthSum += codeLengths[j] = huffmanData[j];
                         }
 
-                        i += 17 + codeLengthSum;
+                        using (var huffmanValues = new Buffer<byte>(codeLengthSum))
+                        {
+                            this.InputStream.Read(huffmanValues.Array, 0, codeLengthSum);
+
+                            i += 17 + codeLengthSum;
 
-                        this.BuildHuffmanTable(
-                            huffmanTableSpec >> 4 == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
-                            huffmanTableSpec & 15,
-                            codeLengths,
-                            huffmanValues);
+                            // Everything I can discover indicates there's a max of two table per DC AC pair though this limits the index to 16?
+                            this.BuildHuffmanTable(
+                                huffmanTableSpec >> 4 == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
+                                huffmanTableSpec & 15,
+                                codeLengths.Array,
+                                huffmanValues.Array);
+                        }
                     }
                 }
             }
         }
 
-        private void BuildHuffmanTable(HuffmanTables tables, int index, byte[] codeLengths, short[] values)
+        /// <summary>
+        /// Builds the huffman tables
+        /// </summary>
+        /// <param name="tables">The tables</param>
+        /// <param name="index">The table index</param>
+        /// <param name="codeLengths">The codelengths</param>
+        /// <param name="values">The values</param>
+        private void BuildHuffmanTable(HuffmanTables tables, int index, byte[] codeLengths, byte[] values)
         {
-            // (╯°□°）╯︵ ┻━┻ Everything up to here is going well. I can't match the JavaScript now though.
             int length = 16;
             while (length > 0 && codeLengths[length - 1] == 0)
             {
                 length--;
             }
 
-            var code = new Queue<HuffmanBranch>();
-            code.Enqueue(new HuffmanBranch(new List<HuffmanBranch>()));
-            HuffmanBranch p = code.Peek();
-            p.Children = new List<HuffmanBranch>();
-            HuffmanBranch q;
+            // TODO: Check the capacity here. Seems to max at 2
+            var code = new List<HuffmanBranch> { new HuffmanBranch(new List<HuffmanBranch>()) };
+            HuffmanBranch p = code[0];
             int k = 0;
-            try
+
+            for (int i = 0; i < length; i++)
             {
-                for (int i = 0; i < length; i++)
+                HuffmanBranch q;
+                for (int j = 0; j < codeLengths[i]; j++)
                 {
-                    for (int j = 0; j < codeLengths[i]; j++)
+                    p = code.Pop();
+                    p.Children.SafeInsert(p.Index, new HuffmanBranch(values[k]));
+                    while (p.Index > 0)
                     {
-                        p = code.Dequeue();
-                        p.Children.Add(new HuffmanBranch(values[k]));
-                        while (p.Index > 0)
-                        {
-                            p = code.Dequeue();
-                        }
-
-                        p.Index++;
-                        code.Enqueue(p);
-                        while (code.Count <= i)
-                        {
-                            q = new HuffmanBranch(new List<HuffmanBranch>());
-                            code.Enqueue(q);
-                            p.Children.Add(new HuffmanBranch(q.Children));
-                            p = q;
-                        }
-
-                        k++;
+                        p = code.Pop();
                     }
 
-                    if (i + 1 < length)
+                    p.Index++;
+                    code.Add(p);
+                    while (code.Count <= i)
                     {
-                        // p here points to last code
                         q = new HuffmanBranch(new List<HuffmanBranch>());
-                        code.Enqueue(q);
-                        p.Children.Add(new HuffmanBranch(q.Children));
+                        code.Add(q);
+                        p.Children.SafeInsert(p.Index, new HuffmanBranch(q.Children));
                         p = q;
                     }
-                }
 
-                Span<HuffmanBranch> tableSpan = tables.Tables.GetRowSpan(index);
+                    k++;
+                }
 
-                List<HuffmanBranch> result = code.Peek().Children;
-                for (int i = 0; i < result.Count; i++)
+                if (i + 1 < length)
                 {
-                    tableSpan[i] = result[i];
+                    // p here points to last code
+                    q = new HuffmanBranch(new List<HuffmanBranch>());
+                    code.Add(q);
+                    p.Children.SafeInsert(p.Index, new HuffmanBranch(q.Children));
+                    p = q;
                 }
             }
-            catch (Exception e)
-            {
-                throw;
-            }
+
+            tables[index] = code[0].Children;
         }
 
         /// <summary>
@@ -533,8 +516,7 @@ namespace ImageSharp.Formats.Jpeg.Port
         private ushort ReadUint16()
         {
             this.InputStream.Read(this.uint16Buffer, 0, 2);
-            ushort value = (ushort)((this.uint16Buffer[0] << 8) | this.uint16Buffer[1]);
-            return value;
+            return (ushort)((this.uint16Buffer[0] << 8) | this.uint16Buffer[1]);
         }
 
         /// <summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegFrame.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegFrame.cs
deleted file mode 100644
index a279339e7..000000000
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegFrame.cs
+++ /dev/null
@@ -1,33 +0,0 @@
-namespace ImageSharp.Formats.Jpeg.Port
-{
-    /// <summary>
-    /// Represents a jpeg frame
-    /// </summary>
-    internal class JpegFrame
-    {
-        /// <summary>
-        /// Gets or sets a value indicating whether the fame is extended
-        /// </summary>
-        public bool Extended { get; set; }
-
-        /// <summary>
-        /// Gets or sets a value indicating whether the fame is progressive
-        /// </summary>
-        public bool Progressive { get; set; }
-
-        /// <summary>
-        /// Gets or sets the precision
-        /// </summary>
-        public byte Precision { get; set; }
-
-        /// <summary>
-        /// Gets or sets the number of scanlines within the frame
-        /// </summary>
-        public short Scanlines { get; set; }
-
-        /// <summary>
-        /// Gets or sets the number of samples per scanline
-        /// </summary>
-        public short SamplesPerLine { get; set; }
-    }
-}