diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
new file mode 100644
index 0000000000..995fd550c9
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
@@ -0,0 +1,136 @@
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    using System;
+    using System.Runtime.CompilerServices;
+
+    /// <summary>
+    /// Represents a HUffman Table
+    /// </summary>
+    internal sealed class HuffmanTable
+    {
+        private short[] huffcode = new short[257];
+        private short[] huffsize = new short[257];
+        private short[] valOffset = new short[18];
+        private long[] maxcode = new long[18];
+
+        private byte[] huffval;
+        private byte[] bits;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="HuffmanTable"/> class.
+        /// </summary>
+        /// <param name="lengths">The code lengths</param>
+        /// <param name="values">The huffman values</param>
+        public HuffmanTable(byte[] lengths, byte[] values)
+        {
+            this.huffval = new byte[values.Length];
+            Buffer.BlockCopy(values, 0, this.huffval, 0, values.Length);
+            this.bits = new byte[lengths.Length];
+            Buffer.BlockCopy(lengths, 0, this.bits, 0, lengths.Length);
+
+            this.GenerateSizeTable();
+            this.GenerateCodeTable();
+            this.GenerateDecoderTables();
+        }
+
+        /// <summary>
+        /// Gets the Huffman value code at the given index
+        /// </summary>
+        /// <param name="i">The index</param>
+        /// <returns>The <see cref="int"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public short GetHuffVal(int i)
+        {
+            return this.huffval[i];
+        }
+
+        /// <summary>
+        /// Gets the max code at the given index
+        /// </summary>
+        /// <param name="i">The index</param>
+        /// <returns>The <see cref="int"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public long GetMaxCode(int i)
+        {
+            return this.maxcode[i];
+        }
+
+        /// <summary>
+        /// Gets the index to the locatation of the huffman value
+        /// </summary>
+        /// <param name="i">The index</param>
+        /// <returns>The <see cref="int"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetValPtr(int i)
+        {
+            return this.valOffset[i];
+        }
+
+        /// <summary>
+        /// Figure C.1: make table of Huffman code length for each symbol
+        /// </summary>
+        private void GenerateSizeTable()
+        {
+            short index = 0;
+            for (short l = 1; l <= 16; l++)
+            {
+                byte i = this.bits[l];
+                for (short j = 0; j < i; j++)
+                {
+                    this.huffsize[index] = l;
+                    index++;
+                }
+            }
+
+            this.huffsize[index] = 0;
+        }
+
+        /// <summary>
+        /// Figure C.2: generate the codes themselves
+        /// </summary>
+        private void GenerateCodeTable()
+        {
+            short k = 0;
+            short si = this.huffsize[0];
+            short code = 0;
+            for (short i = 0; i < this.huffsize.Length; i++)
+            {
+                while (this.huffsize[k] == si)
+                {
+                    this.huffcode[k] = code;
+                    code++;
+                    k++;
+                }
+
+                code <<= 1;
+                si++;
+            }
+        }
+
+        /// <summary>
+        /// Figure F.15: generate decoding tables for bit-sequential decoding
+        /// </summary>
+        private void GenerateDecoderTables()
+        {
+            short bitcount = 0;
+            for (int i = 1; i <= 16; i++)
+            {
+                if (this.bits[i] != 0)
+                {
+                    // valoffset[l] = huffval[] index of 1st symbol of code length i,
+                    // minus the minimum code of length i
+                    this.valOffset[i] = (short)(bitcount - this.huffcode[bitcount]);
+                    bitcount += this.bits[i];
+                    this.maxcode[i] = this.huffcode[bitcount - 1]; // maximum code of length i
+                }
+                else
+                {
+                    this.maxcode[i] = -1; // -1 if no codes of this length
+                }
+            }
+
+            this.valOffset[17] = 0;
+            this.maxcode[17] = 0xFFFFFL;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
index a8644d6451..8aeafd7db8 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTables.cs
@@ -13,14 +13,14 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
     /// </summary>
     internal class HuffmanTables
     {
-        private readonly HuffmanBranch[][] tables = new HuffmanBranch[4][];
+        private readonly HuffmanTable[] tables = new HuffmanTable[4];
 
         /// <summary>
         /// Gets or sets the table at the given index.
         /// </summary>
         /// <param name="index">The index</param>
         /// <returns>The <see cref="List{HuffmanBranch}"/></returns>
-        public HuffmanBranch[] this[int index]
+        public HuffmanTable this[int index]
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
             get
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
index 5a31fd89f4..1d588301fe 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
@@ -574,6 +574,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
 #if DEBUG
                     Debug.WriteLine($"DecodeScan - Unexpected marker {(this.bitsData << 8) | nextByte:X} at {stream.Position}");
 #endif
+
                     // We've encountered an unexpected marker. Reverse the stream and exit.
                     this.unexpectedMarkerReached = true;
                     stream.Position -= 2;
@@ -587,27 +588,31 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private short DecodeHuffman(HuffmanBranch[] tree, Stream stream)
+        private short DecodeHuffman(HuffmanTable tree, Stream stream)
         {
-            // TODO: This is our bottleneck. We should use a faster algorithm with a LUT.
-            HuffmanBranch[] node = tree;
-            while (true)
+            // "DECODE", section F.2.2.3, figure F.16, page 109 of T.81
+            int i = 1;
+            short code = (short)this.ReadBit(stream);
+            if (this.endOfStreamReached || this.unexpectedMarkerReached)
             {
-                int index = this.ReadBit(stream);
-                if (this.endOfStreamReached || this.unexpectedMarkerReached)
-                {
-                    return -1;
-                }
+                return -1;
+            }
 
-                HuffmanBranch branch = node[index];
+            while (code > tree.GetMaxCode(i))
+            {
+                code <<= 1;
+                code |= (short)this.ReadBit(stream);
 
-                if (branch.Value > -1)
+                if (this.endOfStreamReached || this.unexpectedMarkerReached)
                 {
-                    return branch.Value;
+                    return -1;
                 }
 
-                node = branch.Children;
+                i++;
             }
+
+            int j = tree.GetValPtr(i);
+            return tree.GetHuffVal((j + code) & 0xFF);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -682,6 +687,12 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
                 }
 
                 k += r;
+
+                if (k > 63)
+                {
+                    break;
+                }
+
                 byte z = QuantizationTables.DctZigZag[k];
                 short re = (short)this.ReceiveAndExtend(s, stream);
                 component.BlockData[offset + z] = re;
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
index 6a1d6311c5..7bd71048c0 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
@@ -7,6 +7,7 @@ namespace ImageSharp.Formats.Jpeg.Port
 {
     using System;
     using System.Collections.Generic;
+    using System.Diagnostics;
     using System.IO;
     using System.Runtime.CompilerServices;
 
@@ -233,8 +234,6 @@ namespace ImageSharp.Formats.Jpeg.Port
 
                     case JpegConstants.Markers.APP15:
                     case JpegConstants.Markers.COM:
-
-                        // TODO: Read data block
                         this.InputStream.Skip(remaining);
                         break;
 
@@ -676,28 +675,28 @@ namespace ImageSharp.Formats.Jpeg.Port
                 throw new ImageFormatException($"DHT has wrong length: {remaining}");
             }
 
-            using (var huffmanData = Buffer<byte>.CreateClean(16))
+            using (var huffmanData = Buffer<byte>.CreateClean(256))
             {
                 for (int i = 2; i < remaining;)
                 {
                     byte huffmanTableSpec = (byte)this.InputStream.ReadByte();
                     this.InputStream.Read(huffmanData.Array, 0, 16);
 
-                    using (var codeLengths = Buffer<byte>.CreateClean(16))
+                    using (var codeLengths = Buffer<byte>.CreateClean(17))
                     {
                         int codeLengthSum = 0;
 
-                        for (int j = 0; j < 16; j++)
+                        for (int j = 1; j < 17; j++)
                         {
-                            codeLengthSum += codeLengths[j] = huffmanData[j];
+                            codeLengthSum += codeLengths[j] = huffmanData[j - 1];
                         }
 
-                        using (var huffmanValues = Buffer<byte>.CreateClean(codeLengthSum))
+                        using (var huffmanValues = Buffer<byte>.CreateClean(256))
                         {
                             this.InputStream.Read(huffmanValues.Array, 0, codeLengthSum);
 
                             i += 17 + codeLengthSum;
-
+                            Debug.WriteLine(huffmanTableSpec >> 4 == 0 ? "this.dcHuffmanTables" : "this.acHuffmanTables");
                             this.BuildHuffmanTable(
                                 huffmanTableSpec >> 4 == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
                                 huffmanTableSpec & 15,
@@ -812,53 +811,7 @@ namespace ImageSharp.Formats.Jpeg.Port
         /// <param name="values">The values</param>
         private void BuildHuffmanTable(HuffmanTables tables, int index, byte[] codeLengths, byte[] values)
         {
-            int length = 16;
-            while (length > 0 && codeLengths[length - 1] == 0)
-            {
-                length--;
-            }
-
-            // TODO: Check the branch children capacity here. Seems to max at 2
-            var code = new List<HuffmanBranch> { new HuffmanBranch(-1) };
-            HuffmanBranch p = code[0];
-            int k = 0;
-
-            for (int i = 0; i < length; i++)
-            {
-                HuffmanBranch q;
-                for (int j = 0; j < codeLengths[i]; j++)
-                {
-                    p = code.Pop();
-                    p.Children[p.Index] = new HuffmanBranch(values[k]);
-                    while (p.Index > 0)
-                    {
-                        p = code.Pop();
-                    }
-
-                    p.Index++;
-                    code.Add(p);
-                    while (code.Count <= i)
-                    {
-                        q = new HuffmanBranch(-1);
-                        code.Add(q);
-                        p.Children[p.Index] = new HuffmanBranch(q.Children);
-                        p = q;
-                    }
-
-                    k++;
-                }
-
-                if (i + 1 < length)
-                {
-                    // p here points to last code
-                    q = new HuffmanBranch(-1);
-                    code.Add(q);
-                    p.Children[p.Index] = new HuffmanBranch(q.Children);
-                    p = q;
-                }
-            }
-
-            tables[index] = code[0].Children;
+            tables[index] = new HuffmanTable(codeLengths, values);
         }
 
         /// <summary>