diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
index bc31629785..0119f272cf 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/HuffmanTable.cs
@@ -15,6 +15,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
     /// </summary>
     internal struct HuffmanTable : IDisposable
     {
+        private Buffer<short> lookahead;
         private Buffer<short> huffcode;
         private Buffer<short> huffsize;
         private Buffer<short> valOffset;
@@ -30,6 +31,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// <param name="values">The huffman values</param>
         public HuffmanTable(byte[] lengths, byte[] values)
         {
+            this.lookahead = Buffer<short>.CreateClean(256);
             this.huffcode = Buffer<short>.CreateClean(257);
             this.huffsize = Buffer<short>.CreateClean(257);
             this.valOffset = Buffer<short>.CreateClean(18);
@@ -44,6 +46,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
             this.GenerateSizeTable();
             this.GenerateCodeTable();
             this.GenerateDecoderTables();
+            this.GenerateLookaheadTables();
         }
 
         /// <summary>
@@ -74,14 +77,26 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// <param name="i">The index</param>
         /// <returns>The <see cref="int"/></returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int GetValPtr(int i)
+        public int GetValOffset(int i)
         {
             return this.valOffset[i];
         }
 
+        /// <summary>
+        /// Gets the look ahead table balue
+        /// </summary>
+        /// <param name="i">The index</param>
+        /// <returns>The <see cref="int"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int GetLookAhead(int i)
+        {
+            return this.lookahead[i];
+        }
+
         /// <inheritdoc/>
         public void Dispose()
         {
+            this.lookahead?.Dispose();
             this.huffcode?.Dispose();
             this.huffsize?.Dispose();
             this.valOffset?.Dispose();
@@ -89,6 +104,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
             this.huffval?.Dispose();
             this.bits?.Dispose();
 
+            this.lookahead = null;
             this.huffcode = null;
             this.huffsize = null;
             this.valOffset = null;
@@ -163,5 +179,37 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
             this.valOffset[17] = 0;
             this.maxcode[17] = 0xFFFFFL;
         }
+
+        /// <summary>
+        /// Generates lookup tables to speed up decoding
+        /// </summary>
+        private void GenerateLookaheadTables()
+        {
+            int x = 0, code = 0;
+
+            for (int i = 0; i < 8; i++)
+            {
+                code <<= 1;
+
+                for (int j = 0; j < this.bits[i + 1]; j++)
+                {
+                    // The codeLength is 1+i, so shift code by 8-(1+i) to
+                    // calculate the high bits for every 8-bit sequence
+                    // whose codeLength's high bits matches code.
+                    // The high 8 bits of lutValue are the encoded value.
+                    // The low 8 bits are 1 plus the codeLength.
+                    int base2 = code << (7 - i);
+                    int lutValue = (this.huffval[x] << 8) | (2 + i);
+
+                    for (int k = 0; k < 1 << (7 - i); k++)
+                    {
+                        this.lookahead[base2 | k] = (short)lutValue;
+                    }
+
+                    code++;
+                    x++;
+                }
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
index 62b0a82e4c..140f474495 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/ScanDecoder.cs
@@ -23,6 +23,8 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
 
         private int bitsCount;
 
+        private int accumulator;
+
         private int specStart;
 
         private int specEnd;
@@ -584,20 +586,34 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
             }
 
             this.bitsCount = 7;
+
+            // TODO: This line is incorrect.
+            this.accumulator = (this.accumulator << 8) | this.bitsData;
+
             return this.bitsData >> 7;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private short DecodeHuffman(HuffmanTable tree, Stream stream)
         {
-            // "DECODE", section F.2.2.3, figure F.16, page 109 of T.81
-            int i = 1;
             short code = (short)this.ReadBit(stream);
             if (this.endOfStreamReached || this.unexpectedMarkerReached)
             {
                 return -1;
             }
 
+            // TODO: If the following is enabled the decoder breaks.
+            // if (this.bitsCount > 0)
+            // {
+            //    int lutIndex = (this.accumulator >> (this.bitsCount - 7)) & 0xFF;
+            //    int v = tree.GetLookAhead(lutIndex);
+            //    if (v != 0)
+            //    {
+            //        return (short)(v >> 8);
+            //    }
+            // }
+            // "DECODE", section F.2.2.3, figure F.16, page 109 of T.81
+            int i = 1;
             while (code > tree.GetMaxCode(i))
             {
                 code <<= 1;
@@ -611,7 +627,7 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
                 i++;
             }
 
-            int j = tree.GetValPtr(i);
+            int j = tree.GetValOffset(i);
             return tree.GetHuffVal((j + code) & 0xFF);
         }