diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FastACTables.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FastACTables.cs
new file mode 100644
index 0000000000..8d37c567ed
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FastACTables.cs
@@ -0,0 +1,34 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System;
+using SixLabors.ImageSharp.Memory;
+
+namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
+{
+    /// <summary>
+    /// The collection of tables used for fast AC entropy scan decoding.
+    /// </summary>
+    internal sealed class FastACTables : IDisposable
+    {
+        /// <summary>
+        /// Initializes a new instance of the <see cref="FastACTables"/> class.
+        /// </summary>
+        /// <param name="memoryManager">The memory manager used to allocate memory for image processing operations.</param>
+        public FastACTables(MemoryManager memoryManager)
+        {
+            this.Tables = memoryManager.AllocateClean2D<short>(512, 4);
+        }
+
+        /// <summary>
+        /// Gets the collection of tables.
+        /// </summary>
+        public Buffer2D<short> Tables { get; }
+
+        /// <inheritdoc />
+        public void Dispose()
+        {
+            this.Tables?.Dispose();
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FixedInt16Buffer257.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FixedInt16Buffer257.cs
new file mode 100644
index 0000000000..b304dbf8c2
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FixedInt16Buffer257.cs
@@ -0,0 +1,24 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
+{
+    [StructLayout(LayoutKind.Sequential)]
+    internal unsafe struct FixedInt16Buffer257
+    {
+        public fixed short Data[257];
+
+        public short this[int idx]
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get
+            {
+                ref short self = ref Unsafe.As<FixedInt16Buffer257, short>(ref this);
+                return Unsafe.Add(ref self, idx);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsFrameComponent.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsFrameComponent.cs
index 7f50a8529c..f063309eac 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsFrameComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsFrameComponent.cs
@@ -36,9 +36,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         public byte Id { get; }
 
         /// <summary>
-        /// Gets or sets Pred TODO: What does pred stand for?
+        /// Gets or sets DC coefficient predictor
         /// </summary>
-        public int Pred { get; set; }
+        public int DcPredictor { get; set; }
 
         /// <summary>
         /// Gets the horizontal sampling factor.
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
index 875a862638..0541de91b0 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
@@ -27,13 +27,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// <summary>
         /// Gets the huffman value array
         /// </summary>
-        public FixedByteBuffer256 HuffVal;
+        public FixedByteBuffer256 Values;
 
         /// <summary>
         /// Gets the lookahead array
         /// </summary>
         public FixedInt16Buffer256 Lookahead;
 
+        /// <summary>
+        /// Gets the sizes array
+        /// </summary>
+        public FixedInt16Buffer257 Sizes;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="PdfJsHuffmanTable"/> struct.
         /// </summary>
@@ -42,20 +47,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// <param name="values">The huffman values</param>
         public PdfJsHuffmanTable(MemoryManager memoryManager, ReadOnlySpan<byte> lengths, ReadOnlySpan<byte> values)
         {
-            const int length = 257;
-            using (IBuffer<short> huffsize = memoryManager.Allocate<short>(length))
-            using (IBuffer<short> huffcode = memoryManager.Allocate<short>(length))
+            const int Length = 257;
+            using (IBuffer<short> huffcode = memoryManager.Allocate<short>(Length))
             {
-                ref short huffsizeRef = ref MemoryMarshal.GetReference(huffsize.Span);
                 ref short huffcodeRef = ref MemoryMarshal.GetReference(huffcode.Span);
 
-                GenerateSizeTable(lengths, ref huffsizeRef);
-                GenerateCodeTable(ref huffsizeRef, ref huffcodeRef, length);
+                this.GenerateSizeTable(lengths);
+                this.GenerateCodeTable(ref huffcodeRef, Length);
                 this.GenerateDecoderTables(lengths, ref huffcodeRef);
                 this.GenerateLookaheadTables(lengths, values, ref huffcodeRef);
             }
 
-            fixed (byte* huffValRef = this.HuffVal.Data)
+            fixed (byte* huffValRef = this.Values.Data)
             {
                 var huffValSpan = new Span<byte>(huffValRef, 256);
 
@@ -67,45 +70,49 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// Figure C.1: make table of Huffman code length for each symbol
         /// </summary>
         /// <param name="lengths">The code lengths</param>
-        /// <param name="huffsizeRef">The huffman size span ref</param>
-        private static void GenerateSizeTable(ReadOnlySpan<byte> lengths, ref short huffsizeRef)
+        private void GenerateSizeTable(ReadOnlySpan<byte> lengths)
         {
-            short index = 0;
-            for (short l = 1; l <= 16; l++)
+            fixed (short* sizesRef = this.Sizes.Data)
             {
-                byte i = lengths[l];
-                for (short j = 0; j < i; j++)
+                short index = 0;
+                for (short l = 1; l <= 16; l++)
                 {
-                    Unsafe.Add(ref huffsizeRef, index) = l;
-                    index++;
+                    byte i = lengths[l];
+                    for (short j = 0; j < i; j++)
+                    {
+                        sizesRef[index] = l;
+                        index++;
+                    }
                 }
-            }
 
-            Unsafe.Add(ref huffsizeRef, index) = 0;
+                sizesRef[index] = 0;
+            }
         }
 
         /// <summary>
         /// Figure C.2: generate the codes themselves
         /// </summary>
-        /// <param name="huffsizeRef">The huffman size span ref</param>
         /// <param name="huffcodeRef">The huffman code span ref</param>
         /// <param name="length">The length of the huffsize span</param>
-        private static void GenerateCodeTable(ref short huffsizeRef, ref short huffcodeRef, int length)
+        private void GenerateCodeTable(ref short huffcodeRef, int length)
         {
-            short k = 0;
-            short si = huffsizeRef;
-            short code = 0;
-            for (short i = 0; i < length; i++)
+            fixed (short* sizesRef = this.Sizes.Data)
             {
-                while (Unsafe.Add(ref huffsizeRef, k) == si)
+                short k = 0;
+                short si = sizesRef[0];
+                short code = 0;
+                for (short i = 0; i < length; i++)
                 {
-                    Unsafe.Add(ref huffcodeRef, k) = code;
-                    code++;
-                    k++;
-                }
+                    while (sizesRef[k] == si)
+                    {
+                        Unsafe.Add(ref huffcodeRef, k) = code;
+                        code++;
+                        k++;
+                    }
 
-                code <<= 1;
-                si++;
+                    code <<= 1;
+                    si++;
+                }
             }
         }
 
@@ -148,6 +155,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// <param name="huffcodeRef">The huffman code span ref</param>
         private void GenerateLookaheadTables(ReadOnlySpan<byte> lengths, ReadOnlySpan<byte> huffval, ref short huffcodeRef)
         {
+            // TODO: Rewrite this to match stb_Image
             // TODO: This generation code matches the libJpeg code but the lookahead table is not actually used yet.
             // To use it we need to implement fast lookup path in PdfJsScanDecoder.DecodeHuffman
             // This should yield much faster scan decoding as usually, more than 95% of the Huffman codes
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTables.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTables.cs
index 3a559bb864..5cbde2b88c 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTables.cs
@@ -17,7 +17,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// Gets or sets the table at the given index.
         /// </summary>
         /// <param name="index">The index</param>
-        /// <returns>The <see cref="List{HuffmanBranch}"/></returns>
+        /// <returns>The <see cref="PdfJsHuffmanTable"/></returns>
         public ref PdfJsHuffmanTable this[int index]
         {
             [MethodImpl(MethodImplOptions.AggressiveInlining)]
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
index c6b14d6fb0..62c8f984f0 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
@@ -107,7 +107,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 for (int i = 0; i < components.Length; i++)
                 {
                     PdfJsFrameComponent c = components[i];
-                    c.Pred = 0;
+                    c.DcPredictor = 0;
                 }
 
                 this.eobrun = 0;
@@ -552,7 +552,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             }
 
             int j = tree.ValOffset[i];
-            value = tree.HuffVal[(j + code) & 0xFF];
+            value = tree.Values[(j + code) & 0xFF];
             return true;
         }
 
@@ -618,7 +618,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 }
             }
 
-            Unsafe.Add(ref blockDataRef, offset) = (short)(component.Pred += diff);
+            Unsafe.Add(ref blockDataRef, offset) = (short)(component.DcPredictor += diff);
 
             int k = 1;
             while (k < 64)
@@ -673,7 +673,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 }
             }
 
-            Unsafe.Add(ref blockDataRef, offset) = (short)(component.Pred += diff << this.successiveState);
+            Unsafe.Add(ref blockDataRef, offset) = (short)(component.DcPredictor += diff << this.successiveState);
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
@@ -860,5 +860,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 }
             }
         }
+
+        private void Reset()
+        {
+            // Reset
+            // TODO: I do not understand why these values are reset? We should surely be tracking the bits across mcu's?
+            this.bitsCount = 0;
+            this.bitsData = 0;
+            this.unexpectedMarkerReached = false;
+        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/ScanDecoder.cs
new file mode 100644
index 0000000000..af7233bfe8
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/ScanDecoder.cs
@@ -0,0 +1,628 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+using SixLabors.ImageSharp.Formats.Jpeg.Common;
+using SixLabors.ImageSharp.Memory;
+
+namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
+{
+    internal class ScanDecoder
+    {
+        public const int FastBits = 9;
+
+        // bmask[n] = (1 << n) - 1
+        private static readonly uint[] stbi__bmask = { 0, 1, 3, 7, 15, 31, 63, 127, 255, 511, 1023, 2047, 4095, 8191, 16383, 32767, 65535 };
+
+        // bias[n] = (-1 << n) + 1
+        private static readonly int[] stbi__jbias = { 0, -1, -3, -7, -15, -31, -63, -127, -255, -511, -1023, -2047, -4095, -8191, -16383, -32767 };
+
+        private readonly DoubleBufferedStreamReader stream;
+        private readonly PdfJsFrameComponent[] components;
+        private readonly ZigZag dctZigZag;
+        private int codeBits;
+        private uint codeBuffer;
+        private bool nomore;
+        private byte marker;
+
+        private int todo;
+        private int restartInterval;
+        private int componentIndex;
+        private int componentsLength;
+        private int eobrun;
+        private int spectralStart;
+        private int spectralEnd;
+        private int successiveHigh;
+        private int successiveLow;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="ScanDecoder"/> class.
+        /// </summary>
+        /// <param name="stream">The input stream</param>
+        /// <param name="components">The scan components</param>
+        /// <param name="componentIndex">The component index within the array</param>
+        /// <param name="componentsLength">The length of the components. Different to the array length</param>
+        /// <param name="restartInterval">The reset interval</param>
+        /// <param name="spectralStart">The spectral selection start</param>
+        /// <param name="spectralEnd">The spectral selection end</param>
+        /// <param name="successiveHigh">The successive approximation bit high end</param>
+        /// <param name="successiveLow">The successive approximation bit low end</param>
+        public ScanDecoder(
+            DoubleBufferedStreamReader stream,
+            PdfJsFrameComponent[] components,
+            int componentIndex,
+            int componentsLength,
+            int restartInterval,
+            int spectralStart,
+            int spectralEnd,
+            int successiveHigh,
+            int successiveLow)
+        {
+            this.dctZigZag = ZigZag.CreateUnzigTable();
+            this.stream = stream;
+            this.components = components;
+            this.marker = PdfJsJpegConstants.Markers.Prefix;
+            this.componentIndex = componentIndex;
+            this.componentsLength = componentsLength;
+            this.restartInterval = restartInterval;
+            this.spectralStart = spectralStart;
+            this.spectralEnd = spectralEnd;
+            this.successiveHigh = successiveHigh;
+            this.successiveLow = successiveLow;
+        }
+
+        /// <summary>
+        /// Decodes the entropy coded data.
+        /// </summary>
+        /// <param name="frame">The image frame.</param>
+        /// <param name="dcHuffmanTables">The DC Huffman tables.</param>
+        /// <param name="acHuffmanTables">The AC Huffman tables.</param>
+        /// <param name="fastACTables">The fast AC decoding tables.</param>
+        /// <returns>The <see cref="int"/></returns>
+        public int ParseEntropyCodedData(
+            PdfJsFrame frame,
+            PdfJsHuffmanTables dcHuffmanTables,
+            PdfJsHuffmanTables acHuffmanTables,
+            FastACTables fastACTables)
+        {
+            this.Reset();
+
+            if (!frame.Progressive)
+            {
+                if (this.componentsLength == 1)
+                {
+                    int i, j;
+                    int n = this.componentIndex;
+                    PdfJsFrameComponent component = this.components[n];
+
+                    // Non-interleaved data, we just need to process one block at a time,
+                    // in trivial scanline order
+                    // number of blocks to do just depends on how many actual "pixels" this
+                    // component has, independent of interleaved MCU blocking and such
+                    int w = component.WidthInBlocks;
+                    int h = component.HeightInBlocks;
+                    ref short blockDataRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<Block8x8, short>(component.SpectralBlocks.Span));
+                    ref PdfJsHuffmanTable dcHuffmanTable = ref dcHuffmanTables[component.DCHuffmanTableId];
+                    ref PdfJsHuffmanTable acHuffmanTable = ref acHuffmanTables[component.ACHuffmanTableId];
+                    Span<short> fastAC = fastACTables.Tables.GetRowSpan(component.ACHuffmanTableId);
+
+                    int mcu = 0;
+                    for (j = 0; j < h; j++)
+                    {
+                        for (i = 0; i < w; i++)
+                        {
+                            int blockRow = mcu / w;
+                            int blockCol = mcu % w;
+                            int offset = component.GetBlockBufferOffset(blockRow, blockCol);
+                            this.DecodeBlock(component, ref Unsafe.Add(ref blockDataRef, offset), ref dcHuffmanTable, ref acHuffmanTable, fastAC);
+                            mcu++;
+                        }
+                    }
+                }
+            }
+
+            return 1;
+        }
+
+        private int DecodeBlock(
+            PdfJsFrameComponent component,
+            ref short blockDataRef,
+            ref PdfJsHuffmanTable dcTable,
+            ref PdfJsHuffmanTable acTable,
+            Span<short> fac)
+        {
+            if (this.codeBits < 16)
+            {
+                this.GrowBufferUnsafe();
+            }
+
+            int t = this.DecodeHuffman(ref dcTable);
+
+            if (t < 0)
+            {
+                throw new ImageFormatException("Bad Huffman code");
+            }
+
+            int diff = t > 0 ? this.ExtendReceive(t) : 0;
+            int dc = component.DcPredictor + diff;
+            component.DcPredictor = dc;
+            blockDataRef = (short)dc;
+
+            // Decode AC Components, See Jpeg Spec
+            int k = 1;
+            do
+            {
+                int zig;
+                int s;
+
+                this.CheckBits();
+                int c = this.PeekBits();
+                int r = fac[c];
+
+                if (r > 0)
+                {
+                    // Fast AC path
+                    k += (r >> 4) & 15; // Run
+                    s = r & 15; // Combined Length
+                    this.codeBuffer <<= s;
+                    this.codeBits -= s;
+
+                    // Decode into unzigzag location
+                    zig = this.dctZigZag[k++];
+                    Unsafe.Add(ref blockDataRef, zig) = (short)(r >> 8);
+                }
+                else
+                {
+                    int rs = this.DecodeHuffman(ref acTable);
+
+                    if (rs < 0)
+                    {
+                        throw new ImageFormatException("Bad Huffman code");
+                    }
+
+                    s = rs & 15;
+                    r = rs >> 4;
+
+                    if (s == 0)
+                    {
+                        if (rs != 0xF0)
+                        {
+                            break; // End block
+                        }
+
+                        k += 16;
+                    }
+                    else
+                    {
+                        k += r;
+
+                        // Decode into unzigzag location
+                        zig = this.dctZigZag[k++];
+                        Unsafe.Add(ref blockDataRef, zig) = (short)this.ExtendReceive(s);
+                    }
+                }
+            } while (k < 64);
+
+            return 1;
+        }
+
+        private int DecodeBlockProgressiveDC(
+            PdfJsFrameComponent component,
+            ref short blockDataRef,
+            ref PdfJsHuffmanTable dcTable)
+        {
+            if (this.spectralEnd != 0)
+            {
+                throw new ImageFormatException("Can't merge DC and AC.");
+            }
+
+            this.CheckBits();
+
+            if (this.successiveHigh == 0)
+            {
+                // First scan for DC coefficient, must be first
+                int t = this.DecodeHuffman(ref dcTable);
+                int diff = t > 0 ? this.ExtendReceive(t) : 0;
+
+                int dc = component.DcPredictor + diff;
+                component.DcPredictor = dc;
+
+                blockDataRef = (short)(dc << this.successiveLow);
+            }
+            else
+            {
+                // Refinement scan for DC coefficient
+                if (this.GetBit() > 0)
+                {
+                    blockDataRef += (short)(1 << this.successiveLow);
+                }
+            }
+
+            return 1;
+        }
+
+        private int DecodeBlockProgressiveAC(
+            PdfJsFrameComponent component,
+            ref short blockDataRef,
+            ref PdfJsHuffmanTable acTable,
+            Span<short> fac)
+        {
+            int k;
+
+            if (this.spectralStart == 0)
+            {
+                throw new ImageFormatException("Can't merge DC and AC.");
+            }
+
+            if (this.successiveHigh == 0)
+            {
+                int shift = this.successiveLow;
+
+                if (this.eobrun > 0)
+                {
+                    this.eobrun--;
+                    return 1;
+                }
+
+                k = this.spectralStart;
+                do
+                {
+                    int zig;
+                    int s;
+
+                    this.CheckBits();
+                    int c = this.PeekBits();
+                    int r = fac[c];
+
+                    if (r > 0)
+                    {
+                        // Fast AC path
+                        k += (r >> 4) & 15; // Run
+                        s = r & 15; // Combined length
+                        this.codeBuffer <<= s;
+                        this.codeBits -= s;
+
+                        // Decode into unzigzag location
+                        zig = this.dctZigZag[k++];
+                        Unsafe.Add(ref blockDataRef, zig) = (short)((r >> 8) << shift);
+                    }
+                    else
+                    {
+                        int rs = this.DecodeHuffman(ref acTable);
+
+                        if (rs < 0)
+                        {
+                            throw new ImageFormatException("Bad Huffman code.");
+                        }
+
+                        s = rs & 15;
+                        r = rs >> 4;
+
+                        if (s == 0)
+                        {
+                            if (r < 15)
+                            {
+                                this.eobrun = 1 << r;
+                                if (r > 0)
+                                {
+                                    this.eobrun += this.GetBits(r);
+                                }
+
+                                this.eobrun--;
+                                break;
+                            }
+
+                            k += 16;
+                        }
+                        else
+                        {
+                            k += r;
+                            zig = this.dctZigZag[k++];
+                            Unsafe.Add(ref blockDataRef, zig) = (short)(this.ExtendReceive(s) << shift);
+                        }
+                    }
+                }
+                while (k <= this.spectralEnd);
+            }
+            else
+            {
+                // Refinement scan for these AC coefficients
+                short bit = (short)(1 << this.successiveLow);
+
+                if (this.eobrun > 0)
+                {
+                    this.eobrun--;
+                    for (k = this.spectralStart; k < this.spectralEnd; k++)
+                    {
+                        ref short p = ref Unsafe.Add(ref blockDataRef, this.dctZigZag[k]);
+                        if (p != 0)
+                        {
+                            if (this.GetBit() > 0)
+                            {
+                                if ((p & bit) == 0)
+                                {
+                                    if (p > 0)
+                                    {
+                                        p += bit;
+                                    }
+                                    else
+                                    {
+                                        p -= bit;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                else
+                {
+                    k = this.spectralStart;
+                    do
+                    {
+                        int rs = this.DecodeHuffman(ref acTable);
+                        if (rs < 0)
+                        {
+                            throw new ImageFormatException("Bad Huffman code.");
+                        }
+
+                        int s = rs & 15;
+                        int r = rs >> 4;
+
+                        if (s == 0)
+                        {
+                            // r=15 s=0 should write 16 0s, so we just do
+                            // a run of 15 0s and then write s (which is 0),
+                            // so we don't have to do anything special here
+                            if (r < 15)
+                            {
+                                this.eobrun = (1 << r) - 1;
+
+                                if (r > 0)
+                                {
+                                    this.eobrun += this.GetBits(r);
+                                }
+
+                                r = 64; // Force end of block
+                            }
+                        }
+                        else
+                        {
+                            if (s != 1)
+                            {
+                                throw new ImageFormatException("Bad Huffman code.");
+                            }
+
+                            // Sign bit
+                            if (this.GetBit() > 0)
+                            {
+                                s = bit;
+                            }
+                            else
+                            {
+                                s -= bit;
+                            }
+                        }
+
+                        // Advance by r
+                        while (k <= this.spectralEnd)
+                        {
+                            ref short p = ref Unsafe.Add(ref blockDataRef, this.dctZigZag[k++]);
+                            if (p != 0)
+                            {
+                                if (this.GetBit() > 0)
+                                {
+                                    if ((p & bit) == 0)
+                                    {
+                                        if (p > 0)
+                                        {
+                                            p += bit;
+                                        }
+                                        else
+                                        {
+                                            p -= bit;
+                                        }
+                                    }
+                                }
+                            }
+                            else
+                            {
+                                if (r == 0)
+                                {
+                                    p = (short)s;
+                                    break;
+                                }
+
+                                r--;
+                            }
+                        }
+                    }
+                    while (k <= this.spectralEnd);
+                }
+            }
+
+            return 1;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetBits(int n)
+        {
+            if (this.codeBits < n)
+            {
+                this.GrowBufferUnsafe();
+            }
+
+            uint k = this.Lrot(this.codeBuffer, n);
+            this.codeBuffer = k & ~stbi__bmask[n];
+            k &= stbi__bmask[n];
+            this.codeBits -= n;
+            return (int)k;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int GetBit()
+        {
+            if (this.codeBits < 1)
+            {
+                this.GrowBufferUnsafe();
+            }
+
+            uint k = this.codeBuffer;
+            this.codeBuffer <<= 1;
+            this.codeBits--;
+
+            return (int)(k & 0x80000000);
+        }
+
+        private void GrowBufferUnsafe()
+        {
+            do
+            {
+                // TODO: EOF
+                uint b = (uint)(this.nomore ? 0 : this.stream.ReadByte());
+                if (b == PdfJsJpegConstants.Markers.Prefix)
+                {
+                    long position = this.stream.Position - 1;
+                    int c = this.stream.ReadByte();
+                    while (c == PdfJsJpegConstants.Markers.Prefix)
+                    {
+                        if (c != 0)
+                        {
+                            this.marker = (byte)c;
+                            this.nomore = true;
+                            this.stream.Position = position;
+                            return;
+                        }
+                    }
+                }
+
+                this.codeBuffer |= b << (24 - this.codeBits);
+                this.codeBits += 8;
+            }
+            while (this.codeBits <= 24);
+        }
+
+        private int DecodeHuffman(ref PdfJsHuffmanTable table)
+        {
+            this.CheckBits();
+
+            // Look at the top FastBits and determine what symbol ID it is,
+            // if the code is <= FastBits.
+            int c = this.PeekBits();
+            int k = table.Lookahead[c];
+            if (k < byte.MaxValue)
+            {
+                int s = table.Sizes[k];
+                if (s > this.codeBits)
+                {
+                    return -1;
+                }
+
+                this.codeBuffer <<= s;
+                this.codeBits -= s;
+                return table.Values[k];
+            }
+
+            // Naive test is to shift the code_buffer down so k bits are
+            // valid, then test against MaxCode. To speed this up, we've
+            // preshifted maxcode left so that it has (16-k) 0s at the
+            // end; in other words, regardless of the number of bits, it
+            // wants to be compared against something shifted to have 16;
+            // that way we don't need to shift inside the loop.
+            uint temp = this.codeBuffer >> 16;
+            for (k = FastBits + 1; ; ++k)
+            {
+                if (temp < table.MaxCode[k])
+                {
+                    break;
+                }
+            }
+
+            if (k == 17)
+            {
+                // Error! code not found
+                this.codeBits -= 16;
+                return -1;
+            }
+
+            if (k > this.codeBits)
+            {
+                return -1;
+            }
+
+            // Convert the huffman code to the symbol id
+            c = (int)((this.codeBuffer >> (32 - k)) & stbi__bmask[k]) + table.ValOffset[k];
+
+            // Convert the id to a symbol
+            this.codeBits -= k;
+            this.codeBuffer <<= k;
+            return table.Values[c];
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int ExtendReceive(int n)
+        {
+            if (this.codeBits < n)
+            {
+                this.GrowBufferUnsafe();
+            }
+
+            int sgn = (int)(this.codeBuffer >> 31);
+            uint k = this.Lrot(this.codeBuffer, n);
+            this.codeBuffer = k & ~stbi__bmask[n];
+            k &= stbi__bmask[n];
+            this.codeBits -= n;
+            return (int)(k + (stbi__jbias[n] & ~sgn));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void CheckBits()
+        {
+            if (this.codeBuffer < 16)
+            {
+                this.GrowBufferUnsafe();
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private int PeekBits()
+        {
+            return (int)(this.codeBuffer >> ((32 - FastBits) & ((1 << FastBits) - 1)));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private uint Lrot(uint x, int y)
+        {
+            return (x << y) | (x >> (32 - y));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool IsRestartMarker(byte x)
+        {
+            return x >= PdfJsJpegConstants.Markers.RST0 && x <= PdfJsJpegConstants.Markers.RST7;
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void Reset()
+        {
+            this.codeBits = 0;
+            this.codeBuffer = 0;
+            this.nomore = false;
+
+            for (int i = 0; i < this.components.Length; i++)
+            {
+                PdfJsFrameComponent c = this.components[i];
+                c.DcPredictor = 0;
+            }
+
+            this.marker = PdfJsJpegConstants.Markers.Prefix;
+            this.todo = this.restartInterval > 0 ? this.restartInterval : 0x7FFFFFFF;
+            this.eobrun = 0;
+
+            // No more than 1<<31 MCUs if no restartInterval? that's plenty safe,
+            // since we don't even allow 1<<30 pixels
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
index df803a9202..18a444c5bd 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs
@@ -57,6 +57,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
         /// </summary>
         private PdfJsHuffmanTables acHuffmanTables;
 
+        /// <summary>
+        /// The fast AC tables used for entropy decoding
+        /// </summary>
+        private FastACTables fastACTables;
+
         /// <summary>
         /// The reset interval determined by RST markers
         /// </summary>
@@ -228,6 +233,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
                 this.QuantizationTables = new Block8x8F[4];
                 this.dcHuffmanTables = new PdfJsHuffmanTables();
                 this.acHuffmanTables = new PdfJsHuffmanTables();
+                this.fastACTables = new FastACTables(this.configuration.MemoryManager);
             }
 
             while (fileMarker.Marker != PdfJsJpegConstants.Markers.EOI)
@@ -341,12 +347,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
         {
             this.InputStream?.Dispose();
             this.Frame?.Dispose();
+            this.fastACTables?.Dispose();
 
             // Set large fields to null.
             this.InputStream = null;
             this.Frame = null;
             this.dcHuffmanTables = null;
             this.acHuffmanTables = null;
+            this.fastACTables = null;
         }
 
         /// <summary>
@@ -714,11 +722,20 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
 
                             i += 17 + codeLengthSum;
 
+                            int tableType = huffmanTableSpec >> 4;
+                            int tableIndex = huffmanTableSpec & 15;
+
                             this.BuildHuffmanTable(
-                                huffmanTableSpec >> 4 == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
-                                huffmanTableSpec & 15,
+                                tableType == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
+                                tableIndex,
                                 codeLengths.Span,
                                 huffmanValues.Span);
+
+                            if (tableType != 0)
+                            {
+                                // Build a table that decodes both magnitude and value of small ACs in one go.
+                                this.BuildFastACTable(tableIndex);
+                            }
                         }
                     }
                 }
@@ -829,5 +846,43 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
                 return image;
             }
         }
+
+        private void BuildFastACTable(int index)
+        {
+            const int FastBits = ScanDecoder.FastBits;
+            Span<short> fastac = this.fastACTables.Tables.GetRowSpan(index);
+            ref PdfJsHuffmanTable huffman = ref this.acHuffmanTables[index];
+
+            int i;
+            for (i = 0; i < (1 << FastBits); i++)
+            {
+                short fast = huffman.Lookahead[i];
+                fastac[i] = 0;
+                if (fast < 255)
+                {
+                    int rs = huffman.Values[fast];
+                    int run = (rs >> 4) & 15;
+                    int magbits = rs & 15;
+                    int len = huffman.Sizes[fast];
+
+                    if (magbits > 0 && len + magbits <= FastBits)
+                    {
+                        // Magnitude code followed by receive_extend code
+                        int k = ((i << len) & ((1 << FastBits) - 1)) >> (FastBits - magbits);
+                        int m = 1 << (magbits - 1);
+                        if (k < m)
+                        {
+                            k += (int)((~0U << magbits) + 1);
+                        }
+
+                        // if the result is small enough, we can fit it in fastac table
+                        if (k >= -128 && k <= 127)
+                        {
+                            fastac[i] = (short)((k * 256) + (run * 16) + (len + magbits));
+                        }
+                    }
+                }
+            }
+        }
     }
 }
\ No newline at end of file