diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
index 1eea1417f3..80787101b0 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
@@ -18,13 +18,13 @@ namespace ImageSharp.Formats.Jpg
         /// <summary>
         /// Gets or sets the accumulator.
         /// </summary>
-        public uint Accumulator;
+        public int Accumulator;
 
         /// <summary>
         /// Gets or sets the mask.
         /// <![CDATA[mask==1<<(unreadbits-1) when unreadbits>0, with mask==0 when unreadbits==0.]]>
         /// </summary>
-        public uint Mask;
+        public int Mask;
 
         /// <summary>
         /// Gets or sets the  number of unread bits in the accumulator.
@@ -89,7 +89,7 @@ namespace ImageSharp.Formats.Jpg
 
         private DecoderErrorCode EnsureBitsStepImpl(JpegDecoderCore decoder)
         {
-            byte c;
+            int c;
             DecoderErrorCode errorCode = decoder.Bytes.ReadByteStuffedByteUnsafe(decoder.InputStream, out c);
 
             if (errorCode != DecoderErrorCode.NoError)
@@ -117,7 +117,7 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="decoder">Jpeg decoder</param>
         /// <returns>Read bits value</returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int ReceiveExtend(byte t, JpegDecoderCore decoder)
+        public int ReceiveExtend(int t, JpegDecoderCore decoder)
         {
             int x;
             DecoderErrorCode errorCode = this.ReceiveExtendUnsafe(t, decoder, out x);
@@ -132,7 +132,7 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="decoder">Jpeg decoder</param>
         /// <param name="x">Read bits value</param>
         /// <returns>The <see cref="DecoderErrorCode"/></returns>
-        public DecoderErrorCode ReceiveExtendUnsafe(byte t, JpegDecoderCore decoder, out int x)
+        public DecoderErrorCode ReceiveExtendUnsafe(int t, JpegDecoderCore decoder, out int x)
         {
             if (this.UnreadBits < t)
             {
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
index 27f902aa15..b65df51dc4 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
@@ -16,6 +16,11 @@ namespace ImageSharp.Formats.Jpg
     /// </summary>
     internal struct Bytes : IDisposable
     {
+        /// <summary>
+        /// Specifies the buffer size for <see cref="Buffer"/> and <see cref="BufferAsInt"/>
+        /// </summary>
+        public const int BufferSize = 4096;
+
         /// <summary>
         /// Gets or sets the buffer.
         /// buffer[i:j] are the buffered bytes read from the underlying
@@ -23,6 +28,11 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         public byte[] Buffer;
 
+        /// <summary>
+        /// Values of <see cref="Buffer"/> converted to <see cref="int"/>-s
+        /// </summary>
+        public int[] BufferAsInt;
+
         /// <summary>
         /// Start of bytes read
         /// </summary>
@@ -39,7 +49,9 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         public int UnreadableBytes;
 
-        private static readonly ArrayPool<byte> ArrayPool = ArrayPool<byte>.Create(4096, 50);
+        private static readonly ArrayPool<byte> BytePool = ArrayPool<byte>.Create(BufferSize, 50);
+
+        private static readonly ArrayPool<int> IntPool = ArrayPool<int>.Create(BufferSize, 50);
 
         /// <summary>
         /// Creates a new instance of the <see cref="Bytes"/>, and initializes it's buffer.
@@ -47,7 +59,11 @@ namespace ImageSharp.Formats.Jpg
         /// <returns>The bytes created</returns>
         public static Bytes Create()
         {
-            return new Bytes { Buffer = ArrayPool.Rent(4096) };
+            return new Bytes
+                       {
+                           Buffer = BytePool.Rent(BufferSize),
+                           BufferAsInt = IntPool.Rent(BufferSize)
+                       };
         }
 
         /// <summary>
@@ -57,32 +73,34 @@ namespace ImageSharp.Formats.Jpg
         {
             if (this.Buffer != null)
             {
-                ArrayPool.Return(this.Buffer);
+                BytePool.Return(this.Buffer);
+                IntPool.Return(this.BufferAsInt);
             }
 
             this.Buffer = null;
+            this.BufferAsInt = null;
         }
 
         /// <summary>
         /// ReadByteStuffedByte is like ReadByte but is for byte-stuffed Huffman data.
         /// </summary>
         /// <param name="inputStream">Input stream</param>
-        /// <param name="x">The result <see cref="byte"/></param>
+        /// <param name="x">The result byte as <see cref="int"/></param>
         /// <returns>The <see cref="DecoderErrorCode"/></returns>
-        public DecoderErrorCode ReadByteStuffedByteUnsafe(Stream inputStream, out byte x)
+        public DecoderErrorCode ReadByteStuffedByteUnsafe(Stream inputStream, out int x)
         {
             // Take the fast path if bytes.buf contains at least two bytes.
             if (this.I + 2 <= this.J)
             {
-                x = this.Buffer[this.I];
+                x = this.BufferAsInt[this.I];
                 this.I++;
                 this.UnreadableBytes = 1;
-                if (x != JpegConstants.Markers.XFF)
+                if (x != JpegConstants.Markers.XFFInt)
                 {
                     return DecoderErrorCode.NoError;
                 }
 
-                if (this.Buffer[this.I] != 0x00)
+                if (this.BufferAsInt[this.I] != 0x00)
                 {
                     return DecoderErrorCode.MissingFF00;
                 }
@@ -95,7 +113,7 @@ namespace ImageSharp.Formats.Jpg
 
             this.UnreadableBytes = 0;
 
-            DecoderErrorCode errorCode = this.ReadByteUnsafe(inputStream, out x);
+            DecoderErrorCode errorCode = this.ReadByteAsIntUnsafe(inputStream, out x);
             this.UnreadableBytes = 1;
             if (errorCode != DecoderErrorCode.NoError)
             {
@@ -107,7 +125,7 @@ namespace ImageSharp.Formats.Jpg
                 return DecoderErrorCode.NoError;
             }
 
-            errorCode = this.ReadByteUnsafe(inputStream, out x);
+            errorCode = this.ReadByteAsIntUnsafe(inputStream, out x);
             this.UnreadableBytes = 2;
             if (errorCode != DecoderErrorCode.NoError)
             {
@@ -164,6 +182,25 @@ namespace ImageSharp.Formats.Jpg
             return errorCode;
         }
 
+        public DecoderErrorCode ReadByteAsIntUnsafe(Stream inputStream, out int result)
+        {
+            DecoderErrorCode errorCode = DecoderErrorCode.NoError;
+            while (this.I == this.J)
+            {
+                errorCode = this.FillUnsafe(inputStream);
+                if (errorCode != DecoderErrorCode.NoError)
+                {
+                    result = 0;
+                    return errorCode;
+                }
+            }
+
+            result = this.BufferAsInt[this.I];
+            this.I++;
+            this.UnreadableBytes = 0;
+            return errorCode;
+        }
+
         /// <summary>
         /// Fills up the bytes buffer from the underlying stream.
         /// It should only be called when there are no unread bytes in bytes.
@@ -211,6 +248,12 @@ namespace ImageSharp.Formats.Jpg
             }
 
             this.J += n;
+
+            for (int i = 0; i < this.Buffer.Length; i++)
+            {
+                this.BufferAsInt[i] = this.Buffer[i];
+            }
+
             return DecoderErrorCode.NoError;
         }
     }
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
index 84f7579c8a..9ce5ea4146 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
@@ -81,6 +81,15 @@ namespace ImageSharp.Formats.Jpg
             {
                 throw new ImageFormatException("Bad Huffman code!");
             }
+
+            /// <summary>
+            /// Throws "Uninitialized Huffman table".
+            /// </summary>
+            [MethodImpl(MethodImplOptions.NoInlining)]
+            public static void UninitializedHuffmanTable()
+            {
+                throw new ImageFormatException("Uninitialized Huffman table");
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/HuffmanTree.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/HuffmanTree.cs
index e06d644a7f..55cf8c77f6 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/HuffmanTree.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/HuffmanTree.cs
@@ -45,7 +45,7 @@ namespace ImageSharp.Formats.Jpg
         /// <summary>
         /// The log-2 size of the Huffman decoder's look-up table.
         /// </summary>
-        public const int LutSize = 8;
+        public const int LutSizeLog2 = 8;
 
         /// <summary>
         /// Gets or sets the number of codes in the tree.
@@ -58,13 +58,18 @@ namespace ImageSharp.Formats.Jpg
         /// are 1 plus the code length, or 0 if the value is too large to fit in
         /// lutSize bits.
         /// </summary>
-        public ushort[] Lut;
+        public int[] Lut;
 
         /// <summary>
         /// Gets the the decoded values, sorted by their encoding.
         /// </summary>
         public byte[] Values;
 
+        /// <summary>
+        /// Same as <see cref="Values"/>, converted to int-s
+        /// </summary>
+        public int[] ValuesAsInt;
+
         /// <summary>
         /// Gets the array of minimum codes.
         /// MinCodes[i] is the minimum code of length i, or -1 if there are no codes of that length.
@@ -82,11 +87,11 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         public int[] Indices;
 
-        private static readonly ArrayPool<ushort> UshortBuffer = ArrayPool<ushort>.Create(1 << LutSize, 50);
+        private static readonly ArrayPool<int> IntBuffer256 = ArrayPool<int>.Create(MaxNCodes, 50);
 
-        private static readonly ArrayPool<byte> ByteBuffer = ArrayPool<byte>.Create(MaxNCodes, 50);
+        private static readonly ArrayPool<byte> ByteBuffer256 = ArrayPool<byte>.Create(MaxNCodes, 50);
 
-        private static readonly ArrayPool<int> IntBuffer = ArrayPool<int>.Create(MaxCodeLength, 50);
+        private static readonly ArrayPool<int> CodesBuffer16 = ArrayPool<int>.Create(MaxCodeLength, 50);
 
         /// <summary>
         /// Creates and initializes an array of <see cref="HuffmanTree" /> instances of size <see cref="NumberOfTrees" />
@@ -111,11 +116,12 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         public void Dispose()
         {
-            UshortBuffer.Return(this.Lut, true);
-            ByteBuffer.Return(this.Values, true);
-            IntBuffer.Return(this.MinCodes, true);
-            IntBuffer.Return(this.MaxCodes, true);
-            IntBuffer.Return(this.Indices, true);
+            IntBuffer256.Return(this.Lut, true);
+            IntBuffer256.Return(this.ValuesAsInt, true);
+            ByteBuffer256.Return(this.Values, true);
+            CodesBuffer16.Return(this.MinCodes, true);
+            CodesBuffer16.Return(this.MaxCodes, true);
+            CodesBuffer16.Return(this.Indices, true);
         }
 
         /// <summary>
@@ -159,15 +165,20 @@ namespace ImageSharp.Formats.Jpg
 
             decoder.ReadFull(this.Values, 0, this.Length);
 
+            for (int i = 0; i < this.Values.Length; i++)
+            {
+                this.ValuesAsInt[i] = this.Values[i];
+            }
+
             // Derive the look-up table.
             for (int i = 0; i < this.Lut.Length; i++)
             {
                 this.Lut[i] = 0;
             }
 
-            uint x = 0, code = 0;
+            int x = 0, code = 0;
 
-            for (int i = 0; i < LutSize; i++)
+            for (int i = 0; i < LutSizeLog2; i++)
             {
                 code <<= 1;
 
@@ -178,8 +189,8 @@ namespace ImageSharp.Formats.Jpg
                     // whose codeLength's high bits matches code.
                     // The high 8 bits of lutValue are the encoded value.
                     // The low 8 bits are 1 plus the codeLength.
-                    byte base2 = (byte)(code << (7 - i));
-                    ushort lutValue = (ushort)((this.Values[x] << 8) | (2 + i));
+                    int base2 = (code << (7 - i));
+                    int lutValue = (this.ValuesAsInt[x] << 8) | (2 + i);
 
                     for (int k = 0; k < 1 << (7 - i); k++)
                     {
@@ -215,16 +226,22 @@ namespace ImageSharp.Formats.Jpg
             }
         }
 
+        public int GetValue(int code, int i)
+        {
+            return this.ValuesAsInt[this.Indices[i] + code - this.MinCodes[i]];
+        }
+
         /// <summary>
         /// Initializes the Huffman tree
         /// </summary>
         private void Init()
         {
-            this.Lut = UshortBuffer.Rent(1 << LutSize);
-            this.Values = ByteBuffer.Rent(MaxNCodes);
-            this.MinCodes = IntBuffer.Rent(MaxCodeLength);
-            this.MaxCodes = IntBuffer.Rent(MaxCodeLength);
-            this.Indices = IntBuffer.Rent(MaxCodeLength);
+            this.Lut = IntBuffer256.Rent(MaxNCodes);
+            this.Values = ByteBuffer256.Rent(MaxNCodes);
+            this.ValuesAsInt = IntBuffer256.Rent(MaxNCodes);
+            this.MinCodes = CodesBuffer16.Rent(MaxCodeLength);
+            this.MaxCodes = CodesBuffer16.Rent(MaxCodeLength);
+            this.Indices = CodesBuffer16.Rent(MaxCodeLength);
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
index 4dcf6def8f..86c558af7b 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
@@ -85,7 +85,7 @@ namespace ImageSharp.Formats.Jpg
         /// <summary>
         /// End-of-Band run, specified in section G.1.2.2.
         /// </summary>
-        private ushort eobRun;
+        private int eobRun;
 
         /// <summary>
         /// Pointers to elements of <see cref="data"/>
@@ -349,7 +349,7 @@ namespace ImageSharp.Formats.Jpg
                     zig++;
 
                     // Decode the DC coefficient, as specified in section F.2.2.1.
-                    byte value;
+                    int value;
                     int huffmanIndex = (DcTableIndex * HuffmanTree.ThRowSize) + this.pointers.ComponentScan[scanIndex].DcTableSelector;
                     errorCode = decoder.DecodeHuffmanUnsafe(
                             ref decoder.HuffmanTrees[huffmanIndex],
@@ -378,12 +378,12 @@ namespace ImageSharp.Formats.Jpg
                     // Decode the AC coefficients, as specified in section F.2.2.2.
                     for (; zig <= this.zigEnd; zig++)
                     {
-                        byte value;
+                        int value;
                         errorCode = decoder.DecodeHuffmanUnsafe(ref decoder.HuffmanTrees[huffmannIdx], out value);
                         errorCode.EnsureNoEOF();
 
-                        byte val0 = (byte)(value >> 4);
-                        byte val1 = (byte)(value & 0x0f);
+                        int val0 = value >> 4;
+                        int val1 = value & 0x0f;
                         if (val1 != 0)
                         {
                             zig += val0;
@@ -424,14 +424,14 @@ namespace ImageSharp.Formats.Jpg
 
         private DecoderErrorCode DecodeEobRun(int count, JpegDecoderCore decoder)
         {
-            uint bitsResult;
+            int bitsResult;
             DecoderErrorCode errorCode = decoder.DecodeBitsUnsafe(count, out bitsResult);
             if (errorCode != DecoderErrorCode.NoError)
             {
                 return errorCode;
             }
 
-            this.eobRun |= (ushort)bitsResult;
+            this.eobRun |= bitsResult;
             return DecoderErrorCode.NoError;
         }
 
@@ -548,7 +548,7 @@ namespace ImageSharp.Formats.Jpg
                     bool done = false;
                     int z = 0;
 
-                    byte val;
+                    int val;
                     DecoderErrorCode errorCode = decoder.DecodeHuffmanUnsafe(ref h, out val);
                     errorCode.EnsureNoEOF();
 
@@ -560,7 +560,7 @@ namespace ImageSharp.Formats.Jpg
                         case 0:
                             if (val0 != 0x0f)
                             {
-                                this.eobRun = (ushort)(1 << val0);
+                                this.eobRun = 1 << val0;
                                 if (val0 != 0)
                                 {
                                     errorCode = this.DecodeEobRun(val0, decoder);
diff --git a/src/ImageSharp.Formats.Jpeg/JpegConstants.cs b/src/ImageSharp.Formats.Jpeg/JpegConstants.cs
index 19d726e708..74f9a3c07d 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegConstants.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegConstants.cs
@@ -86,6 +86,11 @@ namespace ImageSharp.Formats
             /// </summary>
             public const byte XFF = 0xff;
 
+            /// <summary>
+            /// Same as <see cref="XFF"/> but of type <see cref="int"/>
+            /// </summary>
+            public const int XFFInt = XFF;
+
             /// <summary>
             /// Start of Image
             /// </summary>
diff --git a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
index 1a039191e4..64f19a3a29 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
@@ -307,7 +307,7 @@ namespace ImageSharp.Formats
         /// <param name="count">The number of bits to decode.</param>
         /// <param name="result">The <see cref="uint" /> result</param>
         /// <returns>The <see cref="DecoderErrorCode"/></returns>
-        public DecoderErrorCode DecodeBitsUnsafe(int count, out uint result)
+        public DecoderErrorCode DecodeBitsUnsafe(int count, out int result)
         {
             if (this.Bits.UnreadBits < count)
             {
@@ -315,7 +315,7 @@ namespace ImageSharp.Formats
             }
 
             result = this.Bits.Accumulator >> (this.Bits.UnreadBits - count);
-            result = (uint)(result & ((1 << count) - 1));
+            result = result & ((1 << count) - 1);
             this.Bits.UnreadBits -= count;
             this.Bits.Mask >>= count;
             return DecoderErrorCode.NoError;
@@ -327,14 +327,13 @@ namespace ImageSharp.Formats
         /// <param name="huffmanTree">The huffman value</param>
         /// <param name="result">The decoded <see cref="byte" /></param>
         /// <returns>The <see cref="DecoderErrorCode"/></returns>
-        public DecoderErrorCode DecodeHuffmanUnsafe(ref HuffmanTree huffmanTree, out byte result)
+        public DecoderErrorCode DecodeHuffmanUnsafe(ref HuffmanTree huffmanTree, out int result)
         {
             result = 0;
 
-            // Copy stuff to the stack:
             if (huffmanTree.Length == 0)
             {
-                throw new ImageFormatException("Uninitialized Huffman table");
+                DecoderThrowHelper.ThrowImageFormatException.UninitializedHuffmanTable();
             }
 
             if (this.Bits.UnreadBits < 8)
@@ -344,14 +343,15 @@ namespace ImageSharp.Formats
 
                 if (errorCode == DecoderErrorCode.NoError)
                 {
-                    ushort v = huffmanTree.Lut[(this.Bits.Accumulator >> (this.Bits.UnreadBits - HuffmanTree.LutSize)) & 0xFF];
+                    int lutIndex = (this.Bits.Accumulator >> (this.Bits.UnreadBits - HuffmanTree.LutSizeLog2)) & 0xFF;
+                    int v = huffmanTree.Lut[lutIndex];
 
                     if (v != 0)
                     {
                         int n = (v & 0xFF) - 1;
                         this.Bits.UnreadBits -= n;
                         this.Bits.Mask >>= n;
-                        result = (byte)(v >> 8);
+                        result = v >> 8;
                         return errorCode;
                     }
                 }
@@ -380,7 +380,8 @@ namespace ImageSharp.Formats
 
                 if (code <= huffmanTree.MaxCodes[i])
                 {
-                    result = huffmanTree.Values[huffmanTree.Indices[i] + code - huffmanTree.MinCodes[i]];
+                    // ValuesAsInt[huffmanTree.Indices[i] + code - huffmanTree.MinCodes[i]];
+                    result = huffmanTree.GetValue(code, i);
                     return DecoderErrorCode.NoError;
                 }