diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
index f80e75b99..ab08b0d4c 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bits.cs
@@ -58,12 +58,11 @@ namespace ImageSharp.Formats.Jpg
         {
             while (true)
             {
-                DecoderErrorCode errorCode;
-
                 // Grab the decode bytes, use them and then set them
                 // back on the decoder.
                 Bytes decoderBytes = decoder.Bytes;
-                byte c = decoderBytes.ReadByteStuffedByte(decoder.InputStream, out errorCode);
+                byte c;
+                DecoderErrorCode errorCode = decoderBytes.ReadByteStuffedByteUnsafe(decoder.InputStream, out c);
                 decoder.Bytes = decoderBytes;
 
                 if (errorCode != DecoderErrorCode.NoError)
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
index e33f852ff..176e0cfbe 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/Bytes.cs
@@ -67,14 +67,10 @@ namespace ImageSharp.Formats.Jpg
         /// ReadByteStuffedByte is like ReadByte but is for byte-stuffed Huffman data.
         /// </summary>
         /// <param name="inputStream">Input stream</param>
-        /// <param name="errorCode">Error code</param>
-        /// <returns>The <see cref="byte"/></returns>
-        internal byte ReadByteStuffedByte(Stream inputStream, out DecoderErrorCode errorCode)
+        /// <param name="x">The result <see cref="byte"/></param>
+        /// <returns>The <see cref="DecoderErrorCode"/></returns>
+        public DecoderErrorCode ReadByteStuffedByteUnsafe(Stream inputStream, out byte x)
         {
-            byte x;
-
-            errorCode = DecoderErrorCode.NoError;
-
             // Take the fast path if bytes.buf contains at least two bytes.
             if (this.I + 2 <= this.J)
             {
@@ -83,42 +79,46 @@ namespace ImageSharp.Formats.Jpg
                 this.UnreadableBytes = 1;
                 if (x != JpegConstants.Markers.XFF)
                 {
-                    return x;
+                    return DecoderErrorCode.NoError;
                 }
 
                 if (this.Buffer[this.I] != 0x00)
                 {
-                    errorCode = DecoderErrorCode.MissingFF00;
-                    return 0;
-
-                    // throw new MissingFF00Exception();
+                    return DecoderErrorCode.MissingFF00;
                 }
 
                 this.I++;
                 this.UnreadableBytes = 2;
-                return JpegConstants.Markers.XFF;
+                x = JpegConstants.Markers.XFF;
+                return DecoderErrorCode.NoError;
             }
 
             this.UnreadableBytes = 0;
 
-            x = this.ReadByte(inputStream);
+            DecoderErrorCode errorCode = this.ReadByteUnsafe(inputStream, out x);
             this.UnreadableBytes = 1;
+            if (errorCode != DecoderErrorCode.NoError)
+            {
+                return errorCode;
+            }
             if (x != JpegConstants.Markers.XFF)
             {
-                return x;
+                return DecoderErrorCode.NoError;
             }
 
-            x = this.ReadByte(inputStream);
+            errorCode = this.ReadByteUnsafe(inputStream, out x);
             this.UnreadableBytes = 2;
+            if (errorCode != DecoderErrorCode.NoError)
+            {
+                return errorCode;
+            }
             if (x != 0x00)
             {
-                errorCode = DecoderErrorCode.MissingFF00;
-                return 0;
-
-                // throw new MissingFF00Exception();
+                return DecoderErrorCode.MissingFF00;
             }
 
-            return JpegConstants.Markers.XFF;
+            x = JpegConstants.Markers.XFF;
+            return DecoderErrorCode.NoError;
         }
 
         /// <summary>
@@ -127,30 +127,68 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="inputStream">Input stream</param>
         /// <returns>The <see cref="byte"/></returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal byte ReadByte(Stream inputStream)
+        public byte ReadByte(Stream inputStream)
         {
+            byte result;
+            DecoderErrorCode errorCode = this.ReadByteUnsafe(inputStream, out result);
+            errorCode.EnsureNoError();
+            return result;
+        }
+
+        /// <summary>
+        /// Extracts the next byte, whether buffered or not buffered into the result out parameter. It does not care about byte stuffing.
+        /// This method does not throw on format error, it returns a <see cref="DecoderErrorCode"/> instead.
+        /// </summary>
+        /// <param name="inputStream">Input stream</param>
+        /// <param name="result">The result <see cref="byte"/> as out parameter</param>
+        /// <returns>The <see cref="DecoderErrorCode"/></returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public DecoderErrorCode ReadByteUnsafe(Stream inputStream, out byte result)
+        {
+            DecoderErrorCode errorCode = DecoderErrorCode.NoError;
             while (this.I == this.J)
             {
-                this.Fill(inputStream);
+                errorCode = this.FillUnsafe(inputStream);
+                if (errorCode != DecoderErrorCode.NoError)
+                {
+                    result = 0;
+                    return errorCode;
+                }
             }
 
-            byte x = this.Buffer[this.I];
+            result = this.Buffer[this.I];
             this.I++;
             this.UnreadableBytes = 0;
-            return x;
+            return errorCode;
+        }
+
+        /// <summary>
+        /// Fills up the bytes buffer from the underlying stream.
+        /// It should only be called when there are no unread bytes in bytes.
+        /// </summary>
+        /// <exception cref="EOFException">Thrown when reached end of stream unexpectedly.</exception>
+        /// <param name="inputStream">Input stream</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Fill(Stream inputStream)
+        {
+            DecoderErrorCode errorCode = this.FillUnsafe(inputStream);
+            errorCode.EnsureNoError();
         }
 
         /// <summary>
         /// Fills up the bytes buffer from the underlying stream.
         /// It should only be called when there are no unread bytes in bytes.
+        /// This method does not throw <see cref="EOFException"/>, returns a <see cref="DecoderErrorCode"/> instead!
         /// </summary>
         /// <param name="inputStream">Input stream</param>
+        /// <returns>The <see cref="DecoderErrorCode"/></returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        internal void Fill(Stream inputStream)
+        public DecoderErrorCode FillUnsafe(Stream inputStream)
         {
             if (this.I != this.J)
             {
-                throw new ImageFormatException("Fill called when unread bytes exist.");
+                // Unrecoverable error in the input, throwing!
+                DecoderThrowHelper.ThrowImageFormatException.FillCalledWhenUnreadBytesExist();
             }
 
             // Move the last 2 bytes to the start of the buffer, in case we need
@@ -167,10 +205,11 @@ namespace ImageSharp.Formats.Jpg
             int n = inputStream.Read(this.Buffer, this.J, this.Buffer.Length - this.J);
             if (n == 0)
             {
-                throw new EOFException();
+                return DecoderErrorCode.UnexpectedEndOfStream;
             }
 
             this.J += n;
+            return DecoderErrorCode.NoError;
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderErrorCode.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderErrorCode.cs
index 4adc9fa7f..8b82184fa 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderErrorCode.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderErrorCode.cs
@@ -24,6 +24,6 @@ namespace ImageSharp.Formats
         /// <summary>
         /// End of stream reached unexpectedly
         /// </summary>
-        UnexpectedEndOfFile
+        UnexpectedEndOfStream
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
index 33321bff9..18ba02a9a 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecoderThrowHelper.cs
@@ -26,7 +26,7 @@ namespace ImageSharp.Formats.Jpg
                     throw new ArgumentException("ThrowExceptionForErrorCode() called with NoError!", nameof(errorCode));
                 case DecoderErrorCode.MissingFF00:
                     throw new MissingFF00Exception();
-                case DecoderErrorCode.UnexpectedEndOfFile:
+                case DecoderErrorCode.UnexpectedEndOfStream:
                     throw new EOFException();
                 default:
                     throw new ArgumentOutOfRangeException(nameof(errorCode), errorCode, null);
@@ -45,5 +45,20 @@ namespace ImageSharp.Formats.Jpg
                 ThrowExceptionForErrorCode(errorCode);
             }
         }
+
+        /// <summary>
+        /// Encapsulates methods throwing different flavours of <see cref="ImageFormatException"/>-s.
+        /// </summary>
+        public static class ThrowImageFormatException
+        {
+            /// <summary>
+            /// Throws "Fill called when unread bytes exist."
+            /// </summary>
+            [MethodImpl(MethodImplOptions.NoInlining)]
+            public static void FillCalledWhenUnreadBytesExist()
+            {
+                throw new ImageFormatException("Fill called when unread bytes exist.");
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/EOFException.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/EOFException.cs
index b695e6812..5ed25ef04 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/EOFException.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/EOFException.cs
@@ -10,6 +10,7 @@ namespace ImageSharp.Formats.Jpg
     /// <summary>
     /// The EOF (End of File exception).
     /// Thrown when the decoder encounters an EOF marker without a proceeding EOI (End Of Image) marker
+    /// TODO: Rename to UnexpectedEndOfStreamException
     /// </summary>
     internal class EOFException : Exception
     {
diff --git a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
index b9f1dc77b..5571ba707 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
@@ -112,7 +112,7 @@ namespace ImageSharp.Formats
 
         /// <summary>
         /// Gets the saved state between progressive-mode scans.
-        /// TODO: Also store non-progressive data here. (Helps splitting and parallelizing JpegScanDecoder-s loop)
+        /// TODO: Also save non-progressive data here. (Helps splitting and parallelizing JpegScanDecoder-s loop)
         /// </summary>
         public Block8x8F[][] DecodedBlocks { get; }
 
@@ -490,6 +490,7 @@ namespace ImageSharp.Formats
                 this.Bytes.UnreadableBytes = 0;
             }
 
+            DecoderErrorCode errorCode = DecoderErrorCode.NoError;
             while (length > 0)
             {
                 if (this.Bytes.J - this.Bytes.I >= length)
@@ -505,12 +506,12 @@ namespace ImageSharp.Formats
                     length -= this.Bytes.J - this.Bytes.I;
                     this.Bytes.I += this.Bytes.J - this.Bytes.I;
 
-                    this.Bytes.Fill(this.InputStream);
+                    errorCode = this.Bytes.FillUnsafe(this.InputStream);
                 }
             }
 
-            return DecoderErrorCode.NoError;
-         }
+            return errorCode;
+        }
 
         /// <summary>
         /// Decodes the given number of bits
@@ -560,7 +561,7 @@ namespace ImageSharp.Formats
                         return (byte)(v >> 8);
                     }
                 }
-                else if (errorCode == DecoderErrorCode.UnexpectedEndOfFile)
+                else if (errorCode == DecoderErrorCode.UnexpectedEndOfStream)
                 {
                     errorCode.ThrowExceptionForErrorCode();
                 }
@@ -1419,6 +1420,7 @@ namespace ImageSharp.Formats
             this.MCUCountX = (this.ImageWidth + (8 * h0) - 1) / (8 * h0);
             this.MCUCountY = (this.ImageHeight + (8 * v0) - 1) / (8 * v0);
 
+            // As a preparation for parallelizing Scan decoder, we also allocate DecodedBlocks in the non-progressive case!
             for (int i = 0; i < this.ComponentCount; i++)
             {
                 int size = this.TotalMCUCount * this.ComponentArray[i].HorizontalFactor
@@ -1489,7 +1491,11 @@ namespace ImageSharp.Formats
                     break;
                 }
 
-                this.Bytes.Fill(this.InputStream);
+                DecoderErrorCode errorCode = this.Bytes.FillUnsafe(this.InputStream);
+                if (errorCode != DecoderErrorCode.NoError)
+                {
+                    return errorCode;
+                }
             }
 
             return DecoderErrorCode.NoError;
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegProfilingBenchmarks.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegProfilingBenchmarks.cs
index f2abce6e7..eb9747cce 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/JpegProfilingBenchmarks.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegProfilingBenchmarks.cs
@@ -21,7 +21,7 @@ namespace ImageSharp.Tests
         {
         }
 
-        // [Theory] // Benchmark, enable manually
+        [Theory] // Benchmark, enable manually
         [InlineData(30, TestImages.Jpeg.Baseline.Cmyk)]
         [InlineData(30, TestImages.Jpeg.Baseline.Ycck)]
         [InlineData(30, TestImages.Jpeg.Baseline.Calliphora)]