diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
index a1fe6444b..ab376ee3c 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
@@ -37,6 +37,11 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         private const int DcTableIndex = 0;
 
+        /// <summary>
+        /// The current component index
+        /// </summary>
+        public int ComponentIndex;
+
         /// <summary>
         /// X coordinate of the current block, in units of 8x8. (The third block in the first row has (bx, by) = (2, 0))
         /// </summary>
@@ -72,11 +77,6 @@ namespace ImageSharp.Formats.Jpg
         /// </summary>
         private int componentScanCount;
 
-        /// <summary>
-        /// The current component index
-        /// </summary>
-        private int componentIndex;
-
         /// <summary>
         /// Horizontal sampling factor at the current component index
         /// </summary>
@@ -103,11 +103,23 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="p">Pointer to <see cref="JpegScanDecoder"/> on the stack</param>
         /// <param name="decoder">The <see cref="JpegDecoderCore"/> instance</param>
         /// <param name="remaining">The remaining bytes in the segment block.</param>
-        public static void Init(JpegScanDecoder* p, JpegDecoderCore decoder, int remaining)
+        public static void InitStreamReading(JpegScanDecoder* p, JpegDecoderCore decoder, int remaining)
+        {
+            Init(p);
+            p->InitStreamReadingImpl(decoder, remaining);
+        }
+
+        public static void Init(JpegScanDecoder* p)
         {
             p->data = ComputationData.Create();
             p->pointers = new DataPointers(&p->data);
-            p->InitImpl(decoder, remaining);
+        }
+
+        public void LoadMemento(ref DecodedBlockMemento memento)
+        {
+            this.bx = memento.Bx;
+            this.by = memento.By;
+            this.data.Block = memento.Block;
         }
 
         /// <summary>
@@ -148,9 +160,9 @@ namespace ImageSharp.Formats.Jpg
                 {
                     for (int scanIndex = 0; scanIndex < this.componentScanCount; scanIndex++)
                     {
-                        this.componentIndex = this.pointers.ComponentScan[scanIndex].ComponentIndex;
-                        this.hi = decoder.ComponentArray[this.componentIndex].HorizontalFactor;
-                        int vi = decoder.ComponentArray[this.componentIndex].VerticalFactor;
+                        this.ComponentIndex = this.pointers.ComponentScan[scanIndex].ComponentIndex;
+                        this.hi = decoder.ComponentArray[this.ComponentIndex].HorizontalFactor;
+                        int vi = decoder.ComponentArray[this.ComponentIndex].VerticalFactor;
 
                         for (int j = 0; j < this.hi * vi; j++)
                         {
@@ -172,7 +184,7 @@ namespace ImageSharp.Formats.Jpg
                             }
 
                             this.ReadBlock(decoder, scanIndex);
-                            this.ProcessBlock(decoder);
+                            //this.ProcessBlock(decoder);
                         }
 
                         // for j
@@ -219,7 +231,7 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="decoder">The <see cref="JpegDecoderCore"/> instance</param>
         public void ProcessBlock(JpegDecoderCore decoder)
         {
-            int qtIndex = decoder.ComponentArray[this.componentIndex].Selector;
+            int qtIndex = decoder.ComponentArray[this.ComponentIndex].Selector;
             this.data.QuantiazationTable = decoder.QuantizationTables[qtIndex];
 
             Block8x8F* b = this.pointers.Block;
@@ -228,7 +240,7 @@ namespace ImageSharp.Formats.Jpg
 
             DCT.TransformIDCT(ref *b, ref *this.pointers.Temp1, ref *this.pointers.Temp2);
 
-            var destChannel = decoder.GetDestinationChannel(this.componentIndex);
+            var destChannel = decoder.GetDestinationChannel(this.ComponentIndex);
             var destArea = destChannel.GetOffsetedSubAreaForBlock(this.bx, this.by);
             destArea.LoadColorsFrom(this.pointers.Temp1, this.pointers.Temp2);
         }
@@ -239,11 +251,11 @@ namespace ImageSharp.Formats.Jpg
         }
 
         /// <summary>
-        /// The implementation part of <see cref="Init"/> as an instance method.
+        /// The implementation part of <see cref="InitStreamReading"/> as an instance method.
         /// </summary>
         /// <param name="decoder">The <see cref="JpegDecoderCore"/></param>
         /// <param name="remaining">The remaining bytes</param>
-        private void InitImpl(JpegDecoderCore decoder, int remaining)
+        private void InitStreamReadingImpl(JpegDecoderCore decoder, int remaining)
         {
             if (decoder.ComponentCount == 0)
             {
@@ -313,7 +325,7 @@ namespace ImageSharp.Formats.Jpg
         private void ReadBlock(JpegDecoderCore decoder, int scanIndex)
         {
             int blockIndex = this.GetBlockIndex(decoder);
-            this.data.Block = decoder.DecodedBlocks[this.componentIndex][blockIndex].Block;
+            this.data.Block = decoder.DecodedBlocks[this.ComponentIndex][blockIndex].Block;
 
             var b = this.pointers.Block;
             DecoderErrorCode errorCode;
@@ -344,10 +356,10 @@ namespace ImageSharp.Formats.Jpg
 
                     int deltaDC = decoder.Bits.ReceiveExtend(value, decoder);
 
-                    this.pointers.Dc[this.componentIndex] += deltaDC;
+                    this.pointers.Dc[this.ComponentIndex] += deltaDC;
 
                     // b[0] = dc[compIndex] << al;
-                    Block8x8F.SetScalarAt(b, 0, this.pointers.Dc[this.componentIndex] << this.al);
+                    Block8x8F.SetScalarAt(b, 0, this.pointers.Dc[this.ComponentIndex] << this.al);
                 }
 
                 if (zig <= this.zigEnd && this.eobRun > 0)
@@ -399,7 +411,7 @@ namespace ImageSharp.Formats.Jpg
                 }
             }
 
-            DecodedBlockMemento[] blocks = decoder.DecodedBlocks[this.componentIndex];
+            DecodedBlockMemento[] blocks = decoder.DecodedBlocks[this.ComponentIndex];
             DecodedBlockMemento.Store(blocks, blockIndex, this.bx, this.by, ref *b);
         }
 
diff --git a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
index 30367e93e..f8adedd26 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
@@ -175,8 +175,89 @@ namespace ImageSharp.Formats
         /// <typeparam name="TColor">The pixel format.</typeparam>
         /// <param name="image">The image, where the data should be set to.</param>
         /// <param name="stream">The stream, where the image should be.</param>
-        /// <param name="configOnly">Whether to decode metadata only.</param>
-        public void Decode<TColor>(Image<TColor> image, Stream stream, bool configOnly)
+        /// <param name="metadataOnly">Whether to decode metadata only.</param>
+        public void Decode<TColor>(Image<TColor> image, Stream stream, bool metadataOnly)
+            where TColor : struct, IPackedPixel, IEquatable<TColor>
+        {
+            this.ProcessStream(image, stream, metadataOnly);
+            if (metadataOnly) return;
+            this.ConvertBlocksToImagePixels(image);
+        }
+
+        private void ConvertBlocksToImagePixels<TColor>(Image<TColor> image)
+            where TColor : struct, IPackedPixel, IEquatable<TColor>
+        {
+            this.ProcessBlocks<TColor>();
+
+            if (this.grayImage.IsInitialized)
+            {
+                this.ConvertFromGrayScale(this.ImageWidth, this.ImageHeight, image);
+            }
+            else if (this.ycbcrImage != null)
+            {
+                if (this.ComponentCount == 4)
+                {
+                    if (!this.adobeTransformValid)
+                    {
+                        throw new ImageFormatException(
+                            "Unknown color model: 4-component JPEG doesn't have Adobe APP14 metadata");
+                    }
+
+                    // See http://www.sno.phy.queensu.ca/~phil/exiftool/TagNames/JPEG.html#Adobe
+                    // See https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/jpeg_metadata.html
+                    // TODO: YCbCrA?
+                    if (this.adobeTransform == JpegConstants.Adobe.ColorTransformYcck)
+                    {
+                        this.ConvertFromYcck(this.ImageWidth, this.ImageHeight, image);
+                    }
+                    else if (this.adobeTransform == JpegConstants.Adobe.ColorTransformUnknown)
+                    {
+                        // Assume CMYK
+                        this.ConvertFromCmyk(this.ImageWidth, this.ImageHeight, image);
+                    }
+
+                    return;
+                }
+
+                if (this.ComponentCount == 3)
+                {
+                    if (this.IsRGB())
+                    {
+                        this.ConvertFromRGB(this.ImageWidth, this.ImageHeight, image);
+                        return;
+                    }
+
+                    this.ConvertFromYCbCr(this.ImageWidth, this.ImageHeight, image);
+                    return;
+                }
+
+                throw new ImageFormatException("JpegDecoder only supports RGB, CMYK and Grayscale color spaces.");
+            }
+            else
+            {
+                throw new ImageFormatException("Missing SOS marker.");
+            }
+        }
+
+        private void ProcessBlocks<TColor>()
+            where TColor : struct, IPackedPixel, IEquatable<TColor>
+        {
+            JpegScanDecoder scanDecoder = default(JpegScanDecoder);
+            JpegScanDecoder.Init(&scanDecoder);
+
+            for(int componentIndex = 0; componentIndex < this.ComponentCount; componentIndex++)
+            {
+                scanDecoder.ComponentIndex = componentIndex;
+                DecodedBlockMemento[] blockArray = this.DecodedBlocks[componentIndex];
+                for (int i = 0; i < blockArray.Length; i++)
+                {
+                    scanDecoder.LoadMemento(ref blockArray[i]);
+                    scanDecoder.ProcessBlock(this);
+                }
+            }
+        }
+
+        private void ProcessStream<TColor>(Image<TColor> image, Stream stream, bool metadataOnly)
             where TColor : struct, IPackedPixel, IEquatable<TColor>
         {
             this.InputStream = stream;
@@ -265,14 +346,14 @@ namespace ImageSharp.Formats
                     case JpegConstants.Markers.SOF2:
                         this.IsProgressive = marker == JpegConstants.Markers.SOF2;
                         this.ProcessStartOfFrameMarker(remaining);
-                        if (configOnly && this.isJfif)
+                        if (metadataOnly && this.isJfif)
                         {
                             return;
                         }
 
                         break;
                     case JpegConstants.Markers.DHT:
-                        if (configOnly)
+                        if (metadataOnly)
                         {
                             this.Skip(remaining);
                         }
@@ -283,7 +364,7 @@ namespace ImageSharp.Formats
 
                         break;
                     case JpegConstants.Markers.DQT:
-                        if (configOnly)
+                        if (metadataOnly)
                         {
                             this.Skip(remaining);
                         }
@@ -294,7 +375,7 @@ namespace ImageSharp.Formats
 
                         break;
                     case JpegConstants.Markers.SOS:
-                        if (configOnly)
+                        if (metadataOnly)
                         {
                             return;
                         }
@@ -310,7 +391,7 @@ namespace ImageSharp.Formats
 
                         break;
                     case JpegConstants.Markers.DRI:
-                        if (configOnly)
+                        if (metadataOnly)
                         {
                             this.Skip(remaining);
                         }
@@ -348,55 +429,6 @@ namespace ImageSharp.Formats
                         break;
                 }
             }
-
-            if (this.grayImage.IsInitialized)
-            {
-                this.ConvertFromGrayScale(this.ImageWidth, this.ImageHeight, image);
-            }
-            else if (this.ycbcrImage != null)
-            {
-                if (this.ComponentCount == 4)
-                {
-                    if (!this.adobeTransformValid)
-                    {
-                        throw new ImageFormatException(
-                                  "Unknown color model: 4-component JPEG doesn't have Adobe APP14 metadata");
-                    }
-
-                    // See http://www.sno.phy.queensu.ca/~phil/exiftool/TagNames/JPEG.html#Adobe
-                    // See https://docs.oracle.com/javase/8/docs/api/javax/imageio/metadata/doc-files/jpeg_metadata.html
-                    // TODO: YCbCrA?
-                    if (this.adobeTransform == JpegConstants.Adobe.ColorTransformYcck)
-                    {
-                        this.ConvertFromYcck(this.ImageWidth, this.ImageHeight, image);
-                    }
-                    else if (this.adobeTransform == JpegConstants.Adobe.ColorTransformUnknown)
-                    {
-                        // Assume CMYK
-                        this.ConvertFromCmyk(this.ImageWidth, this.ImageHeight, image);
-                    }
-
-                    return;
-                }
-
-                if (this.ComponentCount == 3)
-                {
-                    if (this.IsRGB())
-                    {
-                        this.ConvertFromRGB(this.ImageWidth, this.ImageHeight, image);
-                        return;
-                    }
-
-                    this.ConvertFromYCbCr(this.ImageWidth, this.ImageHeight, image);
-                    return;
-                }
-
-                throw new ImageFormatException("JpegDecoder only supports RGB, CMYK and Grayscale color spaces.");
-            }
-            else
-            {
-                throw new ImageFormatException("Missing SOS marker.");
-            }
         }
 
         /// <summary>
@@ -1453,7 +1485,7 @@ namespace ImageSharp.Formats
         private void ProcessStartOfScan(int remaining)
         {
             JpegScanDecoder scan = default(JpegScanDecoder);
-            JpegScanDecoder.Init(&scan, this, remaining);
+            JpegScanDecoder.InitStreamReading(&scan, this, remaining);
             this.Bits = default(Bits);
             this.MakeImage();
             scan.ReadBlocks(this);