diff --git a/src/ImageSharp/Formats/Jpg/Components/Bits.cs b/src/ImageSharp/Formats/Jpg/Components/Bits.cs
index 0eebef132..d7449f4a3 100644
--- a/src/ImageSharp/Formats/Jpg/Components/Bits.cs
+++ b/src/ImageSharp/Formats/Jpg/Components/Bits.cs
@@ -45,7 +45,11 @@ namespace ImageSharp.Formats
             {
                 JpegDecoderCore.ErrorCodes errorCode;
 
-                byte c = decoder.Bytes.ReadByteStuffedByte(decoder.InputStream, out errorCode);
+                // Grab the decode bytes, use them and then set them
+                // back on the decoder.
+                var decoderBytes = decoder.Bytes;
+                byte c = decoderBytes.ReadByteStuffedByte(decoder.InputStream, out errorCode);
+                decoder.Bytes = decoderBytes;
 
                 if (errorCode != JpegDecoderCore.ErrorCodes.NoError)
                 {
diff --git a/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
index 90c005096..3991c4e0e 100644
--- a/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
@@ -236,7 +236,7 @@ namespace ImageSharp.Formats
         }
 
         /// <summary>
-        /// Gets the byte buffer.
+        /// Gets or sets the byte buffer.
         /// </summary>
         public Bytes Bytes
         {
@@ -244,6 +244,11 @@ namespace ImageSharp.Formats
             {
                 return this.bytes;
             }
+
+            set
+            {
+                this.bytes = value;
+            }
         }
 
         /// <summary>