diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecodedBlockMemento.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecodedBlockMemento.cs
index 49d9b591f9..04ece04ee8 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecodedBlockMemento.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/DecodedBlockMemento.cs
@@ -5,6 +5,7 @@
 
 namespace ImageSharp.Formats.Jpg
 {
+    using System;
     using System.Buffers;
 
     /// <summary>
@@ -33,44 +34,68 @@ namespace ImageSharp.Formats.Jpg
         public Block8x8F Block;
 
         /// <summary>
-        /// The <see cref="ArrayPool{T}"/> used to pool data in <see cref="JpegDecoderCore.DecodedBlocks"/>.
-        /// Should always clean arrays when returning!
+        /// Store the block data into a <see cref="DecodedBlockMemento"/> at the given index of an <see cref="DecodedBlockMemento.Array"/>.
         /// </summary>
-        private static readonly ArrayPool<DecodedBlockMemento> ArrayPool = ArrayPool<DecodedBlockMemento>.Create();
-
-        /// <summary>
-        /// Rent an array of <see cref="DecodedBlockMemento"/>-s from the pool.
-        /// </summary>
-        /// <param name="size">The requested array size</param>
-        /// <returns>An array of <see cref="DecodedBlockMemento"/>-s</returns>
-        public static DecodedBlockMemento[] RentArray(int size)
+        /// <param name="blockArray">The array <see cref="DecodedBlockMemento.Array"/></param>
+        /// <param name="index">The index in the array</param>
+        /// <param name="bx">X coordinate of the block</param>
+        /// <param name="by">Y coordinate of the block</param>
+        /// <param name="block">The <see cref="Block8x8F"/></param>
+        public static void Store(ref DecodedBlockMemento.Array blockArray, int index, int bx, int by, ref Block8x8F block)
         {
-            return ArrayPool.Rent(size);
-        }
+            if (index >= blockArray.Count)
+            {
+                throw new IndexOutOfRangeException("Block index is out of range in DecodedBlockMemento.Store()!");
+            }
 
-        /// <summary>
-        /// Returns the <see cref="DecodedBlockMemento"/> array to the pool.
-        /// </summary>
-        /// <param name="blockArray">The <see cref="DecodedBlockMemento"/> array</param>
-        public static void ReturnArray(DecodedBlockMemento[] blockArray)
-        {
-            ArrayPool.Return(blockArray, true);
+            blockArray.Buffer[index].Initialized = true;
+            blockArray.Buffer[index].Bx = bx;
+            blockArray.Buffer[index].By = by;
+            blockArray.Buffer[index].Block = block;
         }
 
         /// <summary>
-        /// Store the block data into a <see cref="DecodedBlockMemento"/> at the given index.
+        /// Because <see cref="System.Array.Length"/> has no information for rented arrays, we need to store the count and the buffer separately.
         /// </summary>
-        /// <param name="blockArray">The array of <see cref="DecodedBlockMemento"/></param>
-        /// <param name="index">The index in the array</param>
-        /// <param name="bx">X coordinate of the block</param>
-        /// <param name="by">Y coordinate of the block</param>
-        /// <param name="block">The <see cref="Block8x8F"/></param>
-        public static void Store(DecodedBlockMemento[] blockArray, int index, int bx, int by, ref Block8x8F block)
+        public struct Array : IDisposable
         {
-            blockArray[index].Initialized = true;
-            blockArray[index].Bx = bx;
-            blockArray[index].By = by;
-            blockArray[index].Block = block;
+            /// <summary>
+            /// The <see cref="ArrayPool{T}"/> used to pool data in <see cref="JpegDecoderCore.DecodedBlocks"/>.
+            /// Should always clean arrays when returning!
+            /// </summary>
+            private static readonly ArrayPool<DecodedBlockMemento> ArrayPool = ArrayPool<DecodedBlockMemento>.Create();
+
+            /// <summary>
+            /// Initializes a new instance of the <see cref="Array"/> struct. Rents a buffer.
+            /// </summary>
+            /// <param name="count">The number of valid <see cref="DecodedBlockMemento"/>-s</param>
+            public Array(int count)
+            {
+                this.Count = count;
+                this.Buffer = ArrayPool.Rent(count);
+            }
+
+            /// <summary>
+            /// Gets the number of actual <see cref="DecodedBlockMemento"/>-s inside <see cref="Buffer"/>
+            /// </summary>
+            public int Count { get; }
+
+            /// <summary>
+            /// Gets the rented buffer.
+            /// </summary>
+            public DecodedBlockMemento[] Buffer { get; private set; }
+
+            /// <summary>
+            /// Returns the rented buffer to the pool.
+            /// </summary>
+            public void Dispose()
+            {
+                if (this.Buffer != null)
+                {
+                    ArrayPool.Return(this.Buffer, true);
+                    this.Buffer = null;
+                }
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
index 2d292ddf93..0e389771c0 100644
--- a/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
+++ b/src/ImageSharp.Formats.Jpeg/Components/Decoder/JpegScanDecoder.cs
@@ -196,7 +196,7 @@ namespace ImageSharp.Formats.Jpg
 
                             // Take an existing block (required when progressive):
                             int blockIndex = this.GetBlockIndex(decoder);
-                            this.data.Block = decoder.DecodedBlocks[this.ComponentIndex][blockIndex].Block;
+                            this.data.Block = decoder.DecodedBlocks[this.ComponentIndex].Buffer[blockIndex].Block;
 
                             if (!decoder.InputProcessor.UnexpectedEndOfStreamReached)
                             {
@@ -204,8 +204,8 @@ namespace ImageSharp.Formats.Jpg
                             }
 
                             // Store the decoded block
-                            DecodedBlockMemento[] blocks = decoder.DecodedBlocks[this.ComponentIndex];
-                            DecodedBlockMemento.Store(blocks, blockIndex, this.bx, this.by, ref this.data.Block);
+                            DecodedBlockMemento.Array blocks = decoder.DecodedBlocks[this.ComponentIndex];
+                            DecodedBlockMemento.Store(ref blocks, blockIndex, this.bx, this.by, ref this.data.Block);
                         }
 
                         // for j
diff --git a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
index 127b399ab4..91fca3eb68 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
@@ -87,7 +87,7 @@ namespace ImageSharp.Formats
             this.QuantizationTables = new Block8x8F[MaxTq + 1];
             this.Temp = new byte[2 * Block8x8F.ScalarCount];
             this.ComponentArray = new Component[MaxComponents];
-            this.DecodedBlocks = new DecodedBlockMemento[MaxComponents][];
+            this.DecodedBlocks = new DecodedBlockMemento.Array[MaxComponents];
         }
 
         /// <summary>
@@ -104,7 +104,7 @@ namespace ImageSharp.Formats
         /// Gets the saved state between progressive-mode scans.
         /// TODO: Also save non-progressive data here. (Helps splitting and parallelizing JpegScanDecoder-s loop)
         /// </summary>
-        public DecodedBlockMemento[][] DecodedBlocks { get; }
+        public DecodedBlockMemento.Array[] DecodedBlocks { get; }
 
         /// <summary>
         /// Gets the quantization tables, in zigzag order.
@@ -176,7 +176,7 @@ namespace ImageSharp.Formats
             this.ProcessStream(image, stream, metadataOnly);
             if (!metadataOnly)
             {
-                this.ProcessBlockColorsIntoJpegImageChannels<TColor>();
+                this.ProcessBlocksIntoJpegImageChannels<TColor>();
                 this.ConvertJpegPixelsToImagePixels(image);
             }
         }
@@ -191,12 +191,9 @@ namespace ImageSharp.Formats
                 this.HuffmanTrees[i].Dispose();
             }
 
-            foreach (DecodedBlockMemento[] blockArray in this.DecodedBlocks)
+            foreach (DecodedBlockMemento.Array blockArray in this.DecodedBlocks)
             {
-                if (blockArray != null)
-                {
-                    DecodedBlockMemento.ReturnArray(blockArray);
-                }
+                blockArray.Dispose();
             }
 
             this.ycbcrImage?.Dispose();
@@ -464,9 +461,11 @@ namespace ImageSharp.Formats
 
         /// <summary>
         /// Process the blocks in <see cref="DecodedBlocks"/> into Jpeg image channels (<see cref="YCbCrImage"/> and <see cref="JpegPixelArea"/>)
+        /// The blocks are expected in a "raw" frequency-domain decoded format. We need to apply IDCT and unzigging to transform them into color-space blocks.
+        /// We can copy these blocks into <see cref="JpegPixelArea"/>-s afterwards.
         /// </summary>
         /// <typeparam name="TColor">The pixel type</typeparam>
-        private void ProcessBlockColorsIntoJpegImageChannels<TColor>()
+        private void ProcessBlocksIntoJpegImageChannels<TColor>()
             where TColor : struct, IPackedPixel, IEquatable<TColor>
         {
             Parallel.For(
@@ -478,10 +477,10 @@ namespace ImageSharp.Formats
                         JpegScanDecoder.Init(&scanDecoder);
 
                         scanDecoder.ComponentIndex = componentIndex;
-                        DecodedBlockMemento[] blockArray = this.DecodedBlocks[componentIndex];
-                        for (int i = 0; i < blockArray.Length; i++)
+                        DecodedBlockMemento.Array blockArray = this.DecodedBlocks[componentIndex];
+                        for (int i = 0; i < blockArray.Count; i++)
                         {
-                            scanDecoder.LoadMemento(ref blockArray[i]);
+                            scanDecoder.LoadMemento(ref blockArray.Buffer[i]);
                             scanDecoder.ProcessBlockColors(this);
                         }
                     });
@@ -1315,9 +1314,9 @@ namespace ImageSharp.Formats
             // As a preparation for parallelizing Scan decoder, we also allocate DecodedBlocks in the non-progressive case!
             for (int i = 0; i < this.ComponentCount; i++)
             {
-                int size = this.TotalMCUCount * this.ComponentArray[i].HorizontalFactor
+                int count = this.TotalMCUCount * this.ComponentArray[i].HorizontalFactor
                            * this.ComponentArray[i].VerticalFactor;
-                this.DecodedBlocks[i] = DecodedBlockMemento.RentArray(size);
+                this.DecodedBlocks[i] = new DecodedBlockMemento.Array(count);
             }
         }
     }
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
index 5776bf2e45..12ebee3e5c 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.cs
@@ -7,6 +7,9 @@
 namespace ImageSharp.Tests
 {
     using System;
+    using System.IO;
+
+    using ImageSharp.Formats;
 
     using Xunit;
 
@@ -30,7 +33,7 @@ namespace ImageSharp.Tests
 
             provider.Utility.SaveTestOutputFile(image, "bmp");
         }
-
+        
         [Theory]
         [WithFileCollection(nameof(ProgressiveTestJpegs), PixelTypes.Color | PixelTypes.StandardImageClass | PixelTypes.Argb)]
         public void OpenProgressiveJpeg_SaveBmp<TColor>(TestImageProvider<TColor> provider)
@@ -40,5 +43,34 @@ namespace ImageSharp.Tests
 
             provider.Utility.SaveTestOutputFile(image, "bmp");
         }
+        
+        [Theory]
+        [WithSolidFilledImages(16, 16, 255, 0, 0, PixelTypes.StandardImageClass, JpegSubsample.Ratio420, 75)]
+        [WithSolidFilledImages(16, 16, 255, 0, 0, PixelTypes.StandardImageClass, JpegSubsample.Ratio420, 100)]
+        [WithSolidFilledImages(16, 16, 255, 0, 0, PixelTypes.StandardImageClass, JpegSubsample.Ratio444, 75)]
+        [WithSolidFilledImages(16, 16, 255, 0, 0, PixelTypes.StandardImageClass, JpegSubsample.Ratio444, 100)]
+        [WithSolidFilledImages(8, 8, 255, 0, 0, PixelTypes.StandardImageClass, JpegSubsample.Ratio444, 100)]
+        public void DecodeGenerated_SaveBmp<TColor>(TestImageProvider<TColor> provider, JpegSubsample subsample, int qulaity)
+            where TColor : struct, IPackedPixel, IEquatable<TColor>
+        {
+            Image<TColor> image = provider.GetImage();
+
+            JpegEncoder encoder = new JpegEncoder()
+                                      {
+                                          Subsample = subsample,
+                                          Quality = qulaity
+                                      };
+
+            byte[] data = new byte[65536];
+            using (MemoryStream ms = new MemoryStream(data))
+            {
+                image.Save(ms, encoder);
+            }
+
+            // TODO: Automatic image comparers could help here a lot :P
+            Image<TColor> mirror = provider.Factory.CreateImage(data);
+            provider.Utility.TestName += $"_{subsample}_Q{qulaity}";
+            provider.Utility.SaveTestOutputFile(mirror, "bmp");
+        }
     }
 }
\ No newline at end of file