diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs
index 5b66ad598..a72835e75 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs
@@ -10,10 +10,8 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
     /// <summary>
     /// Contains all the decoded component blocks
     /// </summary>
-    internal class ComponentBlocks : IDisposable
+    internal sealed class ComponentBlocks : IDisposable
     {
-        private bool isDisposed;
-
         /// <summary>
         /// Gets or sets the component blocks
         /// </summary>
@@ -22,31 +20,14 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// <inheritdoc/>
         public void Dispose()
         {
-            this.Dispose(true);
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        /// <param name="disposing">Whether to dispose of managed objects</param>
-        protected virtual void Dispose(bool disposing)
-        {
-            if (!this.isDisposed)
+            if (this.Components != null)
             {
-                if (disposing)
+                for (int i = 0; i < this.Components.Length; i++)
                 {
-                    if (this.Components != null)
-                    {
-                        for (int i = 0; i < this.Components.Length; i++)
-                        {
-                            this.Components[i].Dispose();
-                        }
-                    }
+                    this.Components[i].Dispose();
                 }
 
-                // Set large fields to null.
                 this.Components = null;
-                this.isDisposed = true;
             }
         }
     }
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
index 20fd2e9e8..06b4bbc24 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
@@ -10,10 +10,8 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
     /// <summary>
     /// Represent a single jpeg frame
     /// </summary>
-    internal class Frame : IDisposable
+    internal sealed class Frame : IDisposable
     {
-        private bool isDisposed;
-
         /// <summary>
         /// Gets or sets a value indicating whether the frame uses the extended specification
         /// </summary>
@@ -77,34 +75,15 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// <inheritdoc/>
         public void Dispose()
         {
-            this.Dispose(true);
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        /// <param name="disposing">Whether to dispose of managed objects</param>
-        protected virtual void Dispose(bool disposing)
-        {
-            if (this.isDisposed)
+            if (this.Components != null)
             {
-                return;
-            }
-
-            if (disposing)
-            {
-                if (this.Components != null)
+                for (int i = 0; i < this.Components.Length; i++)
                 {
-                    for (int i = 0; i < this.Components.Length; i++)
-                    {
-                        this.Components[i].Dispose();
-                    }
+                    this.Components[i].Dispose();
                 }
-            }
 
-            // Set large fields to null.
-            this.Components = null;
-            this.isDisposed = true;
+                this.Components = null;
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/QuantizationTables.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/QuantizationTables.cs
index 1ca31b31b..fa57a18dd 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/QuantizationTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/QuantizationTables.cs
@@ -5,15 +5,15 @@
 
 namespace ImageSharp.Formats.Jpeg.Port.Components
 {
+    using System;
     using System.Runtime.CompilerServices;
 
     using ImageSharp.Memory;
 
     /// <summary>
     /// Contains the quantization tables.
-    /// TODO: This all needs optimizing for memory. I'm just stubbing out functionality for now.
     /// </summary>
-    internal class QuantizationTables
+    internal sealed class QuantizationTables : IDisposable
     {
         /// <summary>
         /// Gets the ZigZag scan table
@@ -46,6 +46,16 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// <summary>
         /// Gets or sets the quantization tables.
         /// </summary>
-        public Fast2DArray<short> Tables { get; set; } = new Fast2DArray<short>(64, 4);
+        public Buffer2D<short> Tables { get; set; }
+
+        /// <inheritdoc/>
+        public void Dispose()
+        {
+            if (this.Tables != null)
+            {
+                this.Tables.Dispose();
+                this.Tables = null;
+            }
+        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
index deeec34fc..8d3cec616 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
@@ -19,7 +19,7 @@ namespace ImageSharp.Formats.Jpeg.Port
     /// Performs the jpeg decoding operation.
     /// Ported from <see href="https://github.com/mozilla/pdf.js/blob/master/src/core/jpg.js"/>
     /// </summary>
-    internal class JpegDecoderCore : IDisposable
+    internal sealed class JpegDecoderCore : IDisposable
     {
         /// <summary>
         /// The decoder options.
@@ -66,11 +66,6 @@ namespace ImageSharp.Formats.Jpeg.Port
         /// </summary>
         private Adobe adobe;
 
-        /// <summary>
-        /// A value indicating whether the decoder has been disposed
-        /// </summary>
-        private bool isDisposed;
-
         /// <summary>
         /// Initializes a new instance of the <see cref="JpegDecoderCore" /> class.
         /// </summary>
@@ -210,28 +205,14 @@ namespace ImageSharp.Formats.Jpeg.Port
         /// <inheritdoc/>
         public void Dispose()
         {
-            this.Dispose(true);
-        }
-
-        /// <summary>
-        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
-        /// </summary>
-        /// <param name="disposing">Whether to dispose of managed objects</param>
-        protected virtual void Dispose(bool disposing)
-        {
-            if (!this.isDisposed)
-            {
-                if (disposing)
-                {
-                    this.frame.Dispose();
-                    this.components.Dispose();
-                }
-
-                // Set large fields to null.
-                this.frame = null;
-                this.components = null;
-                this.isDisposed = true;
-            }
+            this.frame.Dispose();
+            this.components.Dispose();
+            this.quantizationTables.Dispose();
+
+            // Set large fields to null.
+            this.frame = null;
+            this.components = null;
+            this.quantizationTables = null;
         }
 
         private void ParseStream()
@@ -253,11 +234,12 @@ namespace ImageSharp.Formats.Jpeg.Port
             while (fileMarker.Marker != JpegConstants.Markers.EOI)
             {
                 // Get the marker length
-                int remaining = this.ReadUint16() - 2;
+                int remaining;
 
                 switch (fileMarker.Marker)
                 {
                     case JpegConstants.Markers.APP0:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessApplicationHeaderMarker(remaining);
                         break;
 
@@ -277,6 +259,7 @@ namespace ImageSharp.Formats.Jpeg.Port
                         break;
 
                     case JpegConstants.Markers.APP14:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessApp14Marker(remaining);
                         break;
 
@@ -287,24 +270,29 @@ namespace ImageSharp.Formats.Jpeg.Port
                         break;
 
                     case JpegConstants.Markers.DQT:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessDqtMarker(remaining);
                         break;
 
                     case JpegConstants.Markers.SOF0:
                     case JpegConstants.Markers.SOF1:
                     case JpegConstants.Markers.SOF2:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessStartOfFrameMarker(remaining, fileMarker);
                         break;
 
                     case JpegConstants.Markers.DHT:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessDefineHuffmanTablesMarker(remaining);
                         break;
 
                     case JpegConstants.Markers.DRI:
+                        remaining = this.ReadUint16() - 2;
                         this.ProcessDefineRestartIntervalMarker(remaining);
                         break;
 
                     case JpegConstants.Markers.SOS:
+                        this.InputStream.Skip(2);
                         this.ProcessStartOfScanMarker();
                         break;
 
@@ -312,11 +300,6 @@ namespace ImageSharp.Formats.Jpeg.Port
                         if ((byte)this.InputStream.ReadByte() != 0xFF)
                         {
                             // Avoid skipping a valid marker
-                            this.InputStream.Position -= 2;
-                        }
-                        else
-                        {
-                            // Rewind that last byte we read
                             this.InputStream.Position -= 1;
                         }
 
@@ -453,6 +436,8 @@ namespace ImageSharp.Formats.Jpeg.Port
         /// </exception>
         private void ProcessDqtMarker(int remaining)
         {
+            // Pooled. Disposed on disposal of decoder
+            this.quantizationTables.Tables = new Buffer2D<short>(64, 4);
             while (remaining > 0)
             {
                 bool done = false;