diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/Adobe.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/Adobe.cs
new file mode 100644
index 000000000..130b7bdb3
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/Adobe.cs
@@ -0,0 +1,38 @@
+// <copyright file="Adobe.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+// ReSharper disable InconsistentNaming
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    /// <summary>
+    /// Provides information about the Adobe marker segment
+    /// </summary>
+    internal struct Adobe
+    {
+        /// <summary>
+        /// The DCT Encode Version
+        /// </summary>
+        public short DCTEncodeVersion;
+
+        /// <summary>
+        /// 0x0 : (none)
+        /// Bit 15 : Encoded with Blend=1 downsampling
+        /// </summary>
+        public short APP14Flags0;
+
+        /// <summary>
+        /// 0x0 : (none)
+        /// </summary>
+        public short APP14Flags1;
+
+        /// <summary>
+        /// Determines the colorspace transform
+        /// 00 : Unknown (RGB or CMYK)
+        /// 01 : YCbCr
+        /// 02 : YCCK
+        /// </summary>
+        public byte ColorTransform;
+    }
+}
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs
new file mode 100644
index 000000000..e5ae70f1f
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/Component.cs
@@ -0,0 +1,48 @@
+// <copyright file="Component.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    using System;
+
+    using ImageSharp.Memory;
+
+    /// <summary>
+    /// Represents a component block
+    /// </summary>
+    internal struct Component : IDisposable
+    {
+        /// <summary>
+        /// Gets or sets the output
+        /// </summary>
+        public Buffer<byte> Output;
+
+        /// <summary>
+        /// Gets or sets the horizontal scaling factor
+        /// </summary>
+        public int ScaleX;
+
+        /// <summary>
+        /// Gets or sets the vertical scaling factor
+        /// </summary>
+        public int ScaleY;
+
+        /// <summary>
+        /// Gets or sets the number of blocks per line
+        /// </summary>
+        public int BlocksPerLine;
+
+        /// <summary>
+        /// Gets or sets the number of blocks per column
+        /// </summary>
+        public int BlocksPerColumn;
+
+        /// <inheritdoc/>
+        public void Dispose()
+        {
+            this.Output?.Dispose();
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs
new file mode 100644
index 000000000..5b66ad598
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/ComponentBlocks.cs
@@ -0,0 +1,53 @@
+// <copyright file="Components.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    using System;
+
+    /// <summary>
+    /// Contains all the decoded component blocks
+    /// </summary>
+    internal class ComponentBlocks : IDisposable
+    {
+        private bool isDisposed;
+
+        /// <summary>
+        /// Gets or sets the component blocks
+        /// </summary>
+        public Component[] Components { get; set; }
+
+        /// <inheritdoc/>
+        public void Dispose()
+        {
+            this.Dispose(true);
+        }
+
+        /// <summary>
+        /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources.
+        /// </summary>
+        /// <param name="disposing">Whether to dispose of managed objects</param>
+        protected virtual void Dispose(bool disposing)
+        {
+            if (!this.isDisposed)
+            {
+                if (disposing)
+                {
+                    if (this.Components != null)
+                    {
+                        for (int i = 0; i < this.Components.Length; i++)
+                        {
+                            this.Components[i].Dispose();
+                        }
+                    }
+                }
+
+                // Set large fields to null.
+                this.Components = null;
+                this.isDisposed = true;
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
index 97c422ca3..20fd2e9e8 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/Frame.cs
@@ -93,15 +93,17 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
 
             if (disposing)
             {
-                foreach (FrameComponent component in this.Components)
+                if (this.Components != null)
                 {
-                    component.Dispose();
+                    for (int i = 0; i < this.Components.Length; i++)
+                    {
+                        this.Components[i].Dispose();
+                    }
                 }
             }
 
             // Set large fields to null.
             this.Components = null;
-
             this.isDisposed = true;
         }
     }
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
index 18176bde7..0cb9bbb1c 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/FrameComponent.cs
@@ -39,11 +39,6 @@ namespace ImageSharp.Formats.Jpeg.Port.Components
         /// </summary>
         public byte QuantizationIdentifier;
 
-        /// <summary>
-        /// Gets or sets the quantization table
-        /// </summary>
-        public short[] QuantizationTable;
-
         /// <summary>
         /// Gets or sets the block data
         /// </summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Port/Components/JFif.cs b/src/ImageSharp/Formats/Jpeg/Port/Components/JFif.cs
new file mode 100644
index 000000000..7fa6c44d0
--- /dev/null
+++ b/src/ImageSharp/Formats/Jpeg/Port/Components/JFif.cs
@@ -0,0 +1,43 @@
+// <copyright file="JFif.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+namespace ImageSharp.Formats.Jpeg.Port.Components
+{
+    /// <summary>
+    /// Provides information about the JFIF marker segment
+    /// </summary>
+    internal struct JFif
+    {
+        /// <summary>
+        /// The major version
+        /// </summary>
+        public byte MajorVersion;
+
+        /// <summary>
+        /// The minor version
+        /// </summary>
+        public byte MinorVersion;
+
+        /// <summary>
+        /// Units for the following pixel density fields
+        ///  00 : No units; width:height pixel aspect ratio = Ydensity:Xdensity
+        ///  01 : Pixels per inch (2.54 cm)
+        ///  02 : Pixels per centimeter
+        /// </summary>
+        public byte DensityUnits;
+
+        /// <summary>
+        /// Horizontal pixel density. Must not be zero.
+        /// </summary>
+        public short XDensity;
+
+        /// <summary>
+        /// Vertical pixel density. Must not be zero.
+        /// </summary>
+        public short YDensity;
+
+        // TODO: Thumbnail?
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegConstants.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegConstants.cs
index 0ad8afa91..08ae5543d 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegConstants.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegConstants.cs
@@ -1,4 +1,9 @@
-// ReSharper disable InconsistentNaming
+// <copyright file="JpegConstants.cs" company="James Jackson-South">
+// Copyright (c) James Jackson-South and contributors.
+// Licensed under the Apache License, Version 2.0.
+// </copyright>
+
+// ReSharper disable InconsistentNaming
 namespace ImageSharp.Formats.Jpeg.Port
 {
     /// <summary>
@@ -191,6 +196,11 @@ namespace ImageSharp.Formats.Jpeg.Port
             /// </summary>
             public const ushort RST7 = 0xFFD7;
 
+            /// <summary>
+            /// Marker prefix. Next byte is a marker.
+            /// </summary>
+            public const ushort XFF = 0xFFFF;
+
             /// <summary>
             /// Contains JFIF specific markers
             /// </summary>
@@ -216,6 +226,52 @@ namespace ImageSharp.Formats.Jpeg.Port
                 /// </summary>
                 public const byte Null = 0;
             }
+
+            /// <summary>
+            /// Contains Adobe specific markers
+            /// </summary>
+            public static class Adobe
+            {
+                /// <summary>
+                /// Represents A in ASCII
+                /// </summary>
+                public const byte A = 0x41;
+
+                /// <summary>
+                /// Represents d in ASCII
+                /// </summary>
+                public const byte D = 0x64;
+
+                /// <summary>
+                /// Represents b in ASCII
+                /// </summary>
+                public const byte O = 0x6F;
+
+                /// <summary>
+                /// Represents b in ASCII
+                /// </summary>
+                public const byte B = 0x62;
+
+                /// <summary>
+                /// Represents e in ASCII
+                /// </summary>
+                public const byte E = 0x65;
+
+                /// <summary>
+                /// The color transform is unknown.(RGB or CMYK)
+                /// </summary>
+                public const byte ColorTransformUnknown = 0;
+
+                /// <summary>
+                /// The color transform is YCbCr (luminance, red chroma, blue chroma)
+                /// </summary>
+                public const byte ColorTransformYCbCr = 1;
+
+                /// <summary>
+                /// The color transform is YCCK (luminance, red chroma, blue chroma, keyline)
+                /// </summary>
+                public const byte ColorTransformYcck = 2;
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
index 873d4623d..deeec34fc 100644
--- a/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/Port/JpegDecoderCore.cs
@@ -46,13 +46,29 @@ namespace ImageSharp.Formats.Jpeg.Port
 
         private Frame frame;
 
+        private ComponentBlocks components;
+
         private ushort resetInterval;
 
+        private int width;
+
+        private int height;
+
+        private int numComponents;
+
         /// <summary>
-        /// Contains information about the jFIF marker
+        /// Contains information about the JFIF marker
         /// </summary>
         private JFif jFif;
 
+        /// <summary>
+        /// Contains information about the Adobe marker
+        /// </summary>
+        private Adobe adobe;
+
+        /// <summary>
+        /// A value indicating whether the decoder has been disposed
+        /// </summary>
         private bool isDisposed;
 
         /// <summary>
@@ -208,9 +224,12 @@ namespace ImageSharp.Formats.Jpeg.Port
                 if (disposing)
                 {
                     this.frame.Dispose();
+                    this.components.Dispose();
                 }
 
-                // TODO: set large fields to null.
+                // Set large fields to null.
+                this.frame = null;
+                this.components = null;
                 this.isDisposed = true;
             }
         }
@@ -255,9 +274,16 @@ namespace ImageSharp.Formats.Jpeg.Port
                     case JpegConstants.Markers.APP11:
                     case JpegConstants.Markers.APP12:
                     case JpegConstants.Markers.APP13:
+                        break;
+
                     case JpegConstants.Markers.APP14:
+                        this.ProcessApp14Marker(remaining);
+                        break;
+
                     case JpegConstants.Markers.APP15:
                     case JpegConstants.Markers.COM:
+
+                        // TODO: Read data block
                         break;
 
                     case JpegConstants.Markers.DQT:
@@ -275,17 +301,66 @@ namespace ImageSharp.Formats.Jpeg.Port
                         break;
 
                     case JpegConstants.Markers.DRI:
-                        this.resetInterval = this.ReadUint16();
+                        this.ProcessDefineRestartIntervalMarker(remaining);
                         break;
 
                     case JpegConstants.Markers.SOS:
                         this.ProcessStartOfScanMarker();
                         break;
+
+                    case JpegConstants.Markers.XFF:
+                        if ((byte)this.InputStream.ReadByte() != 0xFF)
+                        {
+                            // Avoid skipping a valid marker
+                            this.InputStream.Position -= 2;
+                        }
+                        else
+                        {
+                            // Rewind that last byte we read
+                            this.InputStream.Position -= 1;
+                        }
+
+                        break;
+
+                    default:
+
+                        // Skip back as it could be incorrect encoding -- last 0xFF byte of the previous
+                        // block was eaten by the encoder
+                        this.InputStream.Position -= 3;
+                        this.InputStream.Read(this.temp, 0, 2);
+                        if (this.temp[0] == 0xFF && this.temp[1] >= 0xC0 && this.temp[1] <= 0xFE)
+                        {
+                            // Rewind that last bytes we read
+                            this.InputStream.Position -= 2;
+                        }
+
+                        break;
                 }
 
-                // Read on
+                // Read on. TODO: Test this on damaged images.
                 fileMarker = FindNextFileMarker(this.InputStream);
             }
+
+            this.width = this.frame.SamplesPerLine;
+            this.height = this.frame.Scanlines;
+            this.components = new ComponentBlocks { Components = new Component[this.frame.ComponentCount] };
+
+            for (int i = 0; i < this.components.Components.Length; i++)
+            {
+                ref var frameComponent = ref this.frame.Components[i];
+                var component = new Component
+                {
+                    ScaleX = frameComponent.HorizontalFactor / this.frame.MaxHorizontalFactor,
+                    ScaleY = frameComponent.VerticalFactor / this.frame.MaxVerticalFactor,
+                    BlocksPerLine = frameComponent.BlocksPerLine,
+                    BlocksPerColumn = frameComponent.BlocksPerColumn
+                };
+
+                this.BuildComponentData(ref component);
+                this.components.Components[i] = component;
+            }
+
+            this.numComponents = this.components.Components.Length;
         }
 
         /// <summary>
@@ -322,7 +397,47 @@ namespace ImageSharp.Formats.Jpeg.Port
                 };
             }
 
-            // Skip thumbnails for now.
+            // TODO: thumbnail
+            if (remaining > 0)
+            {
+                this.InputStream.Skip(remaining);
+            }
+        }
+
+        /// <summary>
+        /// Processes the application header containing the Adobe identifier
+        /// which stores image encoding information for DCT filters.
+        /// </summary>
+        /// <param name="remaining">The remaining bytes in the segment block.</param>
+        private void ProcessApp14Marker(int remaining)
+        {
+            if (remaining < 12)
+            {
+                // Skip the application header length
+                this.InputStream.Skip(remaining);
+                return;
+            }
+
+            this.InputStream.Read(this.temp, 0, 12);
+            remaining -= 12;
+
+            bool isAdobe = this.temp[0] == JpegConstants.Markers.Adobe.A &&
+                          this.temp[1] == JpegConstants.Markers.Adobe.D &&
+                          this.temp[2] == JpegConstants.Markers.Adobe.O &&
+                          this.temp[3] == JpegConstants.Markers.Adobe.B &&
+                          this.temp[4] == JpegConstants.Markers.Adobe.E;
+
+            if (isAdobe)
+            {
+                this.adobe = new Adobe
+                {
+                    DCTEncodeVersion = (short)((this.temp[5] << 8) | this.temp[6]),
+                    APP14Flags0 = (short)((this.temp[7] << 8) | this.temp[8]),
+                    APP14Flags1 = (short)((this.temp[9] << 8) | this.temp[10]),
+                    ColorTransform = this.temp[11]
+                };
+            }
+
             if (remaining > 0)
             {
                 this.InputStream.Skip(remaining);
@@ -439,7 +554,7 @@ namespace ImageSharp.Formats.Jpeg.Port
             this.frame.ComponentIds = new byte[this.frame.ComponentCount];
             this.frame.Components = new FrameComponent[this.frame.ComponentCount];
 
-            for (int i = 0; i < this.frame.ComponentCount; i++)
+            for (int i = 0; i < this.frame.Components.Length; i++)
             {
                 int h = this.temp[index + 1] >> 4;
                 int v = this.temp[index + 1] & 15;
@@ -462,7 +577,6 @@ namespace ImageSharp.Formats.Jpeg.Port
 
                 this.frame.ComponentIds[i] = (byte)i;
 
-                // Don't assign the table yet.
                 index += 3;
             }
 
@@ -517,6 +631,21 @@ namespace ImageSharp.Formats.Jpeg.Port
             }
         }
 
+        /// <summary>
+        /// Processes the DRI (Define Restart Interval Marker) Which specifies the interval between RSTn markers, in
+        /// macroblocks
+        /// </summary>
+        /// <param name="remaining">The remaining bytes in the segment block.</param>
+        private void ProcessDefineRestartIntervalMarker(int remaining)
+        {
+            if (remaining != 2)
+            {
+                throw new ImageFormatException("DRI has wrong length");
+            }
+
+            this.resetInterval = this.ReadUint16();
+        }
+
         /// <summary>
         /// Processes the SOS (Start of scan marker).
         /// </summary>
@@ -552,6 +681,15 @@ namespace ImageSharp.Formats.Jpeg.Port
                successiveApproximation & 15);
         }
 
+        /// <summary>
+        /// Build the data for the given component
+        /// </summary>
+        /// <param name="component">The component</param>
+        private void BuildComponentData(ref Component component)
+        {
+            // TODO: Write this
+        }
+
         /// <summary>
         /// Builds the huffman tables
         /// </summary>
@@ -648,41 +786,5 @@ namespace ImageSharp.Formats.Jpeg.Port
             this.InputStream.Read(this.uint16Buffer, 0, 2);
             return (ushort)((this.uint16Buffer[0] << 8) | this.uint16Buffer[1]);
         }
-
-        /// <summary>
-        /// Provides information about the JFIF marker segment
-        /// </summary>
-        internal struct JFif
-        {
-            /// <summary>
-            /// The major version
-            /// </summary>
-            public byte MajorVersion;
-
-            /// <summary>
-            /// The minor version
-            /// </summary>
-            public byte MinorVersion;
-
-            /// <summary>
-            /// Units for the following pixel density fields
-            ///  00 : No units; width:height pixel aspect ratio = Ydensity:Xdensity
-            ///  01 : Pixels per inch (2.54 cm)
-            ///  02 : Pixels per centimeter
-            /// </summary>
-            public byte DensityUnits;
-
-            /// <summary>
-            /// Horizontal pixel density. Must not be zero.
-            /// </summary>
-            public short XDensity;
-
-            /// <summary>
-            /// Vertical pixel density. Must not be zero.
-            /// </summary>
-            public short YDensity;
-
-            // TODO: Thumbnail?
-        }
     }
 }
\ No newline at end of file