diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
index 26bcde8e51..06b46746a6 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
@@ -20,54 +20,47 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// Initializes a new instance of the <see cref="FastACTables"/> class.
         /// </summary>
         /// <param name="memoryAllocator">The memory allocator used to allocate memory for image processing operations.</param>
-        public FastACTables(MemoryAllocator memoryAllocator)
-        {
-            this.tables = memoryAllocator.Allocate2D<short>(512, 4, AllocationOptions.Clean);
-        }
+        public FastACTables(MemoryAllocator memoryAllocator) => this.tables = memoryAllocator.Allocate2D<short>(512, 4, AllocationOptions.Clean);
 
         /// <summary>
         /// Gets the <see cref="Span{Int16}"/> representing the table at the index in the collection.
         /// </summary>
         /// <param name="index">The table index.</param>
         /// <returns><see cref="Span{Int16}"/></returns>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public ReadOnlySpan<short> GetTableSpan(int index)
-        {
-            return this.tables.GetRowSpan(index);
-        }
+        [MethodImpl(InliningOptions.ShortMethod)]
+        public ReadOnlySpan<short> GetTableSpan(int index) => this.tables.GetRowSpan(index);
 
         /// <summary>
-        /// Gets a reference to the first element of the AC table indexed by <see cref="JpegComponent.ACHuffmanTableId"/>       /// </summary>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public ref short GetAcTableReference(JpegComponent component)
-        {
-            return ref this.tables.GetRowSpan(component.ACHuffmanTableId)[0];
-        }
+        /// Gets a reference to the first element of the AC table indexed by <see cref="JpegComponent.ACHuffmanTableId"/>
+        /// </summary>
+        /// <param name="component">The frame component.</param>
+        [MethodImpl(InliningOptions.ShortMethod)]
+        public ref short GetAcTableReference(JpegComponent component) => ref this.tables.GetRowSpan(component.ACHuffmanTableId)[0];
 
         /// <summary>
         /// Builds a lookup table for fast AC entropy scan decoding.
         /// </summary>
         /// <param name="index">The table index.</param>
         /// <param name="acHuffmanTables">The collection of AC Huffman tables.</param>
-        public void BuildACTableLut(int index, HuffmanTables acHuffmanTables)
+        public unsafe void BuildACTableLut(int index, HuffmanTables acHuffmanTables)
         {
             const int FastBits = ScanDecoder.FastBits;
             Span<short> fastAC = this.tables.GetRowSpan(index);
-            ref HuffmanTable huffman = ref acHuffmanTables[index];
+            ref HuffmanTable huffmanTable = ref acHuffmanTables[index];
 
             int i;
             for (i = 0; i < (1 << FastBits); i++)
             {
-                byte fast = huffman.Lookahead[i];
+                byte fast = huffmanTable.Lookahead[i];
                 fastAC[i] = 0;
                 if (fast < byte.MaxValue)
                 {
-                    int rs = huffman.Values[fast];
+                    int rs = huffmanTable.Values[fast];
                     int run = (rs >> 4) & 15;
                     int magbits = rs & 15;
-                    int len = huffman.Sizes[fast];
+                    int len = huffmanTable.Sizes[fast];
 
-                    if (magbits > 0 && len + magbits <= FastBits)
+                    if (magbits != 0 && len + magbits <= FastBits)
                     {
                         // Magnitude code followed by receive_extend code
                         int k = ((i << len) & ((1 << FastBits) - 1)) >> (FastBits - magbits);
@@ -80,7 +73,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                         // if the result is small enough, we can fit it in fastAC table
                         if (k >= -128 && k <= 127)
                         {
-                            fastAC[i] = (short)((k * 256) + (run * 16) + (len + magbits));
+                            fastAC[i] = (short)((k << 8) + (run << 4) + (len + magbits));
                         }
                     }
                 }
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs
deleted file mode 100644
index 1d26178e0c..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedByteBuffer256
-    {
-        public fixed byte Data[256];
-
-        public byte this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref byte self = ref Unsafe.As<FixedByteBuffer256, byte>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs
deleted file mode 100644
index 556e74fd58..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedByteBuffer512
-    {
-        public fixed byte Data[1 << ScanDecoder.FastBits];
-
-        public byte this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref byte self = ref Unsafe.As<FixedByteBuffer512, byte>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs
deleted file mode 100644
index a3b67a700b..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedInt16Buffer257
-    {
-        public fixed short Data[257];
-
-        public short this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref short self = ref Unsafe.As<FixedInt16Buffer257, short>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs
deleted file mode 100644
index bba89f072f..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedInt32Buffer18
-    {
-        public fixed int Data[18];
-
-        public int this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref int self = ref Unsafe.As<FixedInt32Buffer18, int>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs
deleted file mode 100644
index 1d3ca99338..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedUInt32Buffer18
-    {
-        public fixed uint Data[18];
-
-        public uint this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref uint self = ref Unsafe.As<FixedUInt32Buffer18, uint>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
index 0138164ed2..24d570bf1c 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
@@ -20,114 +20,105 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <summary>
         /// Gets the max code array
         /// </summary>
-        public FixedUInt32Buffer18 MaxCode;
+        public fixed uint MaxCode[18];
 
         /// <summary>
         /// Gets the value offset array
         /// </summary>
-        public FixedInt32Buffer18 ValOffset;
+        public fixed int ValOffset[18];
 
         /// <summary>
         /// Gets the huffman value array
         /// </summary>
-        public FixedByteBuffer256 Values;
+        public fixed byte Values[256];
 
         /// <summary>
         /// Gets the lookahead array
         /// </summary>
-        public FixedByteBuffer512 Lookahead;
+        public fixed byte Lookahead[512];
 
         /// <summary>
         /// Gets the sizes array
         /// </summary>
-        public FixedInt16Buffer257 Sizes;
+        public fixed short Sizes[257];
 
         /// <summary>
         /// Initializes a new instance of the <see cref="HuffmanTable"/> struct.
         /// </summary>
         /// <param name="memoryAllocator">The <see cref="MemoryAllocator"/> to use for buffer allocations.</param>
-        /// <param name="count">The code lengths</param>
+        /// <param name="codeLengths">The code lengths</param>
         /// <param name="values">The huffman values</param>
-        public HuffmanTable(MemoryAllocator memoryAllocator, ReadOnlySpan<byte> count, ReadOnlySpan<byte> values)
+        public HuffmanTable(MemoryAllocator memoryAllocator, ReadOnlySpan<byte> codeLengths, ReadOnlySpan<byte> values)
         {
             const int Length = 257;
             using (IMemoryOwner<short> huffcode = memoryAllocator.Allocate<short>(Length))
             {
                 ref short huffcodeRef = ref MemoryMarshal.GetReference(huffcode.GetSpan());
+                ref byte codeLengthsRef = ref MemoryMarshal.GetReference(codeLengths);
 
                 // Figure C.1: make table of Huffman code length for each symbol
-                fixed (short* sizesRef = this.Sizes.Data)
+                ref short sizesRef = ref this.Sizes[0];
+                short x = 0;
+
+                for (short i = 1; i < 17; i++)
                 {
-                    short x = 0;
-                    for (short i = 1; i < 17; i++)
+                    byte length = Unsafe.Add(ref codeLengthsRef, i);
+                    for (short j = 0; j < length; j++)
                     {
-                        byte l = count[i];
-                        for (short j = 0; j < l; j++)
-                        {
-                            sizesRef[x] = i;
-                            x++;
-                        }
+                        Unsafe.Add(ref sizesRef, x++) = i;
                     }
+                }
 
-                    sizesRef[x] = 0;
+                Unsafe.Add(ref sizesRef, x) = 0;
 
-                    // Figure C.2: generate the codes themselves
-                    int k = 0;
-                    fixed (int* valOffsetRef = this.ValOffset.Data)
-                    fixed (uint* maxcodeRef = this.MaxCode.Data)
+                // Figure C.2: generate the codes themselves
+                int si = 0;
+                ref int valOffsetRef = ref this.ValOffset[0];
+                ref uint maxcodeRef = ref this.MaxCode[0];
+
+                uint code = 0;
+                int k;
+                for (k = 1; k < 17; k++)
+                {
+                    // Compute delta to add to code to compute symbol id.
+                    Unsafe.Add(ref valOffsetRef, k) = (int)(si - code);
+                    if (Unsafe.Add(ref sizesRef, si) == k)
                     {
-                        uint code = 0;
-                        int j;
-                        for (j = 1; j < 17; j++)
+                        while (Unsafe.Add(ref sizesRef, si) == k)
                         {
-                            // Compute delta to add to code to compute symbol id.
-                            valOffsetRef[j] = (int)(k - code);
-                            if (sizesRef[k] == j)
-                            {
-                                while (sizesRef[k] == j)
-                                {
-                                    Unsafe.Add(ref huffcodeRef, k++) = (short)code++;
-                                }
-                            }
-
-                            // Figure F.15: generate decoding tables for bit-sequential decoding.
-                            // Compute largest code + 1 for this size. preshifted as need later.
-                            maxcodeRef[j] = code << (16 - j);
-                            code <<= 1;
+                            Unsafe.Add(ref huffcodeRef, si++) = (short)code++;
                         }
-
-                        maxcodeRef[j] = 0xFFFFFFFF;
                     }
 
-                    // Generate non-spec lookup tables to speed up decoding.
-                    fixed (byte* lookaheadRef = this.Lookahead.Data)
-                    {
-                        const int FastBits = ScanDecoder.FastBits;
-                        var fast = new Span<byte>(lookaheadRef, 1 << FastBits);
-                        fast.Fill(0xFF); // Flag for non-accelerated
+                    // Figure F.15: generate decoding tables for bit-sequential decoding.
+                    // Compute largest code + 1 for this size. preshifted as we needit later.
+                    Unsafe.Add(ref maxcodeRef, k) = code << (16 - k);
+                    code <<= 1;
+                }
+
+                Unsafe.Add(ref maxcodeRef, k) = 0xFFFFFFFF;
 
-                        for (int i = 0; i < k; i++)
+                // Generate non-spec lookup tables to speed up decoding.
+                const int FastBits = ScanDecoder.FastBits;
+                ref byte fastRef = ref this.Lookahead[0];
+                Unsafe.InitBlockUnaligned(ref fastRef, 0xFF, 1 << FastBits); // Flag for non-accelerated
+
+                for (int i = 0; i < si; i++)
+                {
+                    int size = Unsafe.Add(ref sizesRef, i);
+                    if (size <= FastBits)
+                    {
+                        int c = Unsafe.Add(ref huffcodeRef, i) << (FastBits - size);
+                        int m = 1 << (FastBits - size);
+                        for (int l = 0; l < m; l++)
                         {
-                            int s = sizesRef[i];
-                            if (s <= ScanDecoder.FastBits)
-                            {
-                                int c = Unsafe.Add(ref huffcodeRef, i) << (FastBits - s);
-                                int m = 1 << (FastBits - s);
-                                for (int j = 0; j < m; j++)
-                                {
-                                    fast[c + j] = (byte)i;
-                                }
-                            }
+                            Unsafe.Add(ref fastRef, c + l) = (byte)i;
                         }
                     }
                 }
             }
 
-            fixed (byte* huffValRef = this.Values.Data)
-            {
-                var huffValSpan = new Span<byte>(huffValRef, 256);
-                values.CopyTo(huffValSpan);
-            }
+            Unsafe.CopyBlockUnaligned(ref this.Values[0], ref MemoryMarshal.GetReference(values), 256);
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
index da089fa44a..36a3dc2d26 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegFrame.cs
@@ -45,6 +45,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         public byte[] ComponentIds { get; set; }
 
+        /// <summary>
+        /// Gets or sets the order in which to process the components
+        /// in interleaved mode.
+        /// </summary>
+        public byte[] ComponentOrder { get; set; }
+
         /// <summary>
         /// Gets or sets the frame component collection
         /// </summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
index 8c525335bc..351e453484 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
@@ -34,9 +34,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         // The restart interval.
         private readonly int restartInterval;
 
-        // The current component index.
-        private readonly int componentIndex;
-
         // The number of interleaved components.
         private readonly int componentsLength;
 
@@ -87,7 +84,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <param name="dcHuffmanTables">The DC Huffman tables.</param>
         /// <param name="acHuffmanTables">The AC Huffman tables.</param>
         /// <param name="fastACTables">The fast AC decoding tables.</param>
-        /// <param name="componentIndex">The component index within the array.</param>
         /// <param name="componentsLength">The length of the components. Different to the array length.</param>
         /// <param name="restartInterval">The reset interval.</param>
         /// <param name="spectralStart">The spectral selection start.</param>
@@ -100,7 +96,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             HuffmanTables dcHuffmanTables,
             HuffmanTables acHuffmanTables,
             FastACTables fastACTables,
-            int componentIndex,
             int componentsLength,
             int restartInterval,
             int spectralStart,
@@ -117,7 +112,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             this.components = frame.Components;
             this.marker = JpegConstants.Markers.XFF;
             this.markerPosition = 0;
-            this.componentIndex = componentIndex;
             this.componentsLength = componentsLength;
             this.restartInterval = restartInterval;
             this.spectralStart = spectralStart;
@@ -176,7 +170,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                     // Scan an interleaved mcu... process components in order
                     for (int k = 0; k < this.componentsLength; k++)
                     {
-                        JpegComponent component = this.components[k];
+                        int order = this.frame.ComponentOrder[k];
+                        JpegComponent component = this.components[order];
 
                         ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
                         ref HuffmanTable acHuffmanTable = ref this.acHuffmanTables[component.ACHuffmanTableId];
@@ -223,14 +218,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         /// <summary>
-        /// Non-interleaved data, we just need to process one block at a ti
-        /// in trivial scanline order
-        /// number of blocks to do just depends on how many actual "pixels"
-        /// component has, independent of interleaved MCU blocking and such
+        /// Non-interleaved data, we just need to process one block at a time in trivial scanline order
+        /// number of blocks to do just depends on how many actual "pixels" each component has,
+        /// independent of interleaved MCU blocking and such.
         /// </summary>
         private void ParseBaselineDataNonInterleaved()
         {
-            JpegComponent component = this.components[this.componentIndex];
+            JpegComponent component = this.components[this.frame.ComponentOrder[0]];
 
             int w = component.WidthInBlocks;
             int h = component.HeightInBlocks;
@@ -295,7 +289,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                     // Scan an interleaved mcu... process components in order
                     for (int k = 0; k < this.componentsLength; k++)
                     {
-                        JpegComponent component = this.components[k];
+                        int order = this.frame.ComponentOrder[k];
+                        JpegComponent component = this.components[order];
                         ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
                         int h = component.HorizontalSamplingFactor;
                         int v = component.VerticalSamplingFactor;
@@ -344,7 +339,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         private void ParseProgressiveDataNonInterleaved()
         {
-            JpegComponent component = this.components[this.componentIndex];
+            JpegComponent component = this.components[this.frame.ComponentOrder[0]];
 
             int w = component.WidthInBlocks;
             int h = component.HeightInBlocks;
@@ -729,8 +724,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             }
 
             uint k = LRot(this.codeBuffer, n);
-            this.codeBuffer = k & ~Bmask[n];
-            k &= Bmask[n];
+            uint mask = Bmask[n];
+            this.codeBuffer = k & ~mask;
+            k &= mask;
             this.codeBits -= n;
             return (int)k;
         }
@@ -804,7 +800,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         [MethodImpl(InliningOptions.ShortMethod)]
-        private int DecodeHuffman(ref HuffmanTable table)
+        private unsafe int DecodeHuffman(ref HuffmanTable table)
         {
             this.CheckBits();
 
@@ -829,7 +825,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         [MethodImpl(InliningOptions.ColdPath)]
-        private int DecodeHuffmanSlow(ref HuffmanTable table)
+        private unsafe int DecodeHuffmanSlow(ref HuffmanTable table)
         {
             // Naive test is to shift the code_buffer down so k bits are
             // valid, then test against MaxCode. To speed this up, we've
@@ -839,7 +835,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             // that way we don't need to shift inside the loop.
             uint temp = this.codeBuffer >> 16;
             int k;
-            for (k = FastBits + 1; ; k++)
+            for (k = FastBits + 1; ; ++k)
             {
                 if (temp < table.MaxCode[k])
                 {
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index e3b0b4bdc6..22d9cbdee4 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -747,11 +747,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             if (!metadataOnly)
             {
                 // No need to pool this. They max out at 4
-                this.Frame.ComponentIds = new byte[this.Frame.ComponentCount];
-                this.Frame.Components = new JpegComponent[this.Frame.ComponentCount];
+                this.Frame.ComponentIds = new byte[this.ComponentCount];
+                this.Frame.ComponentOrder = new byte[this.ComponentCount];
+                this.Frame.Components = new JpegComponent[this.ComponentCount];
                 this.ColorSpace = this.DeduceJpegColorSpace();
 
-                for (int i = 0; i < this.Frame.ComponentCount; i++)
+                for (int i = 0; i < this.ComponentCount; i++)
                 {
                     byte hv = this.temp[index + 1];
                     int h = hv >> 4;
@@ -823,10 +824,10 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                                 codeLengths.GetSpan(),
                                 huffmanValues.GetSpan());
 
-                            if (huffmanTableSpec >> 4 != 0)
+                            if (tableType != 0)
                             {
                                 // Build a table that decodes both magnitude and value of small ACs in one go.
-                                this.fastACTables.BuildACTableLut(huffmanTableSpec & 15, this.acHuffmanTables);
+                                this.fastACTables.BuildACTableLut(tableIndex, this.acHuffmanTables);
                             }
                         }
                     }
@@ -867,6 +868,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                     if (selector == id)
                     {
                         componentIndex = j;
+                        break;
                     }
                 }
 
@@ -879,6 +881,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                 int tableSpec = this.InputStream.ReadByte();
                 component.DCHuffmanTableId = tableSpec >> 4;
                 component.ACHuffmanTableId = tableSpec & 15;
+                this.Frame.ComponentOrder[i] = (byte)componentIndex;
             }
 
             this.InputStream.Read(this.temp, 0, 3);
@@ -893,7 +896,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                 this.dcHuffmanTables,
                 this.acHuffmanTables,
                 this.fastACTables,
-                componentIndex,
                 selectorsCount,
                 this.resetInterval,
                 spectralStart,
diff --git a/src/ImageSharp/Processing/CropExtensions.cs b/src/ImageSharp/Processing/CropExtensions.cs
index 34c754a08e..1c0d80afc9 100644
--- a/src/ImageSharp/Processing/CropExtensions.cs
+++ b/src/ImageSharp/Processing/CropExtensions.cs
@@ -35,6 +35,6 @@ namespace SixLabors.ImageSharp.Processing
         /// <returns>The <see cref="Image{TPixel}"/></returns>
         public static IImageProcessingContext<TPixel> Crop<TPixel>(this IImageProcessingContext<TPixel> source, Rectangle cropRectangle)
             where TPixel : struct, IPixel<TPixel>
-            => source.ApplyProcessor(new CropProcessor<TPixel>(cropRectangle));
+            => source.ApplyProcessor(new CropProcessor<TPixel>(cropRectangle, source.GetCurrentSize()));
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Processing/Processors/Transforms/CropProcessor.cs b/src/ImageSharp/Processing/Processors/Transforms/CropProcessor.cs
index 8e6a826fd3..3b1d7e94dd 100644
--- a/src/ImageSharp/Processing/Processors/Transforms/CropProcessor.cs
+++ b/src/ImageSharp/Processing/Processors/Transforms/CropProcessor.cs
@@ -22,8 +22,11 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
         /// Initializes a new instance of the <see cref="CropProcessor{TPixel}"/> class.
         /// </summary>
         /// <param name="cropRectangle">The target cropped rectangle.</param>
-        public CropProcessor(Rectangle cropRectangle)
+        /// <param name="sourceSize">The source image size.</param>
+        public CropProcessor(Rectangle cropRectangle, Size sourceSize)
         {
+            // Check bounds here and throw if we are passed a rectangle exceeding our source bounds.
+            Guard.IsTrue(new Rectangle(Point.Empty, sourceSize).Contains(cropRectangle), nameof(cropRectangle), "Crop rectangle should be smaller than the source bounds.");
             this.CropRectangle = cropRectangle;
         }
 
@@ -53,7 +56,7 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
                 return;
             }
 
-            var rect = Rectangle.Intersect(this.CropRectangle, sourceRectangle);
+            Rectangle rect = this.CropRectangle;
 
             // Copying is cheap, we should process more pixels per task:
             ParallelExecutionSettings parallelSettings = configuration.GetParallelSettings().MultiplyMinimumPixelsPerTask(4);
diff --git a/src/ImageSharp/Processing/Processors/Transforms/EntropyCropProcessor.cs b/src/ImageSharp/Processing/Processors/Transforms/EntropyCropProcessor.cs
index 8eeae5d1fc..6de717afd9 100644
--- a/src/ImageSharp/Processing/Processors/Transforms/EntropyCropProcessor.cs
+++ b/src/ImageSharp/Processing/Processors/Transforms/EntropyCropProcessor.cs
@@ -62,7 +62,7 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
                 rectangle = ImageMaths.GetFilteredBoundingRectangle(temp, 0);
             }
 
-            new CropProcessor<TPixel>(rectangle).Apply(source, sourceRectangle);
+            new CropProcessor<TPixel>(rectangle, source.Size()).Apply(source, sourceRectangle);
         }
 
         /// <inheritdoc/>
diff --git a/tests/ImageSharp.Tests/BaseImageOperationsExtensionTest.cs b/tests/ImageSharp.Tests/BaseImageOperationsExtensionTest.cs
index 34b2f718ee..7adbefb346 100644
--- a/tests/ImageSharp.Tests/BaseImageOperationsExtensionTest.cs
+++ b/tests/ImageSharp.Tests/BaseImageOperationsExtensionTest.cs
@@ -14,7 +14,9 @@ namespace SixLabors.ImageSharp.Tests
         private readonly FakeImageOperationsProvider.FakeImageOperations<Rgba32> internalOperations;
         protected readonly Rectangle rect;
         protected readonly GraphicsOptions options;
-        private Image<Rgba32> source;
+        private readonly Image<Rgba32> source;
+
+        public Rectangle SourceBounds() => this.source.Bounds();
 
         public BaseImageOperationsExtensionTest()
         {
@@ -29,7 +31,7 @@ namespace SixLabors.ImageSharp.Tests
         {
             Assert.InRange(index, 0, this.internalOperations.Applied.Count - 1);
 
-            var operation = this.internalOperations.Applied[index];
+            FakeImageOperationsProvider.FakeImageOperations<Rgba32>.AppliedOperation operation = this.internalOperations.Applied[index];
 
             return Assert.IsType<T>(operation.Processor);
         }
@@ -38,7 +40,7 @@ namespace SixLabors.ImageSharp.Tests
         {
             Assert.InRange(index, 0, this.internalOperations.Applied.Count - 1);
 
-            var operation = this.internalOperations.Applied[index];
+            FakeImageOperationsProvider.FakeImageOperations<Rgba32>.AppliedOperation operation = this.internalOperations.Applied[index];
 
             Assert.Equal(rect, operation.Rectangle);
             return Assert.IsType<T>(operation.Processor);
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.Images.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.Images.cs
index 2f041e3ab4..6bc559978c 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.Images.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegDecoderTests.Images.cs
@@ -44,6 +44,9 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
                 TestImages.Jpeg.Issues.BadRstProgressive518,
                 TestImages.Jpeg.Issues.MissingFF00ProgressiveBedroom159,
                 TestImages.Jpeg.Issues.DhtHasWrongLength624,
+                TestImages.Jpeg.Issues.OrderedInterleavedProgressive723A,
+                TestImages.Jpeg.Issues.OrderedInterleavedProgressive723B,
+                TestImages.Jpeg.Issues.OrderedInterleavedProgressive723C
             };
 
         /// <summary>
diff --git a/tests/ImageSharp.Tests/ImageOperationTests.cs b/tests/ImageSharp.Tests/ImageOperationTests.cs
index d73eea6870..869882f672 100644
--- a/tests/ImageSharp.Tests/ImageOperationTests.cs
+++ b/tests/ImageSharp.Tests/ImageOperationTests.cs
@@ -56,7 +56,7 @@ namespace SixLabors.ImageSharp.Tests
         [Fact]
         public void CloneCallsImageOperationsProvider_Func_WithDuplicateImage()
         {
-            var returned = this.image.Clone(x => x.ApplyProcessor(this.processor));
+            Image<Rgba32> returned = this.image.Clone(x => x.ApplyProcessor(this.processor));
 
             Assert.True(this.provider.HasCreated(returned));
             Assert.Contains(this.processor, this.provider.AppliedOperations(returned).Select(x => x.Processor));
@@ -65,7 +65,7 @@ namespace SixLabors.ImageSharp.Tests
         [Fact]
         public void CloneCallsImageOperationsProvider_ListOfProcessors_WithDuplicateImage()
         {
-            var returned = this.image.Clone(this.processor);
+            Image<Rgba32> returned = this.image.Clone(this.processor);
 
             Assert.True(this.provider.HasCreated(returned));
             Assert.Contains(this.processor, this.provider.AppliedOperations(returned).Select(x => x.Processor));
@@ -74,7 +74,7 @@ namespace SixLabors.ImageSharp.Tests
         [Fact]
         public void CloneCallsImageOperationsProvider_Func_NotOnOrigional()
         {
-            var returned = this.image.Clone(x => x.ApplyProcessor(this.processor));
+            Image<Rgba32> returned = this.image.Clone(x => x.ApplyProcessor(this.processor));
             Assert.False(this.provider.HasCreated(this.image));
             Assert.DoesNotContain(this.processor, this.provider.AppliedOperations(this.image).Select(x => x.Processor));
         }
@@ -82,7 +82,7 @@ namespace SixLabors.ImageSharp.Tests
         [Fact]
         public void CloneCallsImageOperationsProvider_ListOfProcessors_NotOnOrigional()
         {
-            var returned = this.image.Clone(this.processor);
+            Image<Rgba32> returned = this.image.Clone(this.processor);
             Assert.False(this.provider.HasCreated(this.image));
             Assert.DoesNotContain(this.processor, this.provider.AppliedOperations(this.image).Select(x => x.Processor));
         }
@@ -95,9 +95,6 @@ namespace SixLabors.ImageSharp.Tests
             Assert.Contains(this.processor, operations.Applied.Select(x => x.Processor));
         }
 
-        public void Dispose()
-        {
-            this.image.Dispose();
-        }
+        public void Dispose() => this.image.Dispose();
     }
 }
\ No newline at end of file
diff --git a/tests/ImageSharp.Tests/Processing/Processors/Transforms/CropTest.cs b/tests/ImageSharp.Tests/Processing/Processors/Transforms/CropTest.cs
index 2f78915120..c01c3b1bd3 100644
--- a/tests/ImageSharp.Tests/Processing/Processors/Transforms/CropTest.cs
+++ b/tests/ImageSharp.Tests/Processing/Processors/Transforms/CropTest.cs
@@ -17,7 +17,6 @@ namespace SixLabors.ImageSharp.Tests.Processing.Processors.Transforms
     {
         [Theory]
         [WithTestPatternImages(70, 30, PixelTypes.Rgba32, 0, 0, 70, 30)]
-        [WithTestPatternImages(50, 50, PixelTypes.Rgba32, -1, -1, 100, 200)]
         [WithTestPatternImages(30, 70, PixelTypes.Rgba32, 7, 13, 20, 50)]
         public void Crop<TPixel>(TestImageProvider<TPixel> provider, int x, int y, int w, int h)
             where TPixel : struct, IPixel<TPixel>
diff --git a/tests/ImageSharp.Tests/Processing/Transforms/CropTest.cs b/tests/ImageSharp.Tests/Processing/Transforms/CropTest.cs
index 154167f15f..6731debd36 100644
--- a/tests/ImageSharp.Tests/Processing/Transforms/CropTest.cs
+++ b/tests/ImageSharp.Tests/Processing/Transforms/CropTest.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 
+using System;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
 using SixLabors.ImageSharp.Processing.Processors.Transforms;
@@ -33,5 +34,12 @@ namespace SixLabors.ImageSharp.Tests.Processing.Transforms
 
             Assert.Equal(cropRectangle, processor.CropRectangle);
         }
+
+        [Fact]
+        public void CropRectangleWithInvalidBoundsThrowsException()
+        {
+            var cropRectangle = Rectangle.Inflate(this.SourceBounds(), 5, 5);
+            Assert.Throws<ArgumentException>(() => this.operations.Crop(cropRectangle));
+        }
     }
 }
\ No newline at end of file
diff --git a/tests/ImageSharp.Tests/TestImages.cs b/tests/ImageSharp.Tests/TestImages.cs
index 309fb1d4ab..fdf586c430 100644
--- a/tests/ImageSharp.Tests/TestImages.cs
+++ b/tests/ImageSharp.Tests/TestImages.cs
@@ -156,6 +156,9 @@ namespace SixLabors.ImageSharp.Tests
                 public const string InvalidEOI695 = "Jpg/issues/Issue695-Invalid-EOI.jpg";
                 public const string ExifResizeOutOfRange696 = "Jpg/issues/Issue696-Resize-Exif-OutOfRange.jpg";
                 public const string InvalidAPP0721 = "Jpg/issues/Issue721-InvalidAPP0.jpg";
+                public const string OrderedInterleavedProgressive723A = "Jpg/issues/Issue723-Ordered-Interleaved-Progressive-A.jpg";
+                public const string OrderedInterleavedProgressive723B = "Jpg/issues/Issue723-Ordered-Interleaved-Progressive-B.jpg";
+                public const string OrderedInterleavedProgressive723C = "Jpg/issues/Issue723-Ordered-Interleaved-Progressive-C.jpg";
             }
 
             public static readonly string[] All = Baseline.All.Concat(Progressive.All).ToArray();
diff --git a/tests/Images/External b/tests/Images/External
index 5f3cbd839f..c0627f384c 160000
--- a/tests/Images/External
+++ b/tests/Images/External
@@ -1 +1 @@
-Subproject commit 5f3cbd839fbbffae615d294d1dabafdcabc64cf9
+Subproject commit c0627f384c1d3d2f8d914c9578ae31354c35fd2c
diff --git a/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-A.jpg b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-A.jpg
new file mode 100644
index 0000000000..0a11065ce9
Binary files /dev/null and b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-A.jpg differ
diff --git a/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-B.jpg b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-B.jpg
new file mode 100644
index 0000000000..eb52570e1c
Binary files /dev/null and b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-B.jpg differ
diff --git a/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-C.jpg b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-C.jpg
new file mode 100644
index 0000000000..0224cb7f1f
Binary files /dev/null and b/tests/Images/Input/Jpg/issues/Issue723-Ordered-Interleaved-Progressive-C.jpg differ