diff --git a/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.CopyTo.cs b/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.CopyTo.cs
index ca167015b1..d8963a8b60 100644
--- a/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.CopyTo.cs
+++ b/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.CopyTo.cs
@@ -26,6 +26,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
                 return;
             }
 
+            ref float destBase = ref area.GetReferenceToOrigin();
+
             // TODO: Optimize: implement all the cases with loopless special code! (T4?)
             for (int y = 0; y < 8; y++)
             {
@@ -40,9 +42,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
 
                     for (int i = 0; i < verticalScale; i++)
                     {
+                        int baseIdx = ((yy + i) * area.Stride) + xx;
+
                         for (int j = 0; j < horizontalScale; j++)
                         {
-                            area[xx + j, yy + i] = value;
+                            // area[xx + j, yy + i] = value;
+                            Unsafe.Add(ref destBase, baseIdx + j) = value;
                         }
                     }
                 }
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
index f065092004..e9f468a85d 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/InputProcessor.cs
@@ -223,7 +223,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 if (this.LastErrorCode == OrigDecoderErrorCode.NoError)
                 {
                     int lutIndex = (this.Bits.Accumulator >> (this.Bits.UnreadBits - OrigHuffmanTree.LutSizeLog2)) & 0xFF;
-                    int v = huffmanTree.ReadLut(lutIndex);
+                    int v = huffmanTree.Lut[lutIndex];
 
                     if (v != 0)
                     {
@@ -262,7 +262,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 this.Bits.UnreadBits--;
                 this.Bits.Mask >>= 1;
 
-                if (code <= huffmanTree.GetMaxCode(i))
+                if (code <= huffmanTree.MaxCodes[i])
                 {
                     result = huffmanTree.GetValue(code, i);
                     return this.LastErrorCode = OrigDecoderErrorCode.NoError;
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigHuffmanTree.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigHuffmanTree.cs
index f773a12dfa..85273c69ed 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigHuffmanTree.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigHuffmanTree.cs
@@ -4,12 +4,14 @@
 using System;
 using System.Buffers;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
 {
     /// <summary>
     /// Represents a Huffman tree
     /// </summary>
+    [StructLayout(LayoutKind.Sequential)]
     internal unsafe struct OrigHuffmanTree
     {
         /// <summary>
@@ -68,29 +70,29 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// are 1 plus the code length, or 0 if the value is too large to fit in
         /// lutSize bits.
         /// </summary>
-        public fixed int Lut[MaxNCodes];
+        public FixedInt32Buffer256 Lut;
 
         /// <summary>
         /// Gets the the decoded values, sorted by their encoding.
         /// </summary>
-        public fixed int Values[MaxNCodes];
+        public FixedInt32Buffer256 Values;
 
         /// <summary>
         /// Gets the array of minimum codes.
         /// MinCodes[i] is the minimum code of length i, or -1 if there are no codes of that length.
         /// </summary>
-        public fixed int MinCodes[MaxCodeLength];
+        public FixedInt32Buffer16 MinCodes;
 
         /// <summary>
         /// Gets the array of maximum codes.
         /// MaxCodes[i] is the maximum code of length i, or -1 if there are no codes of that length.
         /// </summary>
-        public fixed int MaxCodes[MaxCodeLength];
+        public FixedInt32Buffer16 MaxCodes;
 
         /// <summary>
         /// Gets the array of indices. Indices[i] is the index into Values of MinCodes[i].
         /// </summary>
-        public fixed int Indices[MaxCodeLength];
+        public FixedInt32Buffer16 Indices;
 
         /// <summary>
         /// Creates and initializes an array of <see cref="OrigHuffmanTree" /> instances of size <see cref="NumberOfTrees" />
@@ -143,8 +145,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             byte[] values = new byte[MaxNCodes];
             inputProcessor.ReadFull(values, 0, this.Length);
 
-            fixed (int* valuesPtr = this.Values)
-            fixed (int* lutPtr = this.Lut)
+            fixed (int* valuesPtr = this.Values.Data)
+            fixed (int* lutPtr = this.Lut.Data)
             {
                 for (int i = 0; i < values.Length; i++)
                 {
@@ -184,9 +186,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
                 }
             }
 
-            fixed (int* minCodesPtr = this.MinCodes)
-            fixed (int* maxCodesPtr = this.MaxCodes)
-            fixed (int* indicesPtr = this.Indices)
+            fixed (int* minCodesPtr = this.MinCodes.Data)
+            fixed (int* maxCodesPtr = this.MaxCodes.Data)
+            fixed (int* indicesPtr = this.Indices.Data)
             {
                 // Derive minCodes, maxCodes, and indices.
                 int c = 0, index = 0;
@@ -219,31 +221,41 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
         /// <param name="code">The code</param>
         /// <param name="codeLength">The code length</param>
         /// <returns>The value</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public int GetValue(int code, int codeLength)
         {
-            fixed (int* valuesPtr = this.Values)
-            fixed (int* minCodesPtr = this.MinCodes)
-            fixed (int* indicesPtr = this.Indices)
-            {
-                return valuesPtr[indicesPtr[codeLength] + code - minCodesPtr[codeLength]];
-            }
+            return this.Values[this.Indices[codeLength] + code - this.MinCodes[codeLength]];
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int ReadLut(int index)
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct FixedInt32Buffer256
         {
-            fixed (int* lutPtr = this.Lut)
+            public fixed int Data[256];
+
+            public int this[int idx]
             {
-                return lutPtr[index];
+                [MethodImpl(MethodImplOptions.AggressiveInlining)]
+                get
+                {
+                    ref int self = ref Unsafe.As<FixedInt32Buffer256, int>(ref this);
+                    return Unsafe.Add(ref self, idx);
+                }
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public int GetMaxCode(int index)
+        [StructLayout(LayoutKind.Sequential)]
+        internal struct FixedInt32Buffer16
         {
-            fixed (int* maxCodesPtr = this.MaxCodes)
+            public fixed int Data[16];
+
+            public int this[int idx]
             {
-                return maxCodesPtr[index];
+                [MethodImpl(MethodImplOptions.AggressiveInlining)]
+                get
+                {
+                    ref int self = ref Unsafe.As<FixedInt32Buffer16, int>(ref this);
+                    return Unsafe.Add(ref self, idx);
+                }
             }
         }
     }
diff --git a/src/ImageSharp/Memory/BufferArea{T}.cs b/src/ImageSharp/Memory/BufferArea{T}.cs
index e88ed6ca83..588eae483d 100644
--- a/src/ImageSharp/Memory/BufferArea{T}.cs
+++ b/src/ImageSharp/Memory/BufferArea{T}.cs
@@ -137,7 +137,7 @@ namespace SixLabors.ImageSharp.Memory
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private int GetRowIndex(int y)
+        internal int GetRowIndex(int y)
         {
             return (y + this.Rectangle.Y) * this.DestinationBuffer.Width;
         }