diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
index bfae7fd756..06b46746a6 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FastACTables.cs
@@ -42,7 +42,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         /// <param name="index">The table index.</param>
         /// <param name="acHuffmanTables">The collection of AC Huffman tables.</param>
-        public void BuildACTableLut(int index, HuffmanTables acHuffmanTables)
+        public unsafe void BuildACTableLut(int index, HuffmanTables acHuffmanTables)
         {
             const int FastBits = ScanDecoder.FastBits;
             Span<short> fastAC = this.tables.GetRowSpan(index);
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs
deleted file mode 100644
index 1d26178e0c..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer256.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedByteBuffer256
-    {
-        public fixed byte Data[256];
-
-        public byte this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref byte self = ref Unsafe.As<FixedByteBuffer256, byte>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs
deleted file mode 100644
index 556e74fd58..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedByteBuffer512.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedByteBuffer512
-    {
-        public fixed byte Data[1 << ScanDecoder.FastBits];
-
-        public byte this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref byte self = ref Unsafe.As<FixedByteBuffer512, byte>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs
deleted file mode 100644
index a3b67a700b..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt16Buffer257.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedInt16Buffer257
-    {
-        public fixed short Data[257];
-
-        public short this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref short self = ref Unsafe.As<FixedInt16Buffer257, short>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs
deleted file mode 100644
index bba89f072f..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedInt32Buffer18.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedInt32Buffer18
-    {
-        public fixed int Data[18];
-
-        public int this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref int self = ref Unsafe.As<FixedInt32Buffer18, int>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs
deleted file mode 100644
index 1d3ca99338..0000000000
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/FixedUInt32Buffer18.cs
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-
-namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
-{
-    [StructLayout(LayoutKind.Sequential)]
-    internal unsafe struct FixedUInt32Buffer18
-    {
-        public fixed uint Data[18];
-
-        public uint this[int idx]
-        {
-            [MethodImpl(MethodImplOptions.AggressiveInlining)]
-            get
-            {
-                ref uint self = ref Unsafe.As<FixedUInt32Buffer18, uint>(ref this);
-                return Unsafe.Add(ref self, idx);
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
index ffc4ce9829..08e8604f17 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
@@ -20,27 +20,27 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <summary>
         /// Gets the max code array
         /// </summary>
-        public FixedUInt32Buffer18 MaxCode;
+        public fixed uint MaxCode[18];
 
         /// <summary>
         /// Gets the value offset array
         /// </summary>
-        public FixedInt32Buffer18 ValOffset;
+        public fixed int ValOffset[18];
 
         /// <summary>
         /// Gets the huffman value array
         /// </summary>
-        public FixedByteBuffer256 Values;
+        public fixed byte Values[256];
 
         /// <summary>
         /// Gets the lookahead array
         /// </summary>
-        public FixedByteBuffer512 Lookahead;
+        public fixed byte Lookahead[512];
 
         /// <summary>
         /// Gets the sizes array
         /// </summary>
-        public FixedInt16Buffer257 Sizes;
+        public fixed short Sizes[257];
 
         /// <summary>
         /// Initializes a new instance of the <see cref="HuffmanTable"/> struct.
@@ -57,7 +57,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 ref byte codeLengthsRef = ref MemoryMarshal.GetReference(codeLengths);
 
                 // Figure C.1: make table of Huffman code length for each symbol
-                ref short sizesRef = ref this.Sizes.Data[0];
+                ref short sizesRef = ref this.Sizes[0];
                 short x = 0;
 
                 for (short i = 1; i < 17; i++)
@@ -73,8 +73,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
                 // Figure C.2: generate the codes themselves
                 int si = 0;
-                ref int valOffsetRef = ref this.ValOffset.Data[0];
-                ref uint maxcodeRef = ref this.MaxCode.Data[0];
+                ref int valOffsetRef = ref this.ValOffset[0];
+                ref uint maxcodeRef = ref this.MaxCode[0];
 
                 uint code = 0;
                 int k;
@@ -91,7 +91,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                     }
 
                     // Figure F.15: generate decoding tables for bit-sequential decoding.
-                    // Compute largest code + 1 for this size. preshifted as we need later.
+                    // Compute largest code + 1 for this size. preshifted as we needit later.
                     Unsafe.Add(ref maxcodeRef, k) = code << (16 - k);
                     code <<= 1;
                 }
@@ -100,8 +100,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
                 // Generate non-spec lookup tables to speed up decoding.
                 const int FastBits = ScanDecoder.FastBits;
-                ref byte fastRef = ref this.Lookahead.Data[0];
-                new Span<byte>(Unsafe.AsPointer(ref fastRef), 1 << FastBits).Fill(0xFF); // Flag for non-accelerated
+                ref byte fastRef = ref this.Lookahead[0];
+                Unsafe.InitBlockUnaligned(ref fastRef, 0xFF, 1 << FastBits); // Flag for non-accelerated
 
                 for (int i = 0; i < si; i++)
                 {
@@ -118,7 +118,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 }
             }
 
-            values.CopyTo(new Span<byte>(Unsafe.AsPointer(ref this.Values.Data[0]), 256));
+            values.CopyTo(new Span<byte>(Unsafe.AsPointer(ref this.Values[0]), 256));
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
index 6741ccdac2..351e453484 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ScanDecoder.cs
@@ -800,7 +800,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         [MethodImpl(InliningOptions.ShortMethod)]
-        private int DecodeHuffman(ref HuffmanTable table)
+        private unsafe int DecodeHuffman(ref HuffmanTable table)
         {
             this.CheckBits();
 
@@ -825,7 +825,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         }
 
         [MethodImpl(InliningOptions.ColdPath)]
-        private int DecodeHuffmanSlow(ref HuffmanTable table)
+        private unsafe int DecodeHuffmanSlow(ref HuffmanTable table)
         {
             // Naive test is to shift the code_buffer down so k bits are
             // valid, then test against MaxCode. To speed this up, we've