diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index adb0ad4b0..4c6772a6f 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -1095,12 +1095,15 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// <param name="remaining">The remaining bytes in the segment block.</param>
         private void ProcessDefineHuffmanTablesMarker(BufferedReadStream stream, int remaining)
         {
-            int length = remaining;
+            const int codeLengthsByteSize = 16;
+            const int codeValuesMaxByteSize = 256;
+            const int tableWorkspaceByteSize = 257 * sizeof(uint);
+            const int totalBufferSize = codeLengthsByteSize + codeValuesMaxByteSize + tableWorkspaceByteSize;
 
-            using (IMemoryOwner<byte> huffmanData = this.Configuration.MemoryAllocator.Allocate<byte>(256, AllocationOptions.Clean))
+            int length = remaining;
+            using (IMemoryOwner<byte> huffmanData = this.Configuration.MemoryAllocator.Allocate<byte>(17))
             {
                 Span<byte> huffmanDataSpan = huffmanData.GetSpan();
-                ref byte huffmanDataRef = ref MemoryMarshal.GetReference(huffmanDataSpan);
                 for (int i = 2; i < remaining;)
                 {
                     byte huffmanTableSpec = (byte)stream.ReadByte();
@@ -1119,40 +1122,34 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                         JpegThrowHelper.ThrowInvalidImageContentException($"Bad huffman table index: {tableIndex}.");
                     }
 
-                    stream.Read(huffmanDataSpan, 0, 16);
+                    stream.Read(huffmanDataSpan, 1, 16);
 
-                    using (IMemoryOwner<byte> codeLengths = this.Configuration.MemoryAllocator.Allocate<byte>(17, AllocationOptions.Clean))
+                    int codeLengthSum = 0;
+                    for (int j = 1; j < 17; j++)
                     {
-                        Span<byte> codeLengthsSpan = codeLengths.GetSpan();
-                        ref byte codeLengthsRef = ref MemoryMarshal.GetReference(codeLengthsSpan);
-                        int codeLengthSum = 0;
-
-                        for (int j = 1; j < 17; j++)
-                        {
-                            codeLengthSum += Unsafe.Add(ref codeLengthsRef, j) = Unsafe.Add(ref huffmanDataRef, j - 1);
-                        }
+                        codeLengthSum += huffmanDataSpan[j];
+                    }
 
-                        length -= 17;
+                    length -= 17;
 
-                        if (codeLengthSum > 256 || codeLengthSum > length)
-                        {
-                            JpegThrowHelper.ThrowInvalidImageContentException("Huffman table has excessive length.");
-                        }
+                    if (codeLengthSum > 256 || codeLengthSum > length)
+                    {
+                        JpegThrowHelper.ThrowInvalidImageContentException("Huffman table has excessive length.");
+                    }
 
-                        using (IMemoryOwner<byte> huffmanValues = this.Configuration.MemoryAllocator.Allocate<byte>(256, AllocationOptions.Clean))
-                        {
-                            Span<byte> huffmanValuesSpan = huffmanValues.GetSpan();
-                            stream.Read(huffmanValuesSpan, 0, codeLengthSum);
+                    using (IMemoryOwner<byte> huffmanValues = this.Configuration.MemoryAllocator.Allocate<byte>(256, AllocationOptions.Clean))
+                    {
+                        Span<byte> huffmanValuesSpan = huffmanValues.GetSpan();
+                        stream.Read(huffmanValuesSpan, 0, codeLengthSum);
 
-                            i += 17 + codeLengthSum;
+                        i += 17 + codeLengthSum;
 
-                            this.scanDecoder.BuildHuffmanTable(
-                                tableType,
-                                tableIndex,
-                                codeLengthsSpan,
-                                huffmanValuesSpan,
-                                stackalloc uint[257]);
-                        }
+                        this.scanDecoder.BuildHuffmanTable(
+                            tableType,
+                            tableIndex,
+                            huffmanDataSpan,
+                            huffmanValuesSpan,
+                            stackalloc uint[257]);
                     }
                 }
             }