diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
index c06f3f2a04..2ae3ae86bc 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
@@ -727,6 +727,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// <param name="index">Table index.</param>
         /// <param name="codeLengths">Code lengths.</param>
         /// <param name="values">Code values.</param>
+        /// <param name="workspace">The provided spare workspace memory, can be dirty.</param>
         [MethodImpl(InliningOptions.ShortMethod)]
         public void BuildHuffmanTable(int type, int index, ReadOnlySpan<byte> codeLengths, ReadOnlySpan<byte> values, Span<uint> workspace)
         {
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
index 7e69c4d94f..90a966d53f 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanTable.cs
@@ -53,13 +53,11 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         /// </summary>
         /// <param name="codeLengths">The code lengths.</param>
         /// <param name="values">The huffman values.</param>
-        /// <param name="workspace">The spare workspace memory, must be provided by the caller.</param>
+        /// <param name="workspace">The provided spare workspace memory, can be dirty.</param>
         public HuffmanTable(ReadOnlySpan<byte> codeLengths, ReadOnlySpan<byte> values, Span<uint> workspace)
         {
             Unsafe.CopyBlockUnaligned(ref this.Values[0], ref MemoryMarshal.GetReference(values), (uint)values.Length);
 
-            Span<uint> huffCode = workspace;
-
             // Generate codes
             uint code = 0;
             int si = 1;
@@ -69,7 +67,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 int count = codeLengths[i];
                 for (int j = 0; j < count; j++)
                 {
-                    huffCode[p++] = code;
+                    workspace[p++] = code;
                     code++;
                 }
 
@@ -94,9 +92,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             {
                 if (codeLengths[j] != 0)
                 {
-                    this.ValOffset[j] = p - (int)huffCode[p];
+                    this.ValOffset[j] = p - (int)workspace[p];
                     p += codeLengths[j];
-                    this.MaxCode[j] = huffCode[p - 1]; // Maximum code of length l
+                    this.MaxCode[j] = workspace[p - 1]; // Maximum code of length l
                     this.MaxCode[j] <<= JpegConstants.Huffman.RegisterSize - j; // Left justify
                     this.MaxCode[j] |= (1ul << (JpegConstants.Huffman.RegisterSize - j)) - 1;
                 }
@@ -125,7 +123,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 {
                     // length = current code's length, p = its index in huffCode[] & Values[].
                     // Generate left-justified code followed by all possible bit sequences
-                    int lookBits = (int)(huffCode[p] << jShift);
+                    int lookBits = (int)(workspace[p] << jShift);
                     for (int ctr = 1 << (JpegConstants.Huffman.LookupBits - length); ctr > 0; ctr--)
                     {
                         this.LookaheadSize[lookBits] = (byte)length;
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 4c6772a6f6..a2cccc37f5 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -1095,15 +1095,19 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// <param name="remaining">The remaining bytes in the segment block.</param>
         private void ProcessDefineHuffmanTablesMarker(BufferedReadStream stream, int remaining)
         {
-            const int codeLengthsByteSize = 16;
+            const int codeLengthsByteSize = 17;
             const int codeValuesMaxByteSize = 256;
-            const int tableWorkspaceByteSize = 257 * sizeof(uint);
+            const int tableWorkspaceByteSize = 256 * sizeof(uint);
             const int totalBufferSize = codeLengthsByteSize + codeValuesMaxByteSize + tableWorkspaceByteSize;
 
             int length = remaining;
-            using (IMemoryOwner<byte> huffmanData = this.Configuration.MemoryAllocator.Allocate<byte>(17))
+            using (IMemoryOwner<byte> buffer = this.Configuration.MemoryAllocator.Allocate<byte>(totalBufferSize))
             {
-                Span<byte> huffmanDataSpan = huffmanData.GetSpan();
+                Span<byte> bufferSpan = buffer.GetSpan();
+                Span<byte> huffmanLegthsSpan = buffer.Slice(0, codeLengthsByteSize);
+                Span<byte> huffmanValuesSpan = buffer.Slice(codeLengthsByteSize, codeValuesMaxByteSize);
+                Span<uint> tableWorkspace = MemoryMarshal.Cast<byte, uint>(buffer.Slice(codeLengthsByteSize + codeValuesMaxByteSize));
+
                 for (int i = 2; i < remaining;)
                 {
                     byte huffmanTableSpec = (byte)stream.ReadByte();
@@ -1122,12 +1126,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                         JpegThrowHelper.ThrowInvalidImageContentException($"Bad huffman table index: {tableIndex}.");
                     }
 
-                    stream.Read(huffmanDataSpan, 1, 16);
+                    stream.Read(huffmanLegthsSpan, 1, 16);
 
                     int codeLengthSum = 0;
                     for (int j = 1; j < 17; j++)
                     {
-                        codeLengthSum += huffmanDataSpan[j];
+                        codeLengthSum += huffmanLegthsSpan[j];
                     }
 
                     length -= 17;
@@ -1137,20 +1141,16 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
                         JpegThrowHelper.ThrowInvalidImageContentException("Huffman table has excessive length.");
                     }
 
-                    using (IMemoryOwner<byte> huffmanValues = this.Configuration.MemoryAllocator.Allocate<byte>(256, AllocationOptions.Clean))
-                    {
-                        Span<byte> huffmanValuesSpan = huffmanValues.GetSpan();
-                        stream.Read(huffmanValuesSpan, 0, codeLengthSum);
+                    stream.Read(huffmanValuesSpan, 0, codeLengthSum);
 
-                        i += 17 + codeLengthSum;
+                    i += 17 + codeLengthSum;
 
-                        this.scanDecoder.BuildHuffmanTable(
-                            tableType,
-                            tableIndex,
-                            huffmanDataSpan,
-                            huffmanValuesSpan,
-                            stackalloc uint[257]);
-                    }
+                    this.scanDecoder.BuildHuffmanTable(
+                        tableType,
+                        tableIndex,
+                        huffmanLegthsSpan,
+                        huffmanValuesSpan.Slice(0, codeLengthSum),
+                        tableWorkspace);
                 }
             }
         }