diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
index a09c7ada3e..97ec45ec1e 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/HuffmanScanDecoder.cs
@@ -31,6 +31,16 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         // The End-Of-Block countdown for ending the sequence prematurely when the remaining coefficients are zero.
         private int eobrun;
 
+        /// <summary>
+        /// The DC Huffman tables.
+        /// </summary>
+        private readonly HuffmanTable[] dcHuffmanTables;
+
+        /// <summary>
+        /// The AC Huffman tables
+        /// </summary>
+        private readonly HuffmanTable[] acHuffmanTables;
+
         // The unzig data.
         private ZigZag dctZigZag;
 
@@ -55,12 +65,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             this.stream = stream;
             this.spectralConverter = converter;
             this.cancellationToken = cancellationToken;
-        }
-
-        // huffman tables
-        public HuffmanTable[] DcHuffmanTables { get; set; }
 
-        public HuffmanTable[] AcHuffmanTables { get; set; }
+            // TODO: this is actually a variable value depending on component count
+            const int maxTables = 4;
+            this.dcHuffmanTables = new HuffmanTable[maxTables];
+            this.acHuffmanTables = new HuffmanTable[maxTables];
+        }
 
         // Reset interval
         public int ResetInterval
@@ -148,8 +158,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 int order = this.frame.ComponentOrder[i];
                 JpegComponent component = this.components[order];
 
-                ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
-                ref HuffmanTable acHuffmanTable = ref this.AcHuffmanTables[component.ACHuffmanTableId];
+                ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
+                ref HuffmanTable acHuffmanTable = ref this.acHuffmanTables[component.ACHuffmanTableId];
                 dcHuffmanTable.Configure();
                 acHuffmanTable.Configure();
             }
@@ -168,8 +178,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                         int order = this.frame.ComponentOrder[k];
                         JpegComponent component = this.components[order];
 
-                        ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
-                        ref HuffmanTable acHuffmanTable = ref this.AcHuffmanTables[component.ACHuffmanTableId];
+                        ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
+                        ref HuffmanTable acHuffmanTable = ref this.acHuffmanTables[component.ACHuffmanTableId];
 
                         int h = component.HorizontalSamplingFactor;
                         int v = component.VerticalSamplingFactor;
@@ -221,8 +231,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             int w = component.WidthInBlocks;
             int h = component.HeightInBlocks;
 
-            ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
-            ref HuffmanTable acHuffmanTable = ref this.AcHuffmanTables[component.ACHuffmanTableId];
+            ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
+            ref HuffmanTable acHuffmanTable = ref this.acHuffmanTables[component.ACHuffmanTableId];
             dcHuffmanTable.Configure();
             acHuffmanTable.Configure();
 
@@ -327,7 +337,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             {
                 int order = this.frame.ComponentOrder[k];
                 JpegComponent component = this.components[order];
-                ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
+                ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
                 dcHuffmanTable.Configure();
             }
 
@@ -342,7 +352,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                     {
                         int order = this.frame.ComponentOrder[k];
                         JpegComponent component = this.components[order];
-                        ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
+                        ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
 
                         int h = component.HorizontalSamplingFactor;
                         int v = component.VerticalSamplingFactor;
@@ -390,7 +400,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
             if (this.SpectralStart == 0)
             {
-                ref HuffmanTable dcHuffmanTable = ref this.DcHuffmanTables[component.DCHuffmanTableId];
+                ref HuffmanTable dcHuffmanTable = ref this.dcHuffmanTables[component.DCHuffmanTableId];
                 dcHuffmanTable.Configure();
 
                 for (int j = 0; j < h; j++)
@@ -418,7 +428,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             }
             else
             {
-                ref HuffmanTable acHuffmanTable = ref this.AcHuffmanTables[component.ACHuffmanTableId];
+                ref HuffmanTable acHuffmanTable = ref this.acHuffmanTables[component.ACHuffmanTableId];
                 acHuffmanTable.Configure();
 
                 for (int j = 0; j < h; j++)
@@ -722,5 +732,19 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
             return false;
         }
+
+        /// <summary>
+        /// Build the huffman table using code lengths and code values.
+        /// </summary>
+        /// <param name="type">Table type.</param>
+        /// <param name="index">Table index.</param>
+        /// <param name="codeLengths">Code lengths.</param>
+        /// <param name="values">Code values.</param>
+        [MethodImpl(InliningOptions.ShortMethod)]
+        public void BuildHuffmanTable(int type, int index, ReadOnlySpan<byte> codeLengths, ReadOnlySpan<byte> values)
+        {
+            HuffmanTable[] tables = type == 0 ? this.dcHuffmanTables : this.acHuffmanTables;
+            tables[index] = new HuffmanTable(codeLengths, values);
+        }
     }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 922e9797cb..ee723f062e 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -42,16 +42,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// </summary>
         private readonly byte[] markerBuffer = new byte[2];
 
-        /// <summary>
-        /// The DC Huffman tables.
-        /// </summary>
-        private HuffmanTable[] dcHuffmanTables;
-
-        /// <summary>
-        /// The AC Huffman tables
-        /// </summary>
-        private HuffmanTable[] acHuffmanTables;
-
         /// <summary>
         /// The reset interval determined by RST markers.
         /// </summary>
@@ -270,14 +260,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             fileMarker = new JpegFileMarker(marker, (int)stream.Position - 2);
             this.QuantizationTables = new Block8x8F[4];
 
-            // Only assign what we need
-            if (!metadataOnly)
-            {
-                const int maxTables = 4;
-                this.dcHuffmanTables = new HuffmanTable[maxTables];
-                this.acHuffmanTables = new HuffmanTable[maxTables];
-            }
-
             // Break only when we discover a valid EOI marker.
             // https://github.com/SixLabors/ImageSharp/issues/695
             while (fileMarker.Marker != JpegConstants.Markers.EOI
@@ -392,8 +374,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
             // Set large fields to null.
             this.Frame = null;
-            this.dcHuffmanTables = null;
-            this.acHuffmanTables = null;
+            this.scanDecoder = null;
         }
 
         /// <summary>
@@ -996,8 +977,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
                             i += 17 + codeLengthSum;
 
-                            this.BuildHuffmanTable(
-                                tableType == 0 ? this.dcHuffmanTables : this.acHuffmanTables,
+                            this.scanDecoder.BuildHuffmanTable(
+                                tableType,
                                 tableIndex,
                                 codeLengthsSpan,
                                 huffmanValuesSpan);
@@ -1071,10 +1052,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             // All the comments below are for separate refactoring PR
             // Main reason it's not fixed here is to make this commit less intrusive
 
-            // Huffman tables can be calculated directly in the scan decoder class
-            this.scanDecoder.DcHuffmanTables = this.dcHuffmanTables;
-            this.scanDecoder.AcHuffmanTables = this.acHuffmanTables;
-
             // This can be injectd in DRI marker callback
             this.scanDecoder.ResetInterval = this.resetInterval;
 
@@ -1090,17 +1067,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             this.scanDecoder.ParseEntropyCodedData();
         }
 
-        /// <summary>
-        /// Builds the huffman tables
-        /// </summary>
-        /// <param name="tables">The tables</param>
-        /// <param name="index">The table index</param>
-        /// <param name="codeLengths">The codelengths</param>
-        /// <param name="values">The values</param>
-        [MethodImpl(InliningOptions.ShortMethod)]
-        private void BuildHuffmanTable(HuffmanTable[] tables, int index, ReadOnlySpan<byte> codeLengths, ReadOnlySpan<byte> values)
-            => tables[index] = new HuffmanTable(codeLengths, values);
-
         /// <summary>
         /// Reads a <see cref="ushort"/> from the stream advancing it by two bytes
         /// </summary>