diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
index 9803bcf2a..6b69e4320 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
@@ -60,10 +60,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         private const int OutputBufferLengthMultiplier = 2;
 
         /// <summary>
-        /// Compiled huffman tree to encode given values.
+        /// The DC Huffman tables.
         /// </summary>
-        /// <remarks>Yields codewords by index consisting of [run length | bitsize].</remarks>
-        private HuffmanLut[] huffmanTables;
+        private readonly HuffmanLut[] dcHuffmanTables;
+
+        /// <summary>
+        /// The AC Huffman tables.
+        /// </summary>
+        private readonly HuffmanLut[] acHuffmanTables;
 
         /// <summary>
         /// Emitted bits 'micro buffer' before being transferred to the <see cref="emitBuffer"/>.
@@ -115,6 +119,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
             this.streamWriteBuffer = new byte[emitBufferByteLength * OutputBufferLengthMultiplier];
 
             this.target = outputStream;
+
+            this.dcHuffmanTables = new HuffmanLut[4];
+            this.acHuffmanTables = new HuffmanLut[4];
         }
 
         /// <summary>
@@ -128,6 +135,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
             get => this.emitWriteIndex < (uint)this.emitBuffer.Length / 2;
         }
 
+        public void BuildHuffmanTable(JpegHuffmanTableConfig table)
+        {
+            HuffmanLut[] tables = table.Class == 0 ? this.dcHuffmanTables : this.acHuffmanTables;
+            tables[table.DestinationIndex] = new HuffmanLut(table.TableSpec);
+        }
+
         public void EncodeInterleavedScan<TPixel>(JpegFrame frame, Image<TPixel> image, Block8x8F[] quantTables, Configuration configuration, CancellationToken cancellationToken)
             where TPixel : unmanaged, IPixel<TPixel>
         {
@@ -159,8 +172,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
                     {
                         JpegComponent component = frame.Components[k];
 
-                        ref HuffmanLut dcHuffmanTable = ref HuffmanLut.DcHuffmanLut[component.DcTableId];
-                        ref HuffmanLut acHuffmanTable = ref HuffmanLut.AcHuffmanLut[component.AcTableId];
+                        ref HuffmanLut dcHuffmanTable = ref this.dcHuffmanTables[component.DcTableId];
+                        ref HuffmanLut acHuffmanTable = ref this.acHuffmanTables[component.AcTableId];
 
                         int h = component.HorizontalSamplingFactor;
                         int v = component.VerticalSamplingFactor;
@@ -202,6 +215,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         {
         }
 
+        private HuffmanLut[] huffmanTables;
+
         /// <summary>
         /// Encodes the image with no subsampling.
         /// </summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanSpec.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanSpec.cs
index 51364e3c7..e0b988b43 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanSpec.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanSpec.cs
@@ -6,7 +6,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
     /// <summary>
     ///     The Huffman encoding specifications.
     /// </summary>
-    internal readonly struct HuffmanSpec
+    public readonly struct HuffmanSpec
     {
 #pragma warning disable SA1118 // ParameterMustNotSpanMultipleLines
 
diff --git a/src/ImageSharp/Formats/Jpeg/JpegEncoder.cs b/src/ImageSharp/Formats/Jpeg/JpegEncoder.cs
index 209be9d5b..d92fc457a 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegEncoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegEncoder.cs
@@ -5,6 +5,7 @@ using System;
 using System.IO;
 using System.Threading;
 using System.Threading.Tasks;
+using SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder;
 using SixLabors.ImageSharp.PixelFormats;
 
 namespace SixLabors.ImageSharp.Formats.Jpeg
@@ -22,6 +23,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
         public JpegFrameConfig JpegFrameConfig { get; set; }
 
+        public JpegScanConfig JpegScanConfig { get; set; }
+
         /// <summary>
         /// Encodes the image to the specified stream from the <see cref="Image{TPixel}"/>.
         /// </summary>
@@ -31,7 +34,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         public void Encode<TPixel>(Image<TPixel> image, Stream stream)
         where TPixel : unmanaged, IPixel<TPixel>
         {
-            var encoder = new JpegEncoderCore(this, this.JpegFrameConfig);
+            var encoder = new JpegEncoderCore(this, this.JpegFrameConfig, this.JpegScanConfig);
             encoder.Encode(image, stream);
         }
 
@@ -46,7 +49,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         public Task EncodeAsync<TPixel>(Image<TPixel> image, Stream stream, CancellationToken cancellationToken)
             where TPixel : unmanaged, IPixel<TPixel>
         {
-            var encoder = new JpegEncoderCore(this, this.JpegFrameConfig);
+            var encoder = new JpegEncoderCore(this, this.JpegFrameConfig, this.JpegScanConfig);
             return encoder.EncodeAsync(image, stream, cancellationToken);
         }
     }
@@ -115,4 +118,18 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
         public int acTableSelector { get; set; }
     }
+
+    public class JpegHuffmanTableConfig
+    {
+        public int Class { get; set; }
+
+        public int DestinationIndex { get; set; }
+
+        public HuffmanSpec TableSpec { get; set; }
+    }
+
+    public class JpegScanConfig
+    {
+        public JpegHuffmanTableConfig[] HuffmanTables { get; set; }
+    }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
index 6d720dfd1..414f08248 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
@@ -46,6 +46,10 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
         private JpegFrameConfig frameConfig;
 
+        private JpegScanConfig scanConfig;
+
+        private HuffmanScanEncoder scanEncoder;
+
         /// <summary>
         /// The output stream. All attempted writes after the first error become no-ops.
         /// </summary>
@@ -55,12 +59,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// Initializes a new instance of the <see cref="JpegEncoderCore"/> class.
         /// </summary>
         /// <param name="options">The options.</param>
-        public JpegEncoderCore(IJpegEncoderOptions options, JpegFrameConfig frameConfig)
+        public JpegEncoderCore(IJpegEncoderOptions options, JpegFrameConfig frameConfig, JpegScanConfig scanConfig)
         {
             this.quality = options.Quality;
 
             this.frameConfig = frameConfig;
             this.colorType = frameConfig.ColorType;
+
+            this.scanConfig = scanConfig;
         }
 
         /// <summary>
@@ -83,6 +89,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
 
             cancellationToken.ThrowIfCancellationRequested();
 
+            this.scanEncoder = new HuffmanScanEncoder(3, stream);
+
             this.outputStream = stream;
             ImageMetadata metadata = image.Metadata;
             JpegMetadata jpegMetadata = metadata.GetJpegMetadata();
@@ -116,14 +124,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
             this.WriteStartOfFrame(image.Width, image.Height, this.frameConfig);
 
             // Write the Huffman tables.
-            this.WriteDefineHuffmanTables(this.frameConfig.Components.Length);
+            this.WriteDefineHuffmanTables(this.scanConfig.HuffmanTables);
 
             // Write the scan header.
             this.WriteStartOfScan(this.frameConfig.Components.Length, this.frameConfig.Components);
 
             var frame = new Components.Encoder.JpegFrame(this.frameConfig, Configuration.Default.MemoryAllocator, image, GetTargetColorSpace(this.frameConfig.ColorType));
             var quantTables = new Block8x8F[] { luminanceQuantTable, chrominanceQuantTable };
-            new HuffmanScanEncoder(3, stream).EncodeInterleavedScan(frame, image, quantTables, Configuration.Default, cancellationToken);
+            this.scanEncoder.EncodeInterleavedScan(frame, image, quantTables, Configuration.Default, cancellationToken);
 
             // Write the End Of Image marker.
             this.WriteEndOfImageMarker();
@@ -273,40 +281,26 @@ namespace SixLabors.ImageSharp.Formats.Jpeg
         /// Writes the Define Huffman Table marker and tables.
         /// </summary>
         /// <param name="componentCount">The number of components to write.</param>
-        private void WriteDefineHuffmanTables(int componentCount)
+        private void WriteDefineHuffmanTables(JpegHuffmanTableConfig[] tables)
         {
-            // This uses a C#'s compiler optimization that refers to the static data segment of the assembly,
-            // and doesn't incur any allocation at all.
-            // Table identifiers.
-            ReadOnlySpan<byte> headers = new byte[]
-            {
-                0x00,
-                0x10,
-                0x01,
-                0x11
-            };
-
             int markerlen = 2;
-            HuffmanSpec[] specs = HuffmanSpec.TheHuffmanSpecs;
 
-            if (componentCount == 1)
+            for (int i = 0; i < tables.Length; i++)
             {
-                // Drop the Chrominance tables.
-                specs = new[] { HuffmanSpec.TheHuffmanSpecs[0], HuffmanSpec.TheHuffmanSpecs[1] };
-            }
-
-            for (int i = 0; i < specs.Length; i++)
-            {
-                ref HuffmanSpec s = ref specs[i];
-                markerlen += 1 + 16 + s.Values.Length;
+                markerlen += 1 + 16 + tables[i].TableSpec.Values.Length;
             }
 
             this.WriteMarkerHeader(JpegConstants.Markers.DHT, markerlen);
-            for (int i = 0; i < specs.Length; i++)
+            for (int i = 0; i < tables.Length; i++)
             {
-                this.outputStream.WriteByte(headers[i]);
-                this.outputStream.Write(specs[i].Count);
-                this.outputStream.Write(specs[i].Values);
+                JpegHuffmanTableConfig table = tables[i];
+
+                int header = (table.Class << 4) | table.DestinationIndex;
+                this.outputStream.WriteByte((byte)header);
+                this.outputStream.Write(table.TableSpec.Count);
+                this.outputStream.Write(table.TableSpec.Values);
+
+                this.scanEncoder.BuildHuffmanTable(table);
             }
         }