diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
index 1958de7c65..80cf7d5498 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
@@ -1,7 +1,8 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 
-using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 using SixLabors.ImageSharp.Memory;
 
 namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
@@ -39,12 +40,16 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// <param name="values">The huffman values</param>
         public PdfJsHuffmanTable(MemoryManager memoryManager, byte[] lengths, byte[] values)
         {
-            using (IBuffer<short> huffsize = memoryManager.Allocate<short>(257))
-            using (IBuffer<short> huffcode = memoryManager.Allocate<short>(257))
+            const int length = 257;
+            using (IBuffer<short> huffsize = memoryManager.Allocate<short>(length))
+            using (IBuffer<short> huffcode = memoryManager.Allocate<short>(length))
             {
-                GenerateSizeTable(lengths, huffsize.Span);
-                GenerateCodeTable(huffsize.Span, huffcode.Span);
-                this.GenerateDecoderTables(lengths, huffcode.Span);
+                ref short huffsizeRef = ref MemoryMarshal.GetReference(huffsize.Span);
+                ref short huffcodeRef = ref MemoryMarshal.GetReference(huffcode.Span);
+
+                GenerateSizeTable(lengths, ref huffsizeRef);
+                GenerateCodeTable(ref huffsizeRef, ref huffcodeRef, length);
+                this.GenerateDecoderTables(lengths, ref huffcodeRef);
                 this.GenerateLookaheadTables(lengths, values);
             }
 
@@ -61,8 +66,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// Figure C.1: make table of Huffman code length for each symbol
         /// </summary>
         /// <param name="lengths">The code lengths</param>
-        /// <param name="huffsize">The huffman size span</param>
-        private static void GenerateSizeTable(byte[] lengths, Span<short> huffsize)
+        /// <param name="huffsizeRef">The huffman size span ref</param>
+        private static void GenerateSizeTable(byte[] lengths, ref short huffsizeRef)
         {
             short index = 0;
             for (short l = 1; l <= 16; l++)
@@ -70,29 +75,30 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 byte i = lengths[l];
                 for (short j = 0; j < i; j++)
                 {
-                    huffsize[index] = l;
+                    Unsafe.Add(ref huffsizeRef, index) = l;
                     index++;
                 }
             }
 
-            huffsize[index] = 0;
+            Unsafe.Add(ref huffsizeRef, index) = 0;
         }
 
         /// <summary>
         /// Figure C.2: generate the codes themselves
         /// </summary>
-        /// <param name="huffsize">The huffman size span</param>
-        /// <param name="huffcode">The huffman code span</param>
-        private static void GenerateCodeTable(Span<short> huffsize, Span<short> huffcode)
+        /// <param name="huffsizeRef">The huffman size span ref</param>
+        /// <param name="huffcodeRef">The huffman code span ref</param>
+        /// <param name="length">The length of the huffsize span</param>
+        private static void GenerateCodeTable(ref short huffsizeRef, ref short huffcodeRef, int length)
         {
             short k = 0;
-            short si = huffsize[0];
+            short si = huffsizeRef;
             short code = 0;
-            for (short i = 0; i < huffsize.Length; i++)
+            for (short i = 0; i < length; i++)
             {
-                while (huffsize[k] == si)
+                while (Unsafe.Add(ref huffsizeRef, k) == si)
                 {
-                    huffcode[k] = code;
+                    Unsafe.Add(ref huffcodeRef, k) = code;
                     code++;
                     k++;
                 }
@@ -106,8 +112,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
         /// Figure F.15: generate decoding tables for bit-sequential decoding
         /// </summary>
         /// <param name="lengths">The code lengths</param>
-        /// <param name="huffcode">The huffman code span</param>
-        private void GenerateDecoderTables(byte[] lengths, Span<short> huffcode)
+        /// <param name="huffcodeRef">The huffman code span ref</param>
+        private void GenerateDecoderTables(byte[] lengths, ref short huffcodeRef)
         {
             fixed (short* valOffsetRef = this.ValOffset.Data)
             fixed (long* maxcodeRef = this.MaxCode.Data)
@@ -117,10 +123,10 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                 {
                     if (lengths[i] != 0)
                     {
-                        // valOffsetRef[l] = huffval[] index of 1st symbol of code length i, minus the minimum code of length i
-                        valOffsetRef[i] = (short)(bitcount - huffcode[bitcount]);
+                        // valOffsetRef[l] = huffcodeRef[] index of 1st symbol of code length i, minus the minimum code of length i
+                        valOffsetRef[i] = (short)(bitcount - Unsafe.Add(ref huffcodeRef, bitcount));
                         bitcount += lengths[i];
-                        maxcodeRef[i] = huffcode[bitcount - 1]; // maximum code of length i
+                        maxcodeRef[i] = Unsafe.Add(ref huffcodeRef, bitcount - 1); // maximum code of length i
                     }
                     else
                     {