diff --git a/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.cs b/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.cs
index 3f71c498b2..53297ab550 100644
--- a/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.cs
+++ b/src/ImageSharp/Formats/Jpeg/Common/Block8x8F.cs
@@ -353,13 +353,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
         /// <param name="qtPtr">Qt pointer</param>
         /// <param name="unzigPtr">Unzig pointer</param>
         // [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static unsafe void DequantizeBlock(Block8x8F* blockPtr, Block8x8F* qtPtr, int* unzigPtr)
+        public static unsafe void DequantizeBlock(Block8x8F* blockPtr, Block8x8F* qtPtr, byte* unzigPtr)
         {
             float* b = (float*)blockPtr;
             float* qtp = (float*)qtPtr;
             for (int qtIndex = 0; qtIndex < Size; qtIndex++)
             {
-                int blockIndex = unzigPtr[qtIndex];
+                byte blockIndex = unzigPtr[qtIndex];
                 float* unzigPos = b + blockIndex;
 
                 float val = *unzigPos;
@@ -381,7 +381,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
             Block8x8F* block,
             Block8x8F* dest,
             Block8x8F* qt,
-            int* unzigPtr)
+            byte* unzigPtr)
         {
             float* s = (float*)block;
             float* d = (float*)dest;
diff --git a/src/ImageSharp/Formats/Jpeg/Common/ZigZag.cs b/src/ImageSharp/Formats/Jpeg/Common/ZigZag.cs
index 18270f5bad..cb035a8d3d 100644
--- a/src/ImageSharp/Formats/Jpeg/Common/ZigZag.cs
+++ b/src/ImageSharp/Formats/Jpeg/Common/ZigZag.cs
@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors and contributors.
 // Licensed under the Apache License, Version 2.0.
 using System;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 namespace SixLabors.ImageSharp.Formats.Jpeg.Common
@@ -11,25 +12,52 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
     /// unzig[3] is the column and row of the fourth element in zigzag order. The
     /// value is 16, which means first column (16%8 == 0) and third row (16/8 == 2).
     /// </summary>
+    [StructLayout(LayoutKind.Sequential)]
     internal unsafe struct ZigZag
     {
         /// <summary>
         /// Copy of <see cref="Unzig"/> in a value type
         /// </summary>
-        public fixed int Data[64];
+        public fixed byte Data[64];
 
         /// <summary>
         /// Unzig maps from the zigzag ordering to the natural ordering. For example,
         /// unzig[3] is the column and row of the fourth element in zigzag order. The
         /// value is 16, which means first column (16%8 == 0) and third row (16/8 == 2).
         /// </summary>
-        private static readonly int[] Unzig =
+        private static readonly byte[] Unzig =
+        {
+            0,
+            1, 8,
+            16, 9, 2,
+            3, 10, 17, 24,
+            32, 25, 18, 11, 4,
+            5, 12, 19, 26, 33, 40,
+            48, 41, 34, 27, 20, 13, 6,
+            7, 14, 21, 28, 35, 42, 49, 56,
+            57, 50, 43, 36, 29, 22, 15,
+            23, 30, 37, 44, 51, 58,
+            59, 52, 45, 38, 31,
+            39, 46, 53, 60,
+            61, 54, 47,
+            55, 62,
+            63
+        };
+
+        /// <summary>
+        /// Returns the value at the given index
+        /// </summary>
+        /// <param name="idx">The index</param>
+        /// <returns>The <see cref="byte"/></returns>
+        public byte this[int idx]
+        {
+            [MethodImpl(MethodImplOptions.AggressiveInlining)]
+            get
             {
-                0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, 12, 19, 26, 33,
-                40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, 35, 42, 49, 56, 57, 50,
-                43, 36, 29, 22, 15, 23, 30, 37, 44, 51, 58, 59, 52, 45, 38, 31, 39, 46,
-                53, 60, 61, 54, 47, 55, 62, 63,
-            };
+                ref byte self = ref Unsafe.As<ZigZag, byte>(ref this);
+                return Unsafe.Add(ref self, idx);
+            }
+        }
 
         /// <summary>
         /// Creates and fills an instance of <see cref="ZigZag"/> with Jpeg unzig indices
@@ -37,8 +65,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
         /// <returns>The new instance</returns>
         public static ZigZag CreateUnzigTable()
         {
-            ZigZag result = default(ZigZag);
-            int* unzigPtr = result.Data;
+            ZigZag result = default;
+            byte* unzigPtr = result.Data;
             Marshal.Copy(Unzig, 0, (IntPtr)unzigPtr, 64);
             return result;
         }
@@ -48,7 +76,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Common
         /// </summary>
         public static Block8x8F CreateDequantizationTable(ref Block8x8F qt)
         {
-            Block8x8F result = default(Block8x8F);
+            Block8x8F result = default;
 
             for (int i = 0; i < 64; i++)
             {
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigJpegScanDecoder.DataPointers.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigJpegScanDecoder.DataPointers.cs
index 0098b4a4ed..0207280e3e 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigJpegScanDecoder.DataPointers.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/Components/Decoder/OrigJpegScanDecoder.DataPointers.cs
@@ -21,9 +21,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort.Components.Decoder
             public Block8x8* Block;
 
             /// <summary>
-            /// Pointer to <see cref="ComputationData.Unzig"/> as int*
+            /// Pointer to <see cref="ComputationData.Unzig"/> as byte*
             /// </summary>
-            public int* Unzig;
+            public byte* Unzig;
 
             /// <summary>
             /// Pointer to <see cref="ComputationData.ScanData"/> as Scan*
diff --git a/src/ImageSharp/Formats/Jpeg/GolangPort/JpegEncoderCore.cs b/src/ImageSharp/Formats/Jpeg/GolangPort/JpegEncoderCore.cs
index ba40ef72b8..4fbb20ee82 100644
--- a/src/ImageSharp/Formats/Jpeg/GolangPort/JpegEncoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/GolangPort/JpegEncoderCore.cs
@@ -489,7 +489,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.GolangPort
             Block8x8F* tempDest1,
             Block8x8F* tempDest2,
             Block8x8F* quant,
-            int* unzigPtr)
+            byte* unzigPtr)
         {
             FastFloatingPointDCT.TransformFDCT(ref *src, ref *tempDest1, ref *tempDest2);
 
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
index 0917abef2e..f9320443ac 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsScanDecoder.cs
@@ -17,27 +17,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
     /// </summary>
     internal struct PdfJsScanDecoder
     {
-        /// <summary>
-        /// Gets the ZigZag scan table
-        /// </summary>
-        private static readonly byte[] DctZigZag =
-        {
-            0,
-            1, 8,
-            16, 9, 2,
-            3, 10, 17, 24,
-            32, 25, 18, 11, 4,
-            5, 12, 19, 26, 33, 40,
-            48, 41, 34, 27, 20, 13, 6,
-            7, 14, 21, 28, 35, 42, 49, 56,
-            57, 50, 43, 36, 29, 22, 15,
-            23, 30, 37, 44, 51, 58,
-            59, 52, 45, 38, 31,
-            39, 46, 53, 60,
-            61, 54, 47,
-            55, 62,
-            63
-        };
+        private ZigZag dctZigZag;
 
         private byte[] markerBuffer;
 
@@ -98,6 +78,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             int successivePrev,
             int successive)
         {
+            this.dctZigZag = ZigZag.CreateUnzigTable();
             this.markerBuffer = new byte[2];
             this.compIndex = componentIndex;
             this.specStart = spectralStart;
@@ -195,7 +176,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void DecodeScanBaseline(
             PdfJsHuffmanTables dcHuffmanTables,
             PdfJsHuffmanTables acHuffmanTables,
@@ -256,7 +236,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void DecodeScanProgressive(
             PdfJsHuffmanTables huffmanTables,
             bool isAC,
@@ -598,7 +577,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             return n + (-1 << length) + 1;
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void DecodeBaseline(PdfJsFrameComponent component, ref short blockDataRef, int offset, ref PdfJsHuffmanTable dcHuffmanTable, ref PdfJsHuffmanTable acHuffmanTable, Stream stream)
         {
             short t = this.DecodeHuffman(ref dcHuffmanTable, stream);
@@ -640,7 +618,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
                     break;
                 }
 
-                ref byte z = ref DctZigZag[k];
+                byte z = this.dctZigZag[k];
                 short re = (short)this.ReceiveAndExtend(s, stream);
                 Unsafe.Add(ref blockDataRef, offset + z) = re;
                 k++;
@@ -672,7 +650,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             Unsafe.Add(ref blockDataRef, offset) |= (short)(bit << this.successiveState);
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void DecodeACFirst(PdfJsFrameComponent component, ref short blockDataRef, int offset, ref PdfJsHuffmanTable acHuffmanTable, Stream stream)
         {
             if (this.eobrun > 0)
@@ -708,13 +685,12 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
 
                 k += r;
 
-                ref byte z = ref DctZigZag[k];
+                byte z = this.dctZigZag[k];
                 Unsafe.Add(ref blockDataRef, offset + z) = (short)(this.ReceiveAndExtend(s, stream) * (1 << this.successiveState));
                 k++;
             }
         }
 
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         private void DecodeACSuccessive(PdfJsFrameComponent component, ref short blockDataRef, int offset, ref PdfJsHuffmanTable acHuffmanTable, Stream stream)
         {
             int k = this.specStart;
@@ -723,7 +699,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
 
             while (k <= e)
             {
-                int offsetZ = offset + DctZigZag[k];
+                int offsetZ = offset + this.dctZigZag[k];
                 ref short blockOffsetZRef = ref Unsafe.Add(ref blockDataRef, offsetZ);
                 int sign = blockOffsetZRef < 0 ? -1 : 1;
 
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/Utils/ReferenceImplementations.cs b/tests/ImageSharp.Tests/Formats/Jpg/Utils/ReferenceImplementations.cs
index 92ead8164f..f1eed08b93 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/Utils/ReferenceImplementations.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/Utils/ReferenceImplementations.cs
@@ -19,13 +19,13 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg.Utils
     /// </summary>
     internal static partial class ReferenceImplementations
     {
-        public static unsafe void DequantizeBlock(Block8x8F* blockPtr, Block8x8F* qtPtr, int* unzigPtr)
+        public static unsafe void DequantizeBlock(Block8x8F* blockPtr, Block8x8F* qtPtr, byte* unzigPtr)
         {
             float* b = (float*)blockPtr;
             float* qtp = (float*)qtPtr;
             for (int qtIndex = 0; qtIndex < Block8x8F.Size; qtIndex++)
             {
-                int i = unzigPtr[qtIndex];
+                byte i = unzigPtr[qtIndex];
                 float* unzigPos = b + i;
 
                 float val = *unzigPos;
@@ -115,7 +115,7 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg.Utils
         /// <param name="dest">The destination block of integers</param>
         /// <param name="qt">The quantization table</param>
         /// <param name="unzigPtr">Pointer to <see cref="ZigZag.Data"/> </param>
-        public static unsafe void QuantizeRational(Block8x8F* src, int* dest, Block8x8F* qt, int* unzigPtr)
+        public static unsafe void QuantizeRational(Block8x8F* src, int* dest, Block8x8F* qt, byte* unzigPtr)
         {
             float* s = (float*)src;
             float* q = (float*)qt;