No need for two counters

src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/FourByte.cs

@@ -0,0 +1,19 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System.Runtime.InteropServices;
+
+namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
+{
+    [StructLayout(LayoutKind.Sequential)]
+    internal readonly struct FourByte
+    {
+        public readonly byte X;
+
+        public readonly byte Y;
+
+        public readonly byte Z;
+
+        public readonly byte W;
+    }
+}

src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsJpegPixelArea.cs

@@ -67,31 +67,37 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             using (IBuffer<int> xScaleBlockOffset = this.memoryManager.Allocate<int>(this.Width))
             {
                 ref int xScaleBlockOffsetRef = ref MemoryMarshal.GetReference(xScaleBlockOffset.Span);
-                for (int i = 0; i < this.NumberOfComponents; i++)
+                int numberOfComponents = this.NumberOfComponents;
+                int width = this.Width;
+                int height = this.Height;
+
+                for (int i = 0; i < numberOfComponents; i++)
                 {
                     ref PdfJsComponent component = ref components.Components[i];
                     ref short outputRef = ref MemoryMarshal.GetReference(component.Output.Span);
                     Vector2 componentScale = component.Scale;
+                    float cX = componentScale.X;
+                    float cY = componentScale.Y;
                     int blocksPerScanline = (component.BlocksPerLine + 1) << 3;
 
                     // Precalculate the xScaleBlockOffset
                     int j;
-                    for (int x = 0; x < this.Width; x++)
+                    for (int x = 0; x < width; x++)
                     {
-                        j = (int)(x * componentScale.X);
+                        j = (int)(x * cX);
                         Unsafe.Add(ref xScaleBlockOffsetRef, x) = (int)((j & Mask3Lsb) << 3) | (j & 7);
                     }
 
                     // Linearize the blocks of the component
                     int offset = i;
-                    for (int y = 0; y < this.Height; y++)
+                    for (int y = 0; y < height; y++)
                     {
-                        j = (int)(y * componentScale.Y);
+                        j = (int)(y * cY);
                         int index = blocksPerScanline * (int)(j & Mask3Lsb) | ((j & 7) << 3);
-                        for (int x = 0; x < this.Width; x++)
+                        for (int x = 0; x < width; x++)
                         {
                             Unsafe.Add(ref componentDataRef, offset) = (byte)Unsafe.Add(ref outputRef, index + Unsafe.Add(ref xScaleBlockOffsetRef, x));
-                            offset += this.NumberOfComponents;
+                            offset += numberOfComponents;
                         }
                     }
                 }

src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/ThreeByte.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Six Labors and contributors.
+// Licensed under the Apache License, Version 2.0.
+
+using System.Runtime.InteropServices;
+
+namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
+{
+    [StructLayout(LayoutKind.Sequential)]
+    internal readonly struct ThreeByte
+    {
+        public readonly byte X;
+
+        public readonly byte Y;
+
+        public readonly byte Z;
+    }
+}

src/ImageSharp/Formats/Jpeg/PdfJsPort/PdfJsJpegDecoderCore.cs

@@ -895,15 +895,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
             for (int y = 0; y < image.Height; y++)
             {
                 ref TPixel imageRowRef = ref MemoryMarshal.GetReference(image.GetPixelRowSpan(y));
-                ref byte areaRowRef = ref MemoryMarshal.GetReference(this.pixelArea.GetRowSpan(y));
+                ref ThreeByte areaRowRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<byte, ThreeByte>(this.pixelArea.GetRowSpan(y)));
 
-                for (int x = 0, o = 0; x < image.Width; x++, o += 3)
+                for (int x = 0; x < image.Width; x++)
                 {
-                    ref byte yy = ref Unsafe.Add(ref areaRowRef, o);
-                    ref byte cb = ref Unsafe.Add(ref areaRowRef, o + 1);
-                    ref byte cr = ref Unsafe.Add(ref areaRowRef, o + 2);
+                    ref ThreeByte ycbcr = ref Unsafe.Add(ref areaRowRef, x);
                     ref TPixel pixel = ref Unsafe.Add(ref imageRowRef, x);
-                    PdfJsYCbCrToRgbTables.PackYCbCr(ref pixel, yy, cb, cr);
+                    PdfJsYCbCrToRgbTables.PackYCbCr(ref pixel, ycbcr.X, ycbcr.Y, ycbcr.Z);
                 }
             }
         }
@@ -915,17 +913,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
             for (int y = 0; y < image.Height; y++)
             {
                 ref TPixel imageRowRef = ref MemoryMarshal.GetReference(image.GetPixelRowSpan(y));
-                ref byte areaRowRef = ref MemoryMarshal.GetReference(this.pixelArea.GetRowSpan(y));
+                ref FourByte areaRowRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<byte, FourByte>(this.pixelArea.GetRowSpan(y)));
 
-                for (int x = 0, o = 0; x < image.Width; x++, o += 4)
+                for (int x = 0; x < image.Width; x++)
                 {
-                    ref byte yy = ref Unsafe.Add(ref areaRowRef, o);
-                    ref byte cb = ref Unsafe.Add(ref areaRowRef, o + 1);
-                    ref byte cr = ref Unsafe.Add(ref areaRowRef, o + 2);
-                    ref byte k = ref Unsafe.Add(ref areaRowRef, o + 3);
-
+                    ref FourByte ycbcrk = ref Unsafe.Add(ref areaRowRef, x);
                     ref TPixel pixel = ref Unsafe.Add(ref imageRowRef, x);
-                    PdfJsYCbCrToRgbTables.PackYccK(ref pixel, yy, cb, cr, k);
+                    PdfJsYCbCrToRgbTables.PackYccK(ref pixel, ycbcrk.X, ycbcrk.Y, ycbcrk.Z, ycbcrk.W);
                 }
             }
         }
@@ -937,18 +931,17 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort
             for (int y = 0; y < image.Height; y++)
             {
                 ref TPixel imageRowRef = ref MemoryMarshal.GetReference(image.GetPixelRowSpan(y));
-                ref byte areaRowRef = ref MemoryMarshal.GetReference(this.pixelArea.GetRowSpan(y));
+                ref FourByte areaRowRef = ref MemoryMarshal.GetReference(MemoryMarshal.Cast<byte, FourByte>(this.pixelArea.GetRowSpan(y)));
 
-                for (int x = 0, o = 0; x < image.Width; x++, o += 4)
+                for (int x = 0; x < image.Width; x++)
                 {
-                    ref byte c = ref Unsafe.Add(ref areaRowRef, o);
-                    ref byte m = ref Unsafe.Add(ref areaRowRef, o + 1);
-                    ref byte cy = ref Unsafe.Add(ref areaRowRef, o + 2);
-                    ref byte k = ref Unsafe.Add(ref areaRowRef, o + 3);
-
-                    byte r = (byte)((c * k) / 255);
-                    byte g = (byte)((m * k) / 255);
-                    byte b = (byte)((cy * k) / 255);
+                    ref FourByte cmyk = ref Unsafe.Add(ref areaRowRef, x);
+                    byte k = cmyk.W;
+
+                    // TODO: We should see if Vector3 breaks this.
+                    byte r = (byte)((cmyk.X * k) / 255);
+                    byte g = (byte)((cmyk.Y * k) / 255);
+                    byte b = (byte)((cmyk.Z * k) / 255);
 
                     ref TPixel pixel = ref Unsafe.Add(ref imageRowRef, x);
                     var rgba = new Rgba32(r, g, b);