diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrTables.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrTables.cs
index 199aa52252..94173d3e43 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrTables.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrTables.cs
@@ -104,7 +104,7 @@ namespace ImageSharp.Formats.Jpg
         /// <param name="g">The green value.</param>
         /// <param name="b">The blue value.</param>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static void Allocate(ref float* yBlockRaw, ref float* cbBlockRaw, ref float* crBlockRaw, RgbToYCbCrTables* tables, int index, int r, int g, int b)
+        public static void Allocate(ref float* yBlockRaw, ref float* cbBlockRaw, ref float* crBlockRaw, ref RgbToYCbCrTables* tables, int index, int r, int g, int b)
         {
             // float y = (0.299F * r) + (0.587F * g) + (0.114F * b);
             yBlockRaw[index] = (tables->YRTable[r] + tables->YGTable[g] + tables->YBTable[b]) >> ScaleBits;
diff --git a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
index 2072ec1e12..9df21a3b72 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegDecoderCore.cs
@@ -7,6 +7,7 @@ namespace ImageSharp.Formats
 {
     using System;
     using System.IO;
+    using System.Runtime.CompilerServices;
     using System.Threading.Tasks;
 
     using ImageSharp.Formats.Jpg;
@@ -680,30 +681,34 @@ namespace ImageSharp.Formats
             using (PixelAccessor<TPixel> pixels = image.Lock())
             {
                 Parallel.For(
-                    0,
-                    image.Height,
-                    image.Configuration.ParallelOptions,
-                    y =>
-                    {
-                        // TODO. How can we use the fixed tables inside the lambda?
-                        fixed (YCbCrToRgbTables* tables = &yCbCrToRgbTables)
-                        {
-                            // TODO: Simplify + optimize + share duplicate code across converter methods
-                            int yo = this.ycbcrImage.GetRowYOffset(y);
-                            int co = this.ycbcrImage.GetRowCOffset(y);
-
-                            for (int x = 0; x < image.Width; x++)
-                            {
-                                byte yy = this.ycbcrImage.YChannel.Pixels[yo + x];
-                                byte cb = this.ycbcrImage.CbChannel.Pixels[co + (x / scale)];
-                                byte cr = this.ycbcrImage.CrChannel.Pixels[co + (x / scale)];
-
-                                TPixel packed = default(TPixel);
-                                YCbCrToRgbTables.Pack(ref packed, tables, yy, cb, cr);
-                                pixels[x, y] = packed;
-                            }
-                        }
-                    });
+                 0,
+                 image.Height,
+                 image.Configuration.ParallelOptions,
+                 y =>
+                 {
+                     // TODO. This Parallel loop doesn't give us the boost it should.
+                     ref byte ycRef = ref this.ycbcrImage.YChannel.Pixels[0];
+                     ref byte cbRef = ref this.ycbcrImage.CbChannel.Pixels[0];
+                     ref byte crRef = ref this.ycbcrImage.CrChannel.Pixels[0];
+                     fixed (YCbCrToRgbTables* tables = &yCbCrToRgbTables)
+                     {
+                         // TODO: Simplify + optimize + share duplicate code across converter methods
+                         int yo = this.ycbcrImage.GetRowYOffset(y);
+                         int co = this.ycbcrImage.GetRowCOffset(y);
+
+                         for (int x = 0; x < image.Width; x++)
+                         {
+                             int cOff = co + (x / scale);
+                             byte yy = Unsafe.Add(ref ycRef, yo + x);
+                             byte cb = Unsafe.Add(ref cbRef, cOff);
+                             byte cr = Unsafe.Add(ref crRef, cOff);
+
+                             TPixel packed = default(TPixel);
+                             YCbCrToRgbTables.Pack(ref packed, tables, yy, cb, cr);
+                             pixels[x, y] = packed;
+                         }
+                     }
+                 });
             }
 
             this.AssignResolution(image);
diff --git a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
index f88efb3d2b..0ce59c6dec 100644
--- a/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
+++ b/src/ImageSharp/Formats/Jpeg/JpegEncoderCore.cs
@@ -328,7 +328,7 @@ namespace ImageSharp.Formats
 
                     int index = j8 + i;
 
-                    RgbToYCbCrTables.Allocate(ref yBlockRaw, ref cbBlockRaw, ref crBlockRaw, tables, index, r, g, b);
+                    RgbToYCbCrTables.Allocate(ref yBlockRaw, ref cbBlockRaw, ref crBlockRaw, ref tables, index, r, g, b);
 
                     dataIdx += 3;
                 }