diff --git a/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs b/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
index 0d9e152c50..a362af7db1 100644
--- a/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
+++ b/src/ImageSharp/Formats/Jpg/JpegDecoderCore.cs
@@ -34,8 +34,14 @@ namespace ImageSharp.Formats
         /// </summary>
         private const int MaxComponents = 4;
 
+        /// <summary>
+        /// The maximum number of Huffman table classes
+        /// </summary>
         private const int MaxTc = 1;
 
+        /// <summary>
+        /// The maximum number of Huffman table identifiers
+        /// </summary>
         private const int MaxTh = 3;
 
         /// <summary>
@@ -1383,10 +1389,8 @@ namespace ImageSharp.Formats
                             byte cb = this.ycbcrImage.CbChannel[co + (x / scale)];
                             byte cr = this.ycbcrImage.CrChannel[co + (x / scale)];
 
-                            // Implicit casting FTW
-                            Color color = new YCbCr(yy, cb, cr);
                             TColor packed = default(TColor);
-                            packed.PackFromBytes(color.R, color.G, color.B, 255);
+                            this.PackYcbCr<TColor, TPacked>(ref packed, yy, cb, cr);
                             pixels[x, y] = packed;
                         }
                     });
@@ -2098,6 +2102,30 @@ namespace ImageSharp.Formats
             return this.componentArray[0].Identifier == 'R' && this.componentArray[1].Identifier == 'G' && this.componentArray[2].Identifier == 'B';
         }
 
+        /// <summary>
+        /// Optimized method to pack bytes to the image from the YCbCr color space. 
+        /// This is faster than implicit casting as it avoids double packing.
+        /// </summary>
+        /// <typeparam name="TColor">The pixel format.</typeparam>
+        /// <typeparam name="TPacked">The packed format. <example>uint, long, float.</example></typeparam>
+        /// <param name="packed">The packed pixel.</param>
+        /// <param name="y">The y luminance component.</param>
+        /// <param name="cb">The cb chroma component.</param>
+        /// <param name="cr">The cr chroma component.</param>
+        private void PackYcbCr<TColor, TPacked>(ref TColor packed, byte y, byte cb, byte cr)
+            where TColor : struct, IPackedPixel<TPacked>
+            where TPacked : struct
+        {
+            int ccb = cb - 128;
+            int ccr = cr - 128;
+
+            byte r = (byte)(y + (1.402F * ccr)).Clamp(0, 255);
+            byte g = (byte)(y - (0.34414F * ccb) - (0.71414F * ccr)).Clamp(0, 255);
+            byte b = (byte)(y + (1.772F * ccb)).Clamp(0, 255);
+
+            packed.PackFromBytes(r, g, b, 255);
+        }
+
         /// <summary>
         /// Represents a component scan
         /// </summary>
diff --git a/src/ImageSharp/Formats/Jpg/JpegEncoderCore.cs b/src/ImageSharp/Formats/Jpg/JpegEncoderCore.cs
index ecb11545b0..e1eab40151 100644
--- a/src/ImageSharp/Formats/Jpg/JpegEncoderCore.cs
+++ b/src/ImageSharp/Formats/Jpg/JpegEncoderCore.cs
@@ -538,17 +538,26 @@ namespace ImageSharp.Formats
         {
             int xmax = pixels.Width - 1;
             int ymax = pixels.Height - 1;
-            byte[] b = new byte[3];
+            byte[] color = new byte[3];
             for (int j = 0; j < 8; j++)
             {
                 for (int i = 0; i < 8; i++)
                 {
-                    pixels[Math.Min(x + i, xmax), Math.Min(y + j, ymax)].ToBytes(b, 0, ComponentOrder.XYZ);
-                    YCbCr color = new Color(b[0], b[1], b[2]);
+                    pixels[Math.Min(x + i, xmax), Math.Min(y + j, ymax)].ToBytes(color, 0, ComponentOrder.XYZ);
+
+                    byte r = color[0];
+                    byte g = color[1];
+                    byte b = color[2];
+
+                    // Convert returned bytes into the YCbCr color space. Assume RGBA 
+                    byte yy = (byte)((0.299F * r) + (0.587F * g) + (0.114F * b));
+                    byte cb = (byte)(128 + ((-0.168736F * r) - (0.331264F * g) + (0.5F * b)));
+                    byte cr = (byte)(128 + ((0.5F * r) - (0.418688F * g) - (0.081312F * b)));
+                    
                     int index = (8 * j) + i;
-                    yBlock[index] = color.Y;
-                    cbBlock[index] = color.Cb;
-                    crBlock[index] = color.Cr;
+                    yBlock[index] = yy;
+                    cbBlock[index] = cb;
+                    crBlock[index] = cr;
                 }
             }
         }