diff --git a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
index 6fbbc312f9..fa1b76e696 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegDecoderCore.cs
@@ -607,10 +607,16 @@ namespace ImageSharp.Formats
             int ccb = cb - 128;
             int ccr = cr - 128;
 
-            byte r = (byte)(y + (1.402F * ccr)).Clamp(0, 255);
-            byte g = (byte)(y - (0.34414F * ccb) - (0.71414F * ccr)).Clamp(0, 255);
-            byte b = (byte)(y + (1.772F * ccb)).Clamp(0, 255);
-
+            // Speed up the algorithm by removing floating point calculation
+            // Scale by 1024, add .5F and truncate value. We use bit shifting to divide the result
+            int r0 = 1436 * ccr; // (1.402F * 1024) + .5F
+            int g0 = 352 * ccb; // (0.34414F * 1024) + .5F
+            int g1 = 731 * ccr; // (0.71414F  * 1024) + .5F
+            int b0 = 1815 * ccb; // (1.772F * 1024) + .5F
+
+            byte r = (byte)(y + (r0 >> 10)).Clamp(0, 255);
+            byte g = (byte)(y - (g0 >> 10) - (g1 >> 10)).Clamp(0, 255);
+            byte b = (byte)(y + (b0 >> 10)).Clamp(0, 255);
             packed.PackFromBytes(r, g, b, 255);
         }
 
diff --git a/src/ImageSharp.Formats.Jpeg/JpegEncoderCore.cs b/src/ImageSharp.Formats.Jpeg/JpegEncoderCore.cs
index 59dc5ce39c..92bb79924d 100644
--- a/src/ImageSharp.Formats.Jpeg/JpegEncoderCore.cs
+++ b/src/ImageSharp.Formats.Jpeg/JpegEncoderCore.cs
@@ -304,12 +304,28 @@ namespace ImageSharp.Formats
                 int j8 = j * 8;
                 for (int i = 0; i < 8; i++)
                 {
-                    Vector3 v = new Vector3(data[0], data[1], data[2]);
-
                     // Convert returned bytes into the YCbCr color space. Assume RGBA
-                    float yy = (0.299F * v.X) + (0.587F * v.Y) + (0.114F * v.Z);
-                    float cb = 128 + ((-0.168736F * v.X) - (0.331264F * v.Y) + (0.5F * v.Z));
-                    float cr = 128 + ((0.5F * v.X) - (0.418688F * v.Y) - (0.081312F * v.Z));
+                    int r = data[0];
+                    int g = data[1];
+                    int b = data[2];
+
+                    // Speed up the algorithm by removing floating point calculation
+                    // Scale by 1024, add .5F and truncate value. We use bit shifting to divide the result
+                    int y0 = 306 * r; // (0.299F * 1024) + .5F
+                    int y1 = 601 * g; // (0.587F * 1024) + .5F
+                    int y2 = 117 * b; // (0.114F * 1024) + .5F
+
+                    int cb0 = -172 * r; // (-0.168736F * 1024) + .5F
+                    int cb1 = 339 * g; // (0.331264F * 1024) + .5F
+                    int cb2 = 512 * b; // (0.5F * 1024) + .5F
+
+                    int cr0 = 512 * r; // (0.5F * 1024) + .5F
+                    int cr1 = 429 * g; // (0.418688F * 1024) + .5F
+                    int cr2 = 83 * b; // (0.081312F * 1024) + .5F
+
+                    float yy = (y0 + y1 + y2) >> 10;
+                    float cb = 128 + ((cb0 - cb1 + cb2) >> 10);
+                    float cr = 128 + ((cr0 - cr1 - cr2) >> 10);
 
                     int index = j8 + i;
 
diff --git a/tests/ImageSharp.Benchmarks/Color/YcbCrToRgb.cs b/tests/ImageSharp.Benchmarks/Color/YcbCrToRgb.cs
new file mode 100644
index 0000000000..dad32626d3
--- /dev/null
+++ b/tests/ImageSharp.Benchmarks/Color/YcbCrToRgb.cs
@@ -0,0 +1,49 @@
+namespace ImageSharp.Benchmarks
+{
+    using System.Numerics;
+
+    using BenchmarkDotNet.Attributes;
+
+    public class YcbCrToRgb
+    {
+        [Benchmark(Baseline = true, Description = "Floating Point Conversion")]
+        public Vector3 YcbCrToRgba()
+        {
+            int y = 255;
+            int cb = 128;
+            int cr = 128;
+
+            int ccb = cb - 128;
+            int ccr = cr - 128;
+
+            byte r = (byte)(y + (1.402F * ccr)).Clamp(0, 255);
+            byte g = (byte)(y - (0.34414F * ccb) - (0.71414F * ccr)).Clamp(0, 255);
+            byte b = (byte)(y + (1.772F * ccb)).Clamp(0, 255);
+
+            return new Vector3(r, g, b);
+        }
+
+        [Benchmark(Description = "Scaled Integer Conversion")]
+        public Vector3 YcbCrToRgbaScaled()
+        {
+            int y = 255;
+            int cb = 128;
+            int cr = 128;
+
+            int ccb = cb - 128;
+            int ccr = cr - 128;
+
+            // Scale by 1024, add .5F and truncate value
+            int r0 = 1436 * ccr; // (1.402F * 1024) + .5F
+            int g0 = 352 * ccb; // (0.34414F * 1024) + .5F
+            int g1 = 731 * ccr; // (0.71414F  * 1024) + .5F
+            int b0 = 1815 * ccb; // (1.772F * 1024) + .5F
+
+            byte r = (byte)(y + (r0 >> 10)).Clamp(0, 255);
+            byte g = (byte)(y - (g0 >> 10) - (g1 >> 10)).Clamp(0, 255);
+            byte b = (byte)(y + (b0 >> 10)).Clamp(0, 255);
+
+            return new Vector3(r, g, b);
+        }
+    }
+}