diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.FromGrayScale.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.FromGrayScale.cs
index 5d7a31a12b..7424145c3b 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.FromGrayScale.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.FromGrayScale.cs
@@ -3,6 +3,8 @@
 
 using System;
 using System.Numerics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
 
 namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder.ColorConverters
 {
@@ -17,24 +19,17 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder.ColorConverters
 
             public override void ConvertToRgba(in ComponentValues values, Span<Vector4> result)
             {
-                // TODO: We can optimize a lot here with Vector<float> and SRCS.Unsafe()!
-                ReadOnlySpan<float> yVals = values.Component0;
-
-                var v = new Vector4(0, 0, 0, 1);
-
                 var scale = new Vector4(1 / 255F, 1 / 255F, 1 / 255F, 1F);
 
+                ref float sBase = ref MemoryMarshal.GetReference(values.Component0);
+                ref Vector4 dBase = ref MemoryMarshal.GetReference(result);
+
                 for (int i = 0; i < result.Length; i++)
                 {
-                    float y = yVals[i];
-
-                    v.X = y;
-                    v.Y = y;
-                    v.Z = y;
-
+                    var v = new Vector4(Unsafe.Add(ref sBase, i));
+                    v.W = 1f;
                     v *= scale;
-
-                    result[i] = v;
+                    Unsafe.Add(ref dBase, i) = v;
                 }
             }
         }