diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.cs
index 11ea4cda8..dad46861e 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/ColorConverters/JpegColorConverter.cs
@@ -197,6 +197,23 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder.ColorConverters
             /// </summary>
             public readonly Span<float> Component3;
 
+            /// <summary>
+            /// Initializes a new instance of the <see cref="ComponentValues"/> struct.
+            /// </summary>
+            /// <param name="componentProcessors">The 1-4 sized list of component post processors.</param>
+            /// <param name="row">The row to convert</param>
+            public ComponentValues(IReadOnlyList<JpegComponentPostProcessor> componentProcessors, int row)
+            {
+                this.ComponentCount = componentProcessors.Count;
+
+                this.Component0 = componentProcessors[0].GetColorBufferRowSpan(row);
+
+                // In case of grayscale, Component1 and Component2 point to Component0 memory area
+                this.Component1 = this.ComponentCount > 1 ? componentProcessors[1].GetColorBufferRowSpan(row) : this.Component0;
+                this.Component2 = this.ComponentCount > 2 ? componentProcessors[2].GetColorBufferRowSpan(row) : this.Component0;
+                this.Component3 = this.ComponentCount > 3 ? componentProcessors[3].GetColorBufferRowSpan(row) : Span<float>.Empty;
+            }
+
             /// <summary>
             /// Initializes a new instance of the <see cref="ComponentValues"/> struct.
             /// </summary>
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
index b87d74720..3e04e80b7 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/JpegComponentPostProcessor.cs
@@ -122,5 +122,8 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
                 spectralBlocks.GetRowSpan(i).Clear();
             }
         }
+
+        public Span<float> GetColorBufferRowSpan(int row) =>
+            this.ColorBuffer.GetRowSpan(row);
     }
 }
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
index 2e965e0ac..722570919 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs
@@ -4,7 +4,6 @@
 using System;
 using System.Buffers;
 using System.Linq;
-using System.Numerics;
 using System.Threading;
 using SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder.ColorConverters;
 using SixLabors.ImageSharp.Memory;
@@ -23,7 +22,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
 
         private JpegColorConverter colorConverter;
 
-        // private IMemoryOwner<Vector4> rgbaBuffer;
         private IMemoryOwner<byte> rgbBuffer;
 
         private IMemoryOwner<TPixel> paddedProxyPixelRow;
@@ -121,11 +119,9 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
         {
             int maxY = Math.Min(this.pixelBuffer.Height, this.pixelRowCounter + this.pixelRowsPerStep);
 
-            var buffers = new Buffer2D<float>[this.componentProcessors.Length];
             for (int i = 0; i < this.componentProcessors.Length; i++)
             {
                 this.componentProcessors[i].CopyBlocksToColorBuffer(spectralStep);
-                buffers[i] = this.componentProcessors[i].ColorBuffer;
             }
 
             int width = this.pixelBuffer.Width;
@@ -134,7 +130,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder
             {
                 int y = yy - this.pixelRowCounter;
 
-                var values = new JpegColorConverter.ComponentValues(buffers, y);
+                var values = new JpegColorConverter.ComponentValues(this.componentProcessors, y);
 
                 this.colorConverter.ConvertToRgbInplace(values);
                 values = values.Slice(0, width); // slice away Jpeg padding
diff --git a/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs b/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs
index e6e82b981..079d24c47 100644
--- a/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs
+++ b/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs
@@ -47,12 +47,6 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             benchmarks.EncodeJpeg_SingleMidSize();
         }
 
-        private static void RunJpegColorProfilingTests()
-        {
-            new JpegColorConverterTests(new ConsoleOutput()).BenchmarkYCbCr(false);
-            new JpegColorConverterTests(new ConsoleOutput()).BenchmarkYCbCr(true);
-        }
-
         private static void RunResizeProfilingTest()
         {
             var test = new ResizeProfilingBenchmarks(new ConsoleOutput());