diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
index 3231c5781e..283a98fab6 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/HuffmanScanEncoder.cs
@@ -125,14 +125,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         {
             var unzig = ZigZag.CreateUnzigTable();
 
-            var pixelConverter = YCbCrForwardConverter420<TPixel>.Create();
-
             // ReSharper disable once InconsistentNaming
             int prevDCY = 0, prevDCCb = 0, prevDCCr = 0;
             ImageFrame<TPixel> frame = pixels.Frames.RootFrame;
             Buffer2D<TPixel> pixelBuffer = frame.PixelBuffer;
             RowOctet<TPixel> currentRows = default;
 
+            var pixelConverter = new YCbCrForwardConverter420<TPixel>(frame);
+
             for (int y = 0; y < pixels.Height; y += 16)
             {
                 cancellationToken.ThrowIfCancellationRequested();
@@ -142,7 +142,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
                     {
                         int yOff = i * 8;
                         currentRows.Update(pixelBuffer, y + yOff);
-                        pixelConverter.Convert(frame, x, y, ref currentRows, i);
+                        pixelConverter.Convert(x, y, ref currentRows, i);
 
                         prevDCY = this.WriteBlock(
                             QuantIndex.Luminance,
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrConverterVectorized.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrConverterVectorized.cs
index cf4d477749..b9f0fa4271 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrConverterVectorized.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/RgbToYCbCrConverterVectorized.cs
@@ -27,9 +27,20 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
             }
         }
 
+        public static int AvxRegisterRgbCompatibilityPadding
+        {
+            get
+            {
+                if (IsSupported)
+                {
+                    return 8;
+                }
+
+                return 0;
+            }
+        }
+
 #if SUPPORTS_RUNTIME_INTRINSICS
-        // TODO: documentation
-        public const int AvxRegisterRgbCompatibilityOffset = 8;
 
         private static ReadOnlySpan<byte> MoveFirst24BytesToSeparateLanes => new byte[]
         {
@@ -306,7 +317,6 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
 #endif
         }
 
-
 #if SUPPORTS_RUNTIME_INTRINSICS
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public static Vector256<float> Scale_8x4_4x2(Span<Vector256<float>> v)
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/YCbCrForwardConverter420{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/YCbCrForwardConverter420{TPixel}.cs
index c831b611c8..fdb41a8e20 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/YCbCrForwardConverter420{TPixel}.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/YCbCrForwardConverter420{TPixel}.cs
@@ -16,6 +16,15 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
     internal ref struct YCbCrForwardConverter420<TPixel>
         where TPixel : unmanaged, IPixel<TPixel>
     {
+        // TODO: docs
+        private const int PixelsPerSample = 16 * 8;
+
+        // TODO: docs
+        private static int RgbSpanByteSize = PixelsPerSample * 3;
+
+        // TODO: docs
+        private static readonly Size SampleSize = new Size(16, 8);
+
         /// <summary>
         /// The left Y component
         /// </summary>
@@ -51,35 +60,45 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         /// </summary>
         private Span<Rgb24> rgbSpan;
 
-        public static YCbCrForwardConverter420<TPixel> Create()
+        // TODO: docs
+        private Size samplingAreaSize;
+
+        // TODO: docs
+        private Configuration config;
+
+
+        public YCbCrForwardConverter420(ImageFrame<TPixel> frame)
         {
-            var result = default(YCbCrForwardConverter420<TPixel>);
+            // matrices would be filled during convert calls
+            this.YLeft = default;
+            this.YRight = default;
+            this.Cb = default;
+            this.Cr = default;
 
-            // TODO: this is subject to discuss
-            const int twoBlocksByteSizeWithPadding = 384 + 8; // converter.Convert comments for +8 padding
-            result.rgbSpan = MemoryMarshal.Cast<byte, Rgb24>(new byte[twoBlocksByteSizeWithPadding].AsSpan());
+            // temporal pixel buffers
+            this.pixelSpan = new TPixel[PixelsPerSample].AsSpan();
+            this.rgbSpan = MemoryMarshal.Cast<byte, Rgb24>(new byte[RgbSpanByteSize + RgbToYCbCrConverterVectorized.AvxRegisterRgbCompatibilityPadding].AsSpan());
 
-            // TODO: this size should be configurable
-            result.pixelSpan = new TPixel[128].AsSpan();
+            // frame data
+            this.samplingAreaSize = new Size(frame.Width, frame.Height);
+            this.config = frame.GetConfiguration();
 
-            // Avoid creating lookup tables, when vectorized converter is supported
+            // conversion vector fallback data
             if (!RgbToYCbCrConverterVectorized.IsSupported)
             {
-                result.colorTables = RgbToYCbCrConverterLut.Create();
+                this.colorTables = RgbToYCbCrConverterLut.Create();
+            }
+            else
+            {
+                this.colorTables = default;
             }
-
-            return result;
         }
 
-        /// <summary>
-        /// Converts a 8x8 image area inside 'pixels' at position (x,y) placing the result members of the structure (<see cref="Y"/>, <see cref="Cb"/>, <see cref="Cr"/>)
-        /// </summary>
-        public void Convert(ImageFrame<TPixel> frame, int x, int y, ref RowOctet<TPixel> currentRows, int idx)
+        public void Convert(int x, int y, ref RowOctet<TPixel> currentRows, int idx)
         {
-            Memory.Buffer2D<TPixel> buffer = frame.PixelBuffer;
-            YCbCrForwardConverter<TPixel>.LoadAndStretchEdges(currentRows, this.pixelSpan, new Point(x, y), new Size(16, 8), new Size(buffer.Width, buffer.Height));
+            YCbCrForwardConverter<TPixel>.LoadAndStretchEdges(currentRows, this.pixelSpan, new Point(x, y), SampleSize, this.samplingAreaSize);
 
-            PixelOperations<TPixel>.Instance.ToRgb24(frame.GetConfiguration(), this.pixelSpan, this.rgbSpan);
+            PixelOperations<TPixel>.Instance.ToRgb24(this.config, this.pixelSpan, this.rgbSpan);
 
             if (RgbToYCbCrConverterVectorized.IsSupported)
             {