diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/Component.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/Component.cs
index 59c5c69c80..cf02865241 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/Component.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/Component.cs
@@ -84,11 +84,14 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
         /// <param name="maxSubFactorV">Maximal vertical subsampling factor among all the components.</param>
         public void Init(JpegFrame frame, int maxSubFactorH, int maxSubFactorV)
         {
+            uint widthInBlocks = ((uint)frame.PixelWidth + 7) / 8;
+            uint heightInBlocks = ((uint)frame.PixelHeight + 7) / 8;
+
             this.WidthInBlocks = (int)MathF.Ceiling(
-                ((uint)frame.PixelWidth + 7) / 8 * this.HorizontalSamplingFactor / maxSubFactorH);
+                (float)widthInBlocks * this.HorizontalSamplingFactor / maxSubFactorH);
 
             this.HeightInBlocks = (int)MathF.Ceiling(
-                ((uint)frame.PixelHeight + 7) / 8 * this.VerticalSamplingFactor / maxSubFactorV);
+                (float)heightInBlocks * this.VerticalSamplingFactor / maxSubFactorV);
 
             int blocksPerLineForMcu = frame.McusPerLine * this.HorizontalSamplingFactor;
             int blocksPerColumnForMcu = frame.McusPerColumn * this.VerticalSamplingFactor;
diff --git a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs
index 8c9f826a10..9795ab9534 100644
--- a/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Components/Encoder/ComponentProcessor.cs
@@ -157,18 +157,24 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
 
             static void SumHorizontal(Span<float> target, int factor)
             {
+                Span<float> source = target;
                 if (Avx2.IsSupported)
                 {
                     ref Vector256<float> targetRef = ref Unsafe.As<float, Vector256<float>>(ref MemoryMarshal.GetReference(target));
 
                     // Ideally we need to use log2: Numerics.Log2((uint)factor)
                     // but division by 2 works just fine in this case
-                    // because factor value range is [1, 2, 4]
-                    // log2(1) == 1 / 2 == 0
-                    // log2(2) == 2 / 2 == 1
-                    // log2(4) == 4 / 2 == 2
                     int haddIterationsCount = (int)((uint)factor / 2);
-                    uint length = (uint)target.Length / (uint)Vector256<float>.Count;
+
+                    // Transform spans so that it only contains 'remainder'
+                    // values for the scalar fallback code
+                    int scalarRemainder = target.Length % (Vector<float>.Count * factor);
+                    int touchedCount = target.Length - scalarRemainder;
+                    source = source.Slice(touchedCount);
+                    target = target.Slice(touchedCount / factor);
+
+                    uint length = (uint)touchedCount / (uint)Vector256<float>.Count;
+
                     for (int i = 0; i < haddIterationsCount; i++)
                     {
                         length /= 2;
@@ -181,18 +187,15 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Encoder
                             Unsafe.Add(ref targetRef, j) = Avx2.Permute4x64(sum.AsDouble(), 0b11_01_10_00).AsSingle();
                         }
                     }
-
-                    int summedCount = (int)(length * factor * Vector256<float>.Count);
-                    target = target.Slice(summedCount);
                 }
 
                 // scalar remainder
-                for (int i = 0; i < target.Length / factor; i++)
+                for (int i = 0; i < source.Length / factor; i++)
                 {
-                    target[i] = target[i * factor];
+                    target[i] = source[i * factor];
                     for (int j = 1; j < factor; j++)
                     {
-                        target[i] += target[(i * factor) + j];
+                        target[i] += source[(i * factor) + j];
                     }
                 }
             }
diff --git a/tests/ImageSharp.Tests/Formats/Jpg/JpegEncoderTests.cs b/tests/ImageSharp.Tests/Formats/Jpg/JpegEncoderTests.cs
index 5e38f4455c..52bddd27f1 100644
--- a/tests/ImageSharp.Tests/Formats/Jpg/JpegEncoderTests.cs
+++ b/tests/ImageSharp.Tests/Formats/Jpg/JpegEncoderTests.cs
@@ -143,7 +143,6 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg
         [WithSolidFilledImages(nameof(BitsPerPixel_Quality), 1, 1, 100, 100, 100, 255, PixelTypes.L8)]
         [WithSolidFilledImages(nameof(BitsPerPixel_Quality), 1, 1, 255, 100, 50, 255, PixelTypes.Rgba32)]
         [WithTestPatternImages(nameof(BitsPerPixel_Quality), 143, 81, PixelTypes.Rgba32)]
-        [WithTestPatternImages(nameof(BitsPerPixel_Quality), 7, 5, PixelTypes.Rgba32)]
         [WithTestPatternImages(nameof(BitsPerPixel_Quality), 96, 48, PixelTypes.Rgba32)]
         [WithTestPatternImages(nameof(BitsPerPixel_Quality), 73, 71, PixelTypes.Rgba32)]
         [WithTestPatternImages(nameof(BitsPerPixel_Quality), 48, 24, PixelTypes.Rgba32)]