diff --git a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeProcessor.cs b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeProcessor.cs
index 707be1ec9..b4b537e20 100644
--- a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeProcessor.cs
+++ b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeProcessor.cs
@@ -235,27 +235,27 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
             PixelConversionModifiers conversionModifiers =
                 PixelConversionModifiers.Premultiply.ApplyCompanding(this.Compand);
 
+            BufferArea<TPixel> sourceArea = source.PixelBuffer.GetArea(sourceRectangle);
+
             // Interpolate the image using the calculated weights.
             // A 2-pass 1D algorithm appears to be faster than splitting a 1-pass 2D algorithm
             // First process the columns. Since we are not using multiple threads startY and endY
             // are the upper and lower bounds of the source rectangle.
-            using (var resizeWindow = new ResizeWindow(
+            using (var resizeWindow = new ResizeWindow<TPixel>(
                 configuration,
-                sourceRectangle,
+                sourceArea,
                 conversionModifiers,
                 this.horizontalKernelMap,
                 this.verticalKernelMap,
                 width,
                 workingRect,
                 startX))
-            using (IMemoryOwner<Vector4> tempBuffer = source.MemoryAllocator.Allocate<Vector4>(Math.Max(sourceRectangle.Width, width)))
+            using (IMemoryOwner<Vector4> tempBuffer = source.MemoryAllocator.Allocate<Vector4>(width))
             {
-                Span<Vector4> tempRowSpan = tempBuffer.GetSpan();
-
-                resizeWindow.Initialize(source.PixelBuffer, tempRowSpan);
+                resizeWindow.Initialize();
 
                 // Now process the rows.
-                Span<Vector4> tempColSpan = tempBuffer.GetSpan().Slice(0, width);
+                Span<Vector4> tempColSpan = tempBuffer.GetSpan();
 
                 for (int y = workingRect.Top; y < workingRect.Bottom; y++)
                 {
diff --git a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeWindow.cs b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeWindow.cs
index 130dd93a4..bd68fdba7 100644
--- a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeWindow.cs
+++ b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeWindow.cs
@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0.
 
 using System;
+using System.Buffers;
 using System.Numerics;
 using System.Runtime.CompilerServices;
 
@@ -12,27 +13,32 @@ using SixLabors.Primitives;
 
 namespace SixLabors.ImageSharp.Processing.Processors.Transforms
 {
-    class ResizeWindow : IDisposable
+    class ResizeWindow<TPixel> : IDisposable
+        where TPixel : struct, IPixel<TPixel>
     {
         private readonly Buffer2D<Vector4> buffer;
 
         private readonly Configuration configuration;
 
-        private readonly Rectangle sourceRectangle;
-
         private readonly PixelConversionModifiers conversionModifiers;
 
         private readonly ResizeKernelMap horizontalKernelMap;
 
-        private readonly ResizeKernelMap verticalKernelMap;
+        private readonly BufferArea<TPixel> source;
 
-        private readonly Rectangle workingRectangle;
+        private readonly Rectangle sourceRectangle;
 
         private readonly int startX;
 
+        private readonly IMemoryOwner<Vector4> tempRowBuffer;
+
+        private readonly ResizeKernelMap verticalKernelMap;
+
+        private readonly Rectangle workingRectangle;
+
         public ResizeWindow(
             Configuration configuration,
-            Rectangle sourceRectangle,
+            BufferArea<TPixel> source,
             PixelConversionModifiers conversionModifiers,
             ResizeKernelMap horizontalKernelMap,
             ResizeKernelMap verticalKernelMap,
@@ -41,31 +47,45 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
             int startX)
         {
             this.configuration = configuration;
-            this.sourceRectangle = sourceRectangle;
+            this.source = source;
+            this.sourceRectangle = source.Rectangle;
             this.conversionModifiers = conversionModifiers;
             this.horizontalKernelMap = horizontalKernelMap;
             this.verticalKernelMap = verticalKernelMap;
             this.workingRectangle = workingRectangle;
             this.startX = startX;
-            this.buffer = configuration.MemoryAllocator.Allocate2D<Vector4>(sourceRectangle.Height, destWidth, AllocationOptions.Clean);
+            this.buffer = configuration.MemoryAllocator.Allocate2D<Vector4>(
+                this.sourceRectangle.Height,
+                destWidth,
+                AllocationOptions.Clean);
+            this.tempRowBuffer = configuration.MemoryAllocator.Allocate<Vector4>(this.sourceRectangle.Width);
 
-            this.Top = sourceRectangle.Top;
+            this.Top = this.sourceRectangle.Top;
 
-            this.Bottom = sourceRectangle.Bottom;
+            this.Bottom = this.sourceRectangle.Bottom;
         }
 
+        public int Bottom { get; private set; }
+
         public int Top { get; private set; }
 
-        public int Bottom { get; private set; }
+        public void Dispose()
+        {
+            this.buffer.Dispose();
+        }
 
-        public void Initialize<TPixel>(
-            Buffer2D<TPixel> source,
-            Span<Vector4> tempRowSpan)
-            where TPixel : struct, IPixel<TPixel>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public Span<Vector4> GetColumnSpan(int x)
         {
-            for (int y = this.sourceRectangle.Top; y < this.sourceRectangle.Bottom; y++)
+            return this.buffer.GetRowSpan(x);
+        }
+
+        public void Initialize()
+        {
+            Span<Vector4> tempRowSpan = this.tempRowBuffer.GetSpan();
+            for (int y = 0; y < this.sourceRectangle.Height; y++)
             {
-                Span<TPixel> sourceRow = source.GetRowSpan(y).Slice(this.sourceRectangle.X);
+                Span<TPixel> sourceRow = this.source.GetRowSpan(y);
 
                 PixelOperations<TPixel>.Instance.ToVector4(
                     this.configuration,
@@ -73,7 +93,7 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
                     tempRowSpan,
                     this.conversionModifiers);
 
-                ref Vector4 firstPassBaseRef = ref this.buffer.Span[y - this.sourceRectangle.Y];
+                ref Vector4 firstPassBaseRef = ref this.buffer.Span[y];
 
                 for (int x = this.workingRectangle.Left; x < this.workingRectangle.Right; x++)
                 {
@@ -82,16 +102,5 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms
                 }
             }
         }
-
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public Span<Vector4> GetColumnSpan(int x)
-        {
-            return this.buffer.GetRowSpan(x);
-        }
-
-        public void Dispose()
-        {
-            this.buffer.Dispose();
-        }
     }
 }
\ No newline at end of file