diff --git a/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs b/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
index 7a3adfb53..86f3f0015 100644
--- a/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
+++ b/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
@@ -1,14 +1,17 @@
 using System.Buffers;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 namespace SixLabors.ImageSharp.Memory
 {
+
     /// <summary>
     /// Implements <see cref="MemoryManager"/> by allocating memory from <see cref="ArrayPool{T}"/>.
     /// </summary>
     public class ArrayPoolMemoryManager : MemoryManager
     {
         private readonly int minSizeBytes;
+        private readonly ArrayPool<byte> pool;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="ArrayPoolMemoryManager"/> class.
@@ -22,17 +25,21 @@ namespace SixLabors.ImageSharp.Memory
         public ArrayPoolMemoryManager(int minSizeBytes = 0)
         {
             this.minSizeBytes = minSizeBytes;
+
+            this.pool = ArrayPool<byte>.Create(CalculateMaxArrayLength(), 50);
         }
 
         /// <inheritdoc />
         internal override Buffer<T> Allocate<T>(int size, bool clear = false)
         {
-            if (this.minSizeBytes > 0 && size < this.minSizeBytes * SizeHelper<T>.Size)
+            int itemSize = Unsafe.SizeOf<T>();
+            if (this.minSizeBytes > 0 && itemSize < this.minSizeBytes * itemSize)
             {
-                return new Buffer<T>(new T[size], size);
+                return new Buffer<T>(new T[itemSize], itemSize);
             }
 
-            var buffer = new Buffer<T>(PixelDataPool<T>.Rent(size), size, this);
+            byte[] byteBuffer = this.pool.Rent(itemSize * itemSize);
+            var buffer = new Buffer<T>(Unsafe.As<T[]>(byteBuffer), itemSize, this);
             if (clear)
             {
                 buffer.Clear();
@@ -44,21 +51,19 @@ namespace SixLabors.ImageSharp.Memory
         /// <inheritdoc />
         internal override void Release<T>(Buffer<T> buffer)
         {
-            PixelDataPool<T>.Return(buffer.Array);
+            var byteBuffer = Unsafe.As<byte[]>(buffer.Array);
+            this.pool.Return(byteBuffer);
         }
 
-        internal static class SizeHelper<T>
+        /// <summary>
+        /// Heuristically calculates a reasonable maxArrayLength value for the backing <see cref="ArrayPool{T}"/>.
+        /// </summary>
+        /// <returns>The maxArrayLength value</returns>
+        internal static int CalculateMaxArrayLength()
         {
-            static SizeHelper()
-            {
-                #if NETSTANDARD1_1
-                Size = Marshal.SizeOf(typeof(T));
-                #else
-                Size = Marshal.SizeOf<T>();
-                #endif
-            }
-
-            public static int Size { get; }
+            const int MaximumExpectedImageSize = 16384 * 16384;
+            const int MaximumBytesPerPixel = 4;
+            return MaximumExpectedImageSize * MaximumBytesPerPixel;
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Memory/Buffer{T}.cs b/src/ImageSharp/Memory/Buffer{T}.cs
index 16f4e8599..d25cc232a 100644
--- a/src/ImageSharp/Memory/Buffer{T}.cs
+++ b/src/ImageSharp/Memory/Buffer{T}.cs
@@ -171,7 +171,7 @@ namespace SixLabors.ImageSharp.Memory
 
         /// <summary>
         /// Unpins <see cref="Array"/> and makes the object "quasi-disposed" so the array is no longer owned by this object.
-        /// If <see cref="Array"/> is rented, it's the callers responsibility to return it to it's pool. (Most likely <see cref="PixelDataPool{T}"/>)
+        /// If <see cref="Array"/> is rented, it's the callers responsibility to return it to it's pool.
         /// </summary>
         /// <returns>The unpinned <see cref="Array"/></returns>
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
diff --git a/tests/ImageSharp.Tests/Memory/PixelDataPoolTests.cs b/tests/ImageSharp.Tests/Memory/PixelDataPoolTests.cs
deleted file mode 100644
index fdfd4c4b7..000000000
--- a/tests/ImageSharp.Tests/Memory/PixelDataPoolTests.cs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copyright (c) Six Labors and contributors.
-// Licensed under the Apache License, Version 2.0.
-
-
-
-// ReSharper disable InconsistentNaming
-namespace SixLabors.ImageSharp.Tests.Memory
-{
-    using SixLabors.ImageSharp.Memory;
-
-    using Xunit;
-
-    /// <summary>
-    /// Tests the <see cref="PixelDataPool{T}"/> class.
-    /// </summary>
-    public class PixelDataPoolTests
-    {
-        [Fact]
-        public void PixelDataPoolRentsMinimumSize()
-        {
-            Rgba32[] pixels = PixelDataPool<Rgba32>.Rent(1024);
-
-            Assert.True(pixels.Length >= 1024);
-        }
-
-        [Fact]
-        public void PixelDataPoolDoesNotThrowWhenReturningNonPooled()
-        {
-            Rgba32[] pixels = new Rgba32[1024];
-
-            PixelDataPool<Rgba32>.Return(pixels);
-
-            Assert.True(pixels.Length >= 1024);
-        }
-
-        [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public void CalculateMaxArrayLength(bool isRawData)
-        {
-            int max = isRawData ? PixelDataPool<int>.CalculateMaxArrayLength()
-                          : PixelDataPool<Rgba32>.CalculateMaxArrayLength();
-
-            Assert.Equal(max > 1024 * 1024, !isRawData);
-        }
-
-        [Fact]
-        public void RentNonIPixelData()
-        {
-            byte[] data = PixelDataPool<byte>.Rent(16384);
-
-            Assert.True(data.Length >= 16384);
-        }
-    }
-}
\ No newline at end of file