Use 2 ArrayPool-s with different maxArrayPerBuckets parameters,

so we never have more than 8 buckets for large buffers for	 a single value type.

src/ImageSharp/Memory/PixelDataPool{T}.cs

@@ -17,17 +17,32 @@ namespace SixLabors.ImageSharp.Memory
         /// The maximum size of pooled arrays in bytes.
         /// Currently set to 32MB, which is equivalent to 8 megapixels of raw <see cref="Rgba32"/> data.
         /// </summary>
-        private const int MaxPooledBufferSizeInBytes = 32 * 1024 * 1024;
+        internal const int MaxPooledBufferSizeInBytes = 32 * 1024 * 1024;
 
         /// <summary>
-        /// The maximum array length of the <see cref="ArrayPool"/>.
+        /// The threshold to pool arrays in <see cref="LargeArrayPool"/> which has less buckets for memory safety.
         /// </summary>
-        private static readonly int MaxArrayLength = MaxPooledBufferSizeInBytes / Unsafe.SizeOf<T>();
+        private const int LargeBufferThresholdInBytes = 8 * 1024 * 1024;
 
         /// <summary>
-        /// The <see cref="ArrayPool{T}"/> which is not kept clean.
+        /// The maximum array length of the <see cref="LargeArrayPool"/>.
         /// </summary>
-        private static readonly ArrayPool<T> ArrayPool = ArrayPool<T>.Create(MaxArrayLength, 50);
+        private static readonly int MaxLargeArrayLength = MaxPooledBufferSizeInBytes / Unsafe.SizeOf<T>();
+
+        /// <summary>
+        /// The maximum array length of the <see cref="NormalArrayPool"/>.
+        /// </summary>
+        private static readonly int MaxNormalArrayLength = LargeBufferThresholdInBytes / Unsafe.SizeOf<T>();
+
+        /// <summary>
+        /// The <see cref="ArrayPool{T}"/> for huge buffers, which is not kept clean.
+        /// </summary>
+        private static readonly ArrayPool<T> LargeArrayPool = ArrayPool<T>.Create(MaxLargeArrayLength, 8);
+
+        /// <summary>
+        /// The <see cref="ArrayPool{T}"/> for small-to-medium buffers which is not kept clean.
+        /// </summary>
+        private static readonly ArrayPool<T> NormalArrayPool = ArrayPool<T>.Create(MaxNormalArrayLength, 24);
 
         /// <summary>
         /// Rents the pixel array from the pool.
@@ -36,7 +51,14 @@ namespace SixLabors.ImageSharp.Memory
         /// <returns>The <see cref="T:TPixel[]"/></returns>
         public static T[] Rent(int minimumLength)
         {
-            return ArrayPool.Rent(minimumLength);
+            if (minimumLength <= MaxNormalArrayLength)
+            {
+                return NormalArrayPool.Rent(minimumLength);
+            }
+            else
+            {
+                return LargeArrayPool.Rent(minimumLength);
+            }
         }
 
         /// <summary>
@@ -45,7 +67,14 @@ namespace SixLabors.ImageSharp.Memory
         /// <param name="array">The array to return to the buffer pool.</param>
         public static void Return(T[] array)
         {
-            ArrayPool.Return(array);
+            if (array.Length <= MaxNormalArrayLength)
+            {
+                NormalArrayPool.Return(array);
+            }
+            else
+            {
+                LargeArrayPool.Return(array);
+            }
         }
     }
 }

tests/ImageSharp.Tests/Memory/PixelDataPoolTests.cs

@@ -9,19 +9,25 @@ using Xunit;
 // ReSharper disable InconsistentNaming
 namespace SixLabors.ImageSharp.Tests.Memory
 {
+    using System;
+
     /// <summary>
     /// Tests the <see cref="PixelDataPool{T}"/> class.
     /// </summary>
     public class PixelDataPoolTests
     {
+        private const int MaxPooledBufferSizeInBytes = PixelDataPool<byte>.MaxPooledBufferSizeInBytes;
+
         readonly object monitor = new object();
         
-        [Fact]
-        public void PixelDataPoolRentsMinimumSize()
+        [Theory]
+        [InlineData(1)]
+        [InlineData(1024)]
+        public void PixelDataPoolRentsMinimumSize(int size)
         {
-            Rgba32[] pixels = PixelDataPool<Rgba32>.Rent(1024);
+            Rgba32[] pixels = PixelDataPool<Rgba32>.Rent(size);
 
-            Assert.True(pixels.Length >= 1024);
+            Assert.True(pixels.Length >= size);
         }
 
         [Fact]
@@ -65,17 +71,21 @@ namespace SixLabors.ImageSharp.Tests.Memory
             }
         }
 
-        [Fact]
-        public void SmallBuffersArePooled()
+        [Theory]
+        [InlineData(32)]
+        [InlineData(512)]
+        [InlineData(MaxPooledBufferSizeInBytes-1)]
+        public void SmallBuffersArePooled(int size)
         {
-            Assert.True(this.CheckIsPooled<byte>(5, 512));
+            Assert.True(this.CheckIsPooled<byte>(5, size));
         }
 
-        [Fact]
-        public void LargeBuffersAreNotPooled_OfByte()
+        [Theory]
+        [InlineData(128 * 1024 * 1024)]
+        [InlineData(MaxPooledBufferSizeInBytes+1)]
+        public void LargeBuffersAreNotPooled_OfByte(int size)
         {
-            const int mb128 = 128 * 1024 * 1024;
-            Assert.False(this.CheckIsPooled<byte>(2, mb128));
+            Assert.False(this.CheckIsPooled<byte>(2, size));
         }
 
         [StructLayout(LayoutKind.Explicit, Size = 512)]