diff --git a/src/ImageSharp/Colors/Color.BulkOperations.cs b/src/ImageSharp/Colors/Color.BulkOperations.cs
index 154cd531d..5c040e04c 100644
--- a/src/ImageSharp/Colors/Color.BulkOperations.cs
+++ b/src/ImageSharp/Colors/Color.BulkOperations.cs
@@ -32,6 +32,10 @@ namespace ImageSharp
             /// <see>
             ///     <cref>http://stackoverflow.com/a/5362789</cref>
             /// </see>
+            /// TODO: We can replace this implementation in the future using new Vector API-s:
+            /// <see>
+            ///     <cref>https://github.com/dotnet/corefx/issues/15957</cref>
+            /// </see>
             /// </remarks>
             internal static unsafe void ToVector4SimdAligned(
                 BufferPointer<Color> sourceColors,
@@ -56,8 +60,7 @@ namespace ImageSharp
                 uint* srcEnd = src + count;
 
                 using (PinnedBuffer<uint> tempBuf = new PinnedBuffer<uint>(
-                        unpackedRawCount + Vector<uint>.Count,
-                        PixelDataPool<uint>.Dirty))
+                        unpackedRawCount + Vector<uint>.Count))
                 {
                     uint* tPtr = (uint*)tempBuf.Pointer;
                     uint[] temp = tempBuf.Array;
@@ -66,7 +69,7 @@ namespace ImageSharp
 
                     for (; src < srcEnd; src++, dst++)
                     {
-                        // TODO: This is the bottleneck now. We can improve it with future Vector<T> API-s (https://github.com/dotnet/corefx/issues/15957)
+                        // This call is the bottleneck now:
                         dst->Load(*src);
                     }
 
diff --git a/src/ImageSharp/Common/Memory/BufferPointer{T}.cs b/src/ImageSharp/Common/Memory/BufferPointer{T}.cs
index 4e7a40078..441f6b8ce 100644
--- a/src/ImageSharp/Common/Memory/BufferPointer{T}.cs
+++ b/src/ImageSharp/Common/Memory/BufferPointer{T}.cs
@@ -130,7 +130,14 @@ namespace ImageSharp
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void Clear(int count)
         {
-            
+            if (count < 256)
+            {
+                Unsafe.InitBlock((void*)this.PointerAtOffset, 0, BufferPointer.USizeOf<T>(count));
+            }
+            else
+            {
+                System.Array.Clear(this.Array, this.Offset, count);
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Common/Memory/PinnedBuffer{T}.cs b/src/ImageSharp/Common/Memory/PinnedBuffer{T}.cs
index 6e0d1fcc6..2d3d44dda 100644
--- a/src/ImageSharp/Common/Memory/PinnedBuffer{T}.cs
+++ b/src/ImageSharp/Common/Memory/PinnedBuffer{T}.cs
@@ -24,32 +24,23 @@ namespace ImageSharp
         private GCHandle handle;
 
         /// <summary>
-        /// The <see cref="PixelDataPool{T}"/> if the <see cref="Array"/> is pooled.
+        /// A value indicating wheter <see cref="Array"/> should be returned to <see cref="PixelDataPool{T}"/>
+        /// when disposing this <see cref="PinnedBuffer{T}"/> instance.
         /// </summary>
-        private PixelDataPool<T> pool;
+        private bool isPoolingOwner;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="PinnedBuffer{T}"/> class.
         /// </summary>
         /// <param name="count">The desired count of elements. (Minimum size for <see cref="Array"/>)</param>
-        /// <param name="pool">The <see cref="PixelDataPool{T}"/> to be used to rent the data.</param>
-        public PinnedBuffer(int count, PixelDataPool<T> pool)
+        public PinnedBuffer(int count)
         {
             this.Count = count;
-            this.pool = pool;
-            this.Array = this.pool.Rent(count);
+            this.Array = PixelDataPool<T>.Rent(count);
+            this.isPoolingOwner = true;
             this.Pin();
         }
 
-        /// <summary>
-        /// Initializes a new instance of the <see cref="PinnedBuffer{T}"/> class.
-        /// </summary>
-        /// <param name="count">The desired count of elements. (Minimum size for <see cref="Array"/>)</param>
-        public PinnedBuffer(int count)
-            : this(count, PixelDataPool<T>.Clean)
-        {
-        }
-
         /// <summary>
         /// Initializes a new instance of the <see cref="PinnedBuffer{T}"/> class.
         /// </summary>
@@ -58,7 +49,7 @@ namespace ImageSharp
         {
             this.Count = array.Length;
             this.Array = array;
-            this.pool = null;
+            this.isPoolingOwner = false;
             this.Pin();
         }
 
@@ -76,7 +67,7 @@ namespace ImageSharp
 
             this.Count = count;
             this.Array = array;
-            this.pool = null;
+            this.isPoolingOwner = false;
             this.Pin();
         }
 
@@ -153,8 +144,12 @@ namespace ImageSharp
             this.IsDisposedOrLostArrayOwnership = true;
             this.UnPin();
 
-            this.pool?.Return(this.Array);
-            this.pool = null;
+            if (this.isPoolingOwner)
+            {
+                PixelDataPool<T>.Return(this.Array);
+            }
+
+            this.isPoolingOwner = false;
             this.Array = null;
             this.Count = 0;
 
@@ -178,9 +173,19 @@ namespace ImageSharp
             this.UnPin();
             T[] array = this.Array;
             this.Array = null;
+            this.isPoolingOwner = false;
             return array;
         }
 
+        /// <summary>
+        /// Clears the buffer, filling elements between 0 and <see cref="Count"/>-1 with default(T)
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void Clear()
+        {
+            this.Slice().Clear(this.Count);
+        }
+
         /// <summary>
         /// Pins <see cref="Array"/>.
         /// </summary>
diff --git a/src/ImageSharp/Common/Memory/PixelDataPool{T}.cs b/src/ImageSharp/Common/Memory/PixelDataPool{T}.cs
index a01a941bc..dcd031f6e 100644
--- a/src/ImageSharp/Common/Memory/PixelDataPool{T}.cs
+++ b/src/ImageSharp/Common/Memory/PixelDataPool{T}.cs
@@ -15,63 +15,32 @@ namespace ImageSharp
     public class PixelDataPool<T>
         where T : struct
     {
-        /// <summary>
-        /// The <see cref="ArrayPool{T}"/> which will be always cleared.
-        /// </summary>
-        private static readonly ArrayPool<T> CleanPool = ArrayPool<T>.Create(CalculateMaxArrayLength(), 50);
-
         /// <summary>
         /// The <see cref="ArrayPool{T}"/> which is not kept clean.
         /// </summary>
-        private static readonly ArrayPool<T> DirtyPool = ArrayPool<T>.Create(CalculateMaxArrayLength(), 50);
-
-        /// <summary>
-        /// The backing <see cref="ArrayPool{T}"/>
-        /// </summary>
-        private ArrayPool<T> arrayPool;
-
-        /// <summary>
-        /// A value indicating whether clearArray is requested on <see cref="ArrayPool{T}.Return(T[], bool)"/>.
-        /// </summary>
-        private bool clearArray;
-
-        private PixelDataPool(ArrayPool<T> arrayPool, bool clearArray)
-        {
-            this.clearArray = clearArray;
-            this.arrayPool = arrayPool;
-        }
-
-        /// <summary>
-        /// Gets the <see cref="PixelDataPool{T}"/> which will always return arrays initialized to default(T)
-        /// </summary>
-        public static PixelDataPool<T> Clean { get; } = new PixelDataPool<T>(CleanPool, true);
-
-        /// <summary>
-        /// Gets the <see cref="PixelDataPool{T}"/> which does not keep the arrays clean on Rent/Return.
-        /// </summary>
-        public static PixelDataPool<T> Dirty { get; } = new PixelDataPool<T>(DirtyPool, false);
+        private static readonly ArrayPool<T> ArrayPool = ArrayPool<T>.Create(CalculateMaxArrayLength(), 50);
 
         /// <summary>
         /// Rents the pixel array from the pool.
         /// </summary>
         /// <param name="minimumLength">The minimum length of the array to return.</param>
         /// <returns>The <see cref="T:TColor[]"/></returns>
-        public T[] Rent(int minimumLength)
+        public static T[] Rent(int minimumLength)
         {
-            return CleanPool.Rent(minimumLength);
+            return ArrayPool.Rent(minimumLength);
         }
 
         /// <summary>
         /// Returns the rented pixel array back to the pool.
         /// </summary>
         /// <param name="array">The array to return to the buffer pool.</param>
-        public void Return(T[] array)
+        public static void Return(T[] array)
         {
-            CleanPool.Return(array, this.clearArray);
+            ArrayPool.Return(array);
         }
 
         /// <summary>
-        /// Heuristically calculates a reasonable maxArrayLength value for the backing <see cref="CleanPool"/>.
+        /// Heuristically calculates a reasonable maxArrayLength value for the backing <see cref="ArrayPool{T}"/>.
         /// </summary>
         /// <returns>The maxArrayLength value</returns>
         internal static int CalculateMaxArrayLength()
diff --git a/src/ImageSharp/Image/ImageBase{TColor}.cs b/src/ImageSharp/Image/ImageBase{TColor}.cs
index 7d7fc843f..7afc27a6d 100644
--- a/src/ImageSharp/Image/ImageBase{TColor}.cs
+++ b/src/ImageSharp/Image/ImageBase{TColor}.cs
@@ -60,6 +60,7 @@ namespace ImageSharp
         {
             this.Configuration = configuration ?? Configuration.Default;
             this.InitPixels(width, height);
+            this.ClearPixels();
         }
 
         /// <summary>
@@ -221,7 +222,7 @@ namespace ImageSharp
         /// </summary>
         private void RentPixels()
         {
-            this.pixelBuffer = PixelDataPool<TColor>.Clean.Rent(this.Width * this.Height);
+            this.pixelBuffer = PixelDataPool<TColor>.Rent(this.Width * this.Height);
         }
 
         /// <summary>
@@ -229,8 +230,13 @@ namespace ImageSharp
         /// </summary>
         private void ReturnPixels()
         {
-            PixelDataPool<TColor>.Clean.Return(this.pixelBuffer);
+            PixelDataPool<TColor>.Return(this.pixelBuffer);
             this.pixelBuffer = null;
         }
+
+        private void ClearPixels()
+        {
+            Array.Clear(this.pixelBuffer, 0, this.Width * this.Height);
+        }
     }
 }
\ No newline at end of file
diff --git a/tests/ImageSharp.Sandbox46/Program.cs b/tests/ImageSharp.Sandbox46/Program.cs
index c361fd6a1..467663a53 100644
--- a/tests/ImageSharp.Sandbox46/Program.cs
+++ b/tests/ImageSharp.Sandbox46/Program.cs
@@ -46,7 +46,7 @@ namespace ImageSharp.Sandbox46
 
         private static void RunToVector4ProfilingTest()
         {
-            BulkPixelOperationsTests.Color tests = new BulkPixelOperationsTests.Color();
+            BulkPixelOperationsTests.Color tests = new BulkPixelOperationsTests.Color(new ConsoleOutput());
             tests.Benchmark_ToVector4();
         }
         
diff --git a/tests/ImageSharp.Tests/Common/BufferPointerTests.cs b/tests/ImageSharp.Tests/Common/BufferPointerTests.cs
index 39d76d937..c82b63f11 100644
--- a/tests/ImageSharp.Tests/Common/BufferPointerTests.cs
+++ b/tests/ImageSharp.Tests/Common/BufferPointerTests.cs
@@ -146,6 +146,11 @@ namespace ImageSharp.Tests.Common
 
                 // Act:
                 ap.Clear(count);
+
+                Assert.NotEqual(default(Foo), array[offset-1]);
+                Assert.Equal(default(Foo), array[offset]);
+                Assert.Equal(default(Foo), array[offset + count-1]);
+                Assert.NotEqual(default(Foo), array[offset + count]);
             }
         }
 
diff --git a/tests/ImageSharp.Tests/Common/PinnedBufferTests.cs b/tests/ImageSharp.Tests/Common/PinnedBufferTests.cs
index 65077ae7f..3688763b9 100644
--- a/tests/ImageSharp.Tests/Common/PinnedBufferTests.cs
+++ b/tests/ImageSharp.Tests/Common/PinnedBufferTests.cs
@@ -47,6 +47,21 @@
             }
         }
 
+        [Theory]
+        [InlineData(42)]
+        [InlineData(1111)]
+        public void Clear(int count)
+        {
+            Foo[] a = { new Foo() { A = 1, B = 2 }, new Foo() { A = 3, B = 4 } };
+            using (PinnedBuffer<Foo> buffer = new PinnedBuffer<Foo>(a))
+            {
+                buffer.Clear();
+
+                Assert.Equal(default(Foo), a[0]);
+                Assert.Equal(default(Foo), a[1]);
+            }
+        }
+
         [Fact]
         public void Dispose()
         {
diff --git a/tests/ImageSharp.Tests/Common/PixelDataPoolTests.cs b/tests/ImageSharp.Tests/Common/PixelDataPoolTests.cs
index db560ba6b..ea747af7d 100644
--- a/tests/ImageSharp.Tests/Common/PixelDataPoolTests.cs
+++ b/tests/ImageSharp.Tests/Common/PixelDataPoolTests.cs
@@ -15,17 +15,10 @@ namespace ImageSharp.Tests
     /// </summary>
     public class PixelDataPoolTests
     {
-        private static PixelDataPool<Color> GetPool(bool clean)
-        {
-            return clean ? PixelDataPool<Color>.Clean : PixelDataPool<Color>.Dirty;
-        }
-
-        [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public void PixelDataPoolRentsMinimumSize(bool clean)
+        [Fact]
+        public void PixelDataPoolRentsMinimumSize()
         {
-            Color[] pixels = GetPool(clean).Rent(1024);
+            Color[] pixels = PixelDataPool<Color>.Rent(1024);
 
             Assert.True(pixels.Length >= 1024);
         }
@@ -35,31 +28,29 @@ namespace ImageSharp.Tests
         {
             for (int i = 16; i < 1024; i += 16)
             {
-                Color[] pixels = PixelDataPool<Color>.Clean.Rent(i);
+                Color[] pixels = PixelDataPool<Color>.Rent(i);
 
                 Assert.True(pixels.All(p => p == default(Color)));
 
-                PixelDataPool<Color>.Clean.Return(pixels);
+                PixelDataPool<Color>.Return(pixels);
             }
 
             for (int i = 16; i < 1024; i += 16)
             {
-                Color[] pixels = PixelDataPool<Color>.Clean.Rent(i);
+                Color[] pixels = PixelDataPool<Color>.Rent(i);
 
                 Assert.True(pixels.All(p => p == default(Color)));
 
-                PixelDataPool<Color>.Clean.Return(pixels);
+                PixelDataPool<Color>.Return(pixels);
             }
         }
 
-        [Theory]
-        [InlineData(false)]
-        [InlineData(true)]
-        public void PixelDataPoolDoesNotThrowWhenReturningNonPooled(bool clean)
+        [Fact]
+        public void PixelDataPoolDoesNotThrowWhenReturningNonPooled()
         {
             Color[] pixels = new Color[1024];
 
-            GetPool(clean).Return(pixels);
+            PixelDataPool<Color>.Return(pixels);
 
             Assert.True(pixels.Length >= 1024);
         }
@@ -67,7 +58,7 @@ namespace ImageSharp.Tests
         [Fact]
         public void PixelDataPool_Clean_CleansRentedArray()
         {
-            Color[] pixels = PixelDataPool<Color>.Clean.Rent(256);
+            Color[] pixels = PixelDataPool<Color>.Rent(256);
 
             for (int i = 0; i < pixels.Length; i++)
             {
@@ -76,7 +67,7 @@ namespace ImageSharp.Tests
 
             Assert.True(pixels.All(p => p == Color.Azure));
 
-            PixelDataPool<Color>.Clean.Return(pixels);
+            PixelDataPool<Color>.Return(pixels);
 
             Assert.True(pixels.All(p => p == default(Color)));
         }
@@ -95,7 +86,7 @@ namespace ImageSharp.Tests
         [Fact]
         public void RentNonIPixelData()
         {
-            byte[] data = PixelDataPool<byte>.Clean.Rent(16384);
+            byte[] data = PixelDataPool<byte>.Rent(16384);
 
             Assert.True(data.Length >= 16384);
         }