diff --git a/src/ImageSharp.Drawing/Processors/FillRegionProcessor.cs b/src/ImageSharp.Drawing/Processors/FillRegionProcessor.cs
index ae5ee2d9f6..076785526c 100644
--- a/src/ImageSharp.Drawing/Processors/FillRegionProcessor.cs
+++ b/src/ImageSharp.Drawing/Processors/FillRegionProcessor.cs
@@ -183,7 +183,7 @@ namespace SixLabors.ImageSharp.Drawing.Processors
                                 }
                             }
 
-                            applicator.Apply(scanline, minX, y);
+                            applicator.Apply(scanline.Span, minX, y);
                         }
                     }
                 }
diff --git a/src/ImageSharp/Formats/Jpeg/Common/Decoder/JpegBlockPostProcessor.cs b/src/ImageSharp/Formats/Jpeg/Common/Decoder/JpegBlockPostProcessor.cs
index 574967b6bf..5e8e8fa2cc 100644
--- a/src/ImageSharp/Formats/Jpeg/Common/Decoder/JpegBlockPostProcessor.cs
+++ b/src/ImageSharp/Formats/Jpeg/Common/Decoder/JpegBlockPostProcessor.cs
@@ -8,7 +8,7 @@ using SixLabors.Primitives;
 namespace SixLabors.ImageSharp.Formats.Jpeg.Common.Decoder
 {
     /// <summary>
-    /// Encapsulates the implementation of processing "raw" <see cref="Buffer{T}"/>-s into Jpeg image channels.
+    /// Encapsulates the implementation of processing "raw" <see cref="IBuffer{T}"/>-s into Jpeg image channels.
     /// </summary>
     [StructLayout(LayoutKind.Sequential)]
     internal struct JpegBlockPostProcessor
diff --git a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
index b8694c538e..3c43ba2444 100644
--- a/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
+++ b/src/ImageSharp/Formats/Jpeg/PdfJsPort/Components/PdfJsHuffmanTable.cs
@@ -30,13 +30,13 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.PdfJsPort.Components
             this.valOffset = memoryManager.AllocateFake<short>(18);
             this.maxcode = memoryManager.AllocateFake<long>(18);
 
-            using (BasicArrayBuffer<short> huffsize = memoryManager.AllocateFake<short>(257))
-            using (BasicArrayBuffer<short> huffcode = memoryManager.AllocateFake<short>(257))
+            using (IBuffer<short> huffsize = memoryManager.Allocate<short>(257))
+            using (IBuffer<short> huffcode = memoryManager.Allocate<short>(257))
             {
-                GenerateSizeTable(lengths, huffsize);
-                GenerateCodeTable(huffsize, huffcode);
-                GenerateDecoderTables(lengths, huffcode, this.valOffset, this.maxcode);
-                GenerateLookaheadTables(lengths, values, this.lookahead);
+                GenerateSizeTable(lengths, huffsize.Span);
+                GenerateCodeTable(huffsize.Span, huffcode.Span);
+                GenerateDecoderTables(lengths, huffcode.Span, this.valOffset.Span, this.maxcode.Span);
+                GenerateLookaheadTables(lengths, values, this.lookahead.Span);
             }
 
             this.huffval = memoryManager.AllocateManagedByteBuffer(values.Length, true);
diff --git a/src/ImageSharp/Image/PixelAccessor{TPixel}.cs b/src/ImageSharp/Image/PixelAccessor{TPixel}.cs
index 80c0ce4e66..3a894fb9ad 100644
--- a/src/ImageSharp/Image/PixelAccessor{TPixel}.cs
+++ b/src/ImageSharp/Image/PixelAccessor{TPixel}.cs
@@ -19,7 +19,7 @@ namespace SixLabors.ImageSharp
     {
 #pragma warning disable SA1401 // Fields must be private
         /// <summary>
-        /// The <see cref="Buffer{T}"/> containing the pixel data.
+        /// The <see cref="Buffer2D{T}"/> containing the pixel data.
         /// </summary>
         internal Buffer2D<TPixel> PixelBuffer;
         private bool ownedBuffer;
diff --git a/src/ImageSharp/Memory/ArrayPoolMemoryManager.Buffer{T}.cs b/src/ImageSharp/Memory/ArrayPoolMemoryManager.Buffer{T}.cs
new file mode 100644
index 0000000000..5b03520106
--- /dev/null
+++ b/src/ImageSharp/Memory/ArrayPoolMemoryManager.Buffer{T}.cs
@@ -0,0 +1,50 @@
+using System;
+
+namespace SixLabors.ImageSharp.Memory
+{
+    /// <summary>
+    /// Contains <see cref="Buffer{T}"/> and <see cref="ManagedByteBuffer"/>
+    /// </summary>
+    public partial class ArrayPoolMemoryManager
+    {
+        private class Buffer<T> : IBuffer<T>
+            where T : struct
+        {
+            private readonly ArrayPoolMemoryManager memoryManager;
+
+            private readonly int length;
+
+            public Buffer(byte[] data, int length, ArrayPoolMemoryManager memoryManager)
+            {
+                this.memoryManager = memoryManager;
+                this.Data = data;
+                this.length = length;
+            }
+
+            protected byte[] Data { get; private set; }
+
+            public Span<T> Span => this.Data.AsSpan().NonPortableCast<byte, T>().Slice(0, this.length);
+
+            public void Dispose()
+            {
+                if (this.Data == null)
+                {
+                    return;
+                }
+
+                this.memoryManager.pool.Return(this.Data);
+                this.Data = null;
+            }
+        }
+
+        private class ManagedByteBuffer : Buffer<byte>, IManagedByteBuffer
+        {
+            public ManagedByteBuffer(byte[] data, int length, ArrayPoolMemoryManager memoryManager)
+                : base(data, length, memoryManager)
+            {
+            }
+
+            public byte[] Array => this.Data;
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs b/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
index de26b0ba5e..41ef847847 100644
--- a/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
+++ b/src/ImageSharp/Memory/ArrayPoolMemoryManager.cs
@@ -1,5 +1,4 @@
-using System;
-using System.Buffers;
+using System.Buffers;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -8,7 +7,7 @@ namespace SixLabors.ImageSharp.Memory
     /// <summary>
     /// Implements <see cref="MemoryManager"/> by allocating memory from <see cref="ArrayPool{T}"/>.
     /// </summary>
-    public class ArrayPoolMemoryManager : MemoryManager
+    public partial class ArrayPoolMemoryManager : MemoryManager
     {
         /// <summary>
         /// Defines the default maximum size of pooled arrays.
@@ -44,7 +43,7 @@ namespace SixLabors.ImageSharp.Memory
             int bufferSizeInBytes = length * itemSizeBytes;
 
             byte[] byteBuffer = this.pool.Rent(bufferSizeInBytes);
-            var buffer = new Buffer<T>(Unsafe.As<T[]>(byteBuffer), length, this);
+            var buffer = new Buffer<T>(byteBuffer, length, this);
             if (clear)
             {
                 buffer.Clear();
@@ -64,19 +63,5 @@ namespace SixLabors.ImageSharp.Memory
 
             return buffer;
         }
-
-        /// <inheritdoc />
-        internal override void Release<T>(Buffer<T> buffer)
-        {
-            T[] array = (buffer as IGetArray<T>)?.GetArray();
-            if (array == null)
-            {
-                return;
-            }
-
-            // TODO: OMG Do not do this!
-            byte[] byteBuffer = Unsafe.As<byte[]>(array);
-            this.pool.Return(byteBuffer);
-        }
     }
 }
\ No newline at end of file
diff --git a/src/ImageSharp/Memory/BasicArrayBuffer.cs b/src/ImageSharp/Memory/BasicArrayBuffer.cs
index d9eb5a19a1..17bf4c8439 100644
--- a/src/ImageSharp/Memory/BasicArrayBuffer.cs
+++ b/src/ImageSharp/Memory/BasicArrayBuffer.cs
@@ -36,27 +36,7 @@ namespace SixLabors.ImageSharp.Memory
                 return ref span[index];
             }
         }
-
-        /// <summary>
-        /// Converts <see cref="Buffer{T}"/> to an <see cref="ReadOnlySpan{T}"/>.
-        /// </summary>
-        /// <param name="buffer">The <see cref="Buffer{T}"/> to convert.</param>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static implicit operator ReadOnlySpan<T>(BasicArrayBuffer<T> buffer)
-        {
-            return new ReadOnlySpan<T>(buffer.Array, 0, buffer.Length);
-        }
-
-        /// <summary>
-        /// Converts <see cref="Buffer{T}"/> to an <see cref="Span{T}"/>.
-        /// </summary>
-        /// <param name="buffer">The <see cref="Buffer{T}"/> to convert.</param>
-        [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        public static implicit operator Span<T>(BasicArrayBuffer<T> buffer)
-        {
-            return new Span<T>(buffer.Array, 0, buffer.Length);
-        }
-
+        
         public void Dispose()
         {
         }
diff --git a/src/ImageSharp/Memory/BasicByteBuffer.cs b/src/ImageSharp/Memory/BasicByteBuffer.cs
new file mode 100644
index 0000000000..96b69ad3bf
--- /dev/null
+++ b/src/ImageSharp/Memory/BasicByteBuffer.cs
@@ -0,0 +1,10 @@
+namespace SixLabors.ImageSharp.Memory
+{
+    internal class BasicByteBuffer : BasicArrayBuffer<byte>, IManagedByteBuffer
+    {
+        internal BasicByteBuffer(byte[] array)
+            : base(array)
+        {
+        }
+    }
+}
\ No newline at end of file
diff --git a/src/ImageSharp/Memory/Buffer{T}.cs b/src/ImageSharp/Memory/Buffer{T}.cs
index b1449f9b61..547e6f209d 100644
--- a/src/ImageSharp/Memory/Buffer{T}.cs
+++ b/src/ImageSharp/Memory/Buffer{T}.cs
@@ -58,9 +58,7 @@ namespace SixLabors.ImageSharp.Memory
             {
                 return;
             }
-
-            this.memoryManager?.Release(this);
-
+            
             this.memoryManager = null;
             this.array = null;
             this.Length = 0;
diff --git a/src/ImageSharp/Memory/ManagedByteBuffer.cs b/src/ImageSharp/Memory/ManagedByteBuffer.cs
deleted file mode 100644
index 94d08e2aa7..0000000000
--- a/src/ImageSharp/Memory/ManagedByteBuffer.cs
+++ /dev/null
@@ -1,12 +0,0 @@
-namespace SixLabors.ImageSharp.Memory
-{
-    internal class ManagedByteBuffer : Buffer<byte>, IManagedByteBuffer
-    {
-        internal ManagedByteBuffer(byte[] array, int length, MemoryManager memoryManager)
-            : base(array, length, memoryManager)
-        {
-        }
-
-        public byte[] Array => this.array;
-    }
-}
\ No newline at end of file
diff --git a/src/ImageSharp/Memory/MemoryManager.cs b/src/ImageSharp/Memory/MemoryManager.cs
index fe6bb7823f..7445c66b7e 100644
--- a/src/ImageSharp/Memory/MemoryManager.cs
+++ b/src/ImageSharp/Memory/MemoryManager.cs
@@ -12,7 +12,7 @@ namespace SixLabors.ImageSharp.Memory
     public abstract class MemoryManager
     {
         /// <summary>
-        /// Allocates a <see cref="Buffer{T}"/> of size <paramref name="length"/>, optionally
+        /// Allocates an <see cref="IBuffer{T}"/> of size <paramref name="length"/>, optionally
         /// clearing the buffer before it gets returned.
         /// </summary>
         /// <typeparam name="T">Type of the data stored in the buffer</typeparam>
@@ -24,15 +24,6 @@ namespace SixLabors.ImageSharp.Memory
 
         internal abstract IManagedByteBuffer AllocateManagedByteBuffer(int length, bool clear);
 
-        /// <summary>
-        /// Releases the memory allocated for <paramref name="buffer"/>. After this, the buffer
-        /// is no longer usable.
-        /// </summary>
-        /// <typeparam name="T">Type of the data stored in the buffer</typeparam>
-        /// <param name="buffer">The buffer to release</param>
-        internal abstract void Release<T>(Buffer<T> buffer)
-            where T : struct;
-
         /// <summary>
         /// Temporal workaround. A method providing a "Buffer" based on a generic array without the 'Unsafe.As()' hackery.
         /// Should be replaced with 'Allocate()' as soon as SixLabors.Shapes has Span-based API-s!
diff --git a/src/ImageSharp/Memory/MemoryManagerExtensions.cs b/src/ImageSharp/Memory/MemoryManagerExtensions.cs
index 21c8f71248..b7fcaf4b36 100644
--- a/src/ImageSharp/Memory/MemoryManagerExtensions.cs
+++ b/src/ImageSharp/Memory/MemoryManagerExtensions.cs
@@ -6,7 +6,7 @@
     internal static class MemoryManagerExtensions
     {
         /// <summary>
-        /// Allocates a <see cref="Buffer{T}"/> of size <paramref name="length"/>.
+        /// Allocates a <see cref="IBuffer{T}"/> of size <paramref name="length"/>.
         /// Note: Depending on the implementation, the buffer may not cleared before
         /// returning, so it may contain data from an earlier use.
         /// </summary>
diff --git a/src/ImageSharp/Memory/SimpleManagedMemoryManager.cs b/src/ImageSharp/Memory/SimpleManagedMemoryManager.cs
index 804a468fd6..701c71ad43 100644
--- a/src/ImageSharp/Memory/SimpleManagedMemoryManager.cs
+++ b/src/ImageSharp/Memory/SimpleManagedMemoryManager.cs
@@ -8,17 +8,12 @@
         /// <inheritdoc />
         internal override IBuffer<T> Allocate<T>(int length, bool clear)
         {
-            return new Buffer<T>(new T[length], length, this);
+            return new BasicArrayBuffer<T>(new T[length]);
         }
 
         internal override IManagedByteBuffer AllocateManagedByteBuffer(int length, bool clear)
         {
-            return new ManagedByteBuffer(new byte[length], length, this);
-        }
-
-        /// <inheritdoc />
-        internal override void Release<T>(Buffer<T> buffer)
-        {
+            return new BasicByteBuffer(new byte[length]);
         }
     }
 }