Revert "Pin the refs for faster access"

src/ImageSharp/Formats/Webp/Lossless/Vp8LBackwardRefs.cs

@@ -8,33 +8,21 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossless;
 
 internal class Vp8LBackwardRefs : IDisposable
 {
-    private readonly IMemoryOwner<PixOrCopy> owner;
-    private readonly MemoryHandle handle;
+    private readonly IMemoryOwner<PixOrCopy> refs;
     private int count;
 
     public Vp8LBackwardRefs(MemoryAllocator memoryAllocator, int pixels)
     {
-        this.owner = memoryAllocator.Allocate<PixOrCopy>(pixels);
-        this.handle = this.owner.Memory.Pin();
+        this.refs = memoryAllocator.Allocate<PixOrCopy>(pixels);
         this.count = 0;
     }
 
-    public void Add(PixOrCopy pixOrCopy)
-    {
-        unsafe
-        {
-            ((PixOrCopy*)this.handle.Pointer)[this.count++] = pixOrCopy;
-        }
-    }
+    public void Add(PixOrCopy pixOrCopy) => this.refs.Memory.Span[this.count++] = pixOrCopy;
 
     public void Clear() => this.count = 0;
 
-    public Span<PixOrCopy>.Enumerator GetEnumerator() => this.owner.Slice(0, this.count).GetEnumerator();
+    public Span<PixOrCopy>.Enumerator GetEnumerator() => this.refs.Slice(0, this.count).GetEnumerator();
 
     /// <inheritdoc/>
-    public void Dispose()
-    {
-        this.handle.Dispose();
-        this.owner.Dispose();
-    }
+    public void Dispose() => this.refs.Dispose();
 }