diff --git a/src/ImageSharp/Formats/Webp/Lossless/Vp8LBackwardRefs.cs b/src/ImageSharp/Formats/Webp/Lossless/Vp8LBackwardRefs.cs
index 634fac5e8..3b7dc99ff 100644
--- a/src/ImageSharp/Formats/Webp/Lossless/Vp8LBackwardRefs.cs
+++ b/src/ImageSharp/Formats/Webp/Lossless/Vp8LBackwardRefs.cs
@@ -8,21 +8,33 @@ namespace SixLabors.ImageSharp.Formats.Webp.Lossless;
 
 internal class Vp8LBackwardRefs : IDisposable
 {
-    private readonly IMemoryOwner<PixOrCopy> refs;
+    private readonly IMemoryOwner<PixOrCopy> owner;
+    private readonly MemoryHandle handle;
     private int count;
 
     public Vp8LBackwardRefs(MemoryAllocator memoryAllocator, int pixels)
     {
-        this.refs = memoryAllocator.Allocate<PixOrCopy>(pixels);
+        this.owner = memoryAllocator.Allocate<PixOrCopy>(pixels);
+        this.handle = this.owner.Memory.Pin();
         this.count = 0;
     }
 
-    public void Add(PixOrCopy pixOrCopy) => this.refs.Memory.Span[this.count++] = pixOrCopy;
+    public void Add(PixOrCopy pixOrCopy)
+    {
+        unsafe
+        {
+            ((PixOrCopy*)this.handle.Pointer)[this.count++] = pixOrCopy;
+        }
+    }
 
     public void Clear() => this.count = 0;
 
-    public Span<PixOrCopy>.Enumerator GetEnumerator() => this.refs.Slice(0, this.count).GetEnumerator();
+    public Span<PixOrCopy>.Enumerator GetEnumerator() => this.owner.Slice(0, this.count).GetEnumerator();
 
     /// <inheritdoc/>
-    public void Dispose() => this.refs.Dispose();
+    public void Dispose()
+    {
+        this.handle.Dispose();
+        this.owner.Dispose();
+    }
 }