diff --git a/src/ImageSharp/Memory/ByteMemoryManager{T}.cs b/src/ImageSharp/Memory/ByteMemoryManager{T}.cs
index 70d1b1c305..223709df65 100644
--- a/src/ImageSharp/Memory/ByteMemoryManager{T}.cs
+++ b/src/ImageSharp/Memory/ByteMemoryManager{T}.cs
@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0.
 using System;
 using System.Buffers;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
 namespace SixLabors.ImageSharp.Memory
@@ -42,7 +43,10 @@ namespace SixLabors.ImageSharp.Memory
         /// <inheritdoc/>
         public override MemoryHandle Pin(int elementIndex = 0)
         {
-            return this.memory.Slice(elementIndex).Pin();
+            // We need to adjust the offset into the wrapped byte segment,
+            // as the input index refers to the target-cast memory of T.
+            // We just have to shift this index by the byte size of T.
+            return this.memory.Slice(elementIndex * Unsafe.SizeOf<T>()).Pin();
         }
 
         /// <inheritdoc/>
diff --git a/tests/ImageSharp.Tests/Image/ImageTests.WrapMemory.cs b/tests/ImageSharp.Tests/Image/ImageTests.WrapMemory.cs
index 06ee069c67..ee8e4b97af 100644
--- a/tests/ImageSharp.Tests/Image/ImageTests.WrapMemory.cs
+++ b/tests/ImageSharp.Tests/Image/ImageTests.WrapMemory.cs
@@ -105,7 +105,15 @@ namespace SixLabors.ImageSharp.Tests
                 /// <inheritdoc/>
                 public override MemoryHandle Pin(int elementIndex = 0)
                 {
-                    return this.memory.Slice(elementIndex).Pin();
+                    int byteOffset = elementIndex * Unsafe.SizeOf<TTo>();
+                    int shiftedOffset = Math.DivRem(byteOffset, Unsafe.SizeOf<TFrom>(), out int remainder);
+
+                    if (remainder != 0)
+                    {
+                        ThrowHelper.ThrowArgumentException("The input index doesn't result in an aligned item access", nameof(elementIndex));
+                    }
+
+                    return this.memory.Slice(shiftedOffset).Pin();
                 }
 
                 /// <inheritdoc/>