Speed up DangerousGetRowSpan with SpanCache

src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs

@@ -13,34 +13,35 @@ namespace SixLabors.ImageSharp.Memory.Internals
         where T : struct
     {
         private readonly int lengthInBytes;
-        private byte[] array;
         private LifetimeGuard lifetimeGuard;
 
         public SharedArrayPoolBuffer(int lengthInElements)
         {
             this.lengthInBytes = lengthInElements * Unsafe.SizeOf<T>();
-            this.array = ArrayPool<byte>.Shared.Rent(this.lengthInBytes);
+            this.Array = ArrayPool<byte>.Shared.Rent(this.lengthInBytes);
-            this.lifetimeGuard = new LifetimeGuard(this.array);
+            this.lifetimeGuard = new LifetimeGuard(this.Array);
         }
 
+        public byte[] Array { get; private set; }
+
         protected override void Dispose(bool disposing)
         {
-            if (this.array == null)
+            if (this.Array == null)
             {
                 return;
             }
 
             this.lifetimeGuard.Dispose();
-            this.array = null;
+            this.Array = null;
         }
 
         public override Span<T> GetSpan()
         {
             this.CheckDisposed();
-            return MemoryMarshal.Cast<byte, T>(this.array.AsSpan(0, this.lengthInBytes));
+            return MemoryMarshal.Cast<byte, T>(this.Array.AsSpan(0, this.lengthInBytes));
         }
 
-        protected override object GetPinnableObject() => this.array;
+        protected override object GetPinnableObject() => this.Array;
 
         public void AddRef()
         {
@@ -53,7 +54,7 @@ namespace SixLabors.ImageSharp.Memory.Internals
         [Conditional("DEBUG")]
         private void CheckDisposed()
         {
-            if (this.array == null)
+            if (this.Array == null)
             {
                 throw new ObjectDisposedException("SharedArrayPoolBuffer");
             }

src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs

@@ -8,12 +8,10 @@ using System.Threading;
 
 namespace SixLabors.ImageSharp.Memory.Internals
 {
-    internal partial class UniformUnmanagedMemoryPool
+    // CriticalFinalizerObject:
-#if !NETSTANDARD1_3
+    // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers,
-        // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers,
+    // but we should not rely on this.
-        // but we should not rely on this.
+    internal partial class UniformUnmanagedMemoryPool : System.Runtime.ConstrainedExecution.CriticalFinalizerObject
-        : System.Runtime.ConstrainedExecution.CriticalFinalizerObject
-#endif
     {
         private static int minTrimPeriodMilliseconds = int.MaxValue;
         private static readonly List<WeakReference<UniformUnmanagedMemoryPool>> AllPools = new();

src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs

@@ -31,7 +31,7 @@ namespace SixLabors.ImageSharp.Memory.Internals
             this.lifetimeGuard = lifetimeGuard;
         }
 
-        private void* Pointer => this.lifetimeGuard.Handle.Pointer;
+        public void* Pointer => this.lifetimeGuard.Handle.Pointer;
 
         public override Span<T> GetSpan()
         {

src/ImageSharp/Memory/Buffer2D{T}.cs

@@ -97,10 +97,12 @@ namespace SixLabors.ImageSharp.Memory
         [MethodImpl(InliningOptions.ShortMethod)]
         public Span<T> DangerousGetRowSpan(int y)
         {
-            DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y));
+            if (y < 0 || y >= this.Height)
-            DebugGuard.MustBeLessThan(y, this.Height, nameof(y));
+            {
+                this.ThrowYOutOfRangeException(y);
+            }
 
-            return this.GetRowMemoryCore(y).Span;
+            return this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width);
         }
 
         internal bool TryGetPaddedRowSpan(int y, int padding, out Span<T> paddedSpan)
@@ -125,7 +127,7 @@ namespace SixLabors.ImageSharp.Memory
         [MethodImpl(InliningOptions.ShortMethod)]
         internal ref T GetElementUnsafe(int x, int y)
         {
-            Span<T> span = this.GetRowMemoryCore(y).Span;
+            Span<T> span = this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width);
             return ref span[x];
         }
 
@@ -139,7 +141,7 @@ namespace SixLabors.ImageSharp.Memory
         {
             DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y));
             DebugGuard.MustBeLessThan(y, this.Height, nameof(y));
-            return this.FastMemoryGroup.View.GetBoundedSlice(y * (long)this.Width, this.Width);
+            return this.FastMemoryGroup.View.GetBoundedMemorySlice(y * (long)this.Width, this.Width);
         }
 
         /// <summary>
@@ -195,7 +197,9 @@ namespace SixLabors.ImageSharp.Memory
             return swapped;
         }
 
-        [MethodImpl(InliningOptions.ShortMethod)]
+        [MethodImpl(InliningOptions.ColdPath)]
-        private Memory<T> GetRowMemoryCore(int y) => this.FastMemoryGroup.GetBoundedSlice(y * (long)this.Width, this.Width);
+        private void ThrowYOutOfRangeException(int y) =>
+            throw new ArgumentOutOfRangeException(
+                $"DangerousGetRowSpan({y}). Y was out of range. Height={this.Height}");
     }
 }

src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs

@@ -29,7 +29,7 @@ namespace SixLabors.ImageSharp.Memory
         /// Returns a slice that is expected to be within the bounds of a single buffer.
         /// Otherwise <see cref="ArgumentOutOfRangeException"/> is thrown.
         /// </summary>
-        internal static Memory<T> GetBoundedSlice<T>(this IMemoryGroup<T> group, long start, int length)
+        internal static Memory<T> GetBoundedMemorySlice<T>(this IMemoryGroup<T> group, long start, int length)
             where T : struct
         {
             Guard.NotNull(group, nameof(group));

src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Six Labors.
+// Licensed under the Apache License, Version 2.0.
+
+using System.Buffers;
+using SixLabors.ImageSharp.Memory.Internals;
+
+namespace SixLabors.ImageSharp.Memory
+{
+    internal unsafe struct MemoryGroupSpanCache
+    {
+        public SpanCacheMode Mode;
+        public byte[] SingleArray;
+        public void* SinglePointer;
+        public void*[] MultiPointer;
+
+        public static MemoryGroupSpanCache Create<T>(IMemoryOwner<T>[] memoryOwners)
+            where T : struct
+        {
+            IMemoryOwner<T> owner0 = memoryOwners[0];
+            MemoryGroupSpanCache memoryGroupSpanCache = default;
+            if (memoryOwners.Length == 1)
+            {
+                if (owner0 is SharedArrayPoolBuffer<T> sharedPoolBuffer)
+                {
+                    memoryGroupSpanCache.Mode = SpanCacheMode.SingleArray;
+                    memoryGroupSpanCache.SingleArray = sharedPoolBuffer.Array;
+                }
+                else if (owner0 is UnmanagedBuffer<T> unmanagedBuffer)
+                {
+                    memoryGroupSpanCache.Mode = SpanCacheMode.SinglePointer;
+                    memoryGroupSpanCache.SinglePointer = unmanagedBuffer.Pointer;
+                }
+            }
+            else
+            {
+                if (owner0 is UnmanagedBuffer<T>)
+                {
+                    memoryGroupSpanCache.Mode = SpanCacheMode.MultiPointer;
+                    memoryGroupSpanCache.MultiPointer = new void*[memoryOwners.Length];
+                    for (int i = 0; i < memoryOwners.Length; i++)
+                    {
+                        memoryGroupSpanCache.MultiPointer[i] = ((UnmanagedBuffer<T>)memoryOwners[i]).Pointer;
+                    }
+                }
+            }
+
+            return memoryGroupSpanCache;
+        }
+    }
+}

src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs

@@ -26,6 +26,7 @@ namespace SixLabors.ImageSharp.Memory
                 this.memoryOwners = memoryOwners;
                 this.Swappable = swappable;
                 this.View = new MemoryGroupView<T>(this);
+                this.memoryGroupSpanCache = MemoryGroupSpanCache.Create(memoryOwners);
             }
 
             public Owned(
@@ -173,32 +174,6 @@ namespace SixLabors.ImageSharp.Memory
             [MethodImpl(MethodImplOptions.NoInlining)]
             private static void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(MemoryGroup<T>));
 
-            internal static void SwapContents(Owned a, Owned b)
-            {
-                a.EnsureNotDisposed();
-                b.EnsureNotDisposed();
-
-                IMemoryOwner<T>[] tempOwners = a.memoryOwners;
-                long tempTotalLength = a.TotalLength;
-                int tempBufferLength = a.BufferLength;
-                RefCountedLifetimeGuard tempGroupOwner = a.groupLifetimeGuard;
-
-                a.memoryOwners = b.memoryOwners;
-                a.TotalLength = b.TotalLength;
-                a.BufferLength = b.BufferLength;
-                a.groupLifetimeGuard = b.groupLifetimeGuard;
-
-                b.memoryOwners = tempOwners;
-                b.TotalLength = tempTotalLength;
-                b.BufferLength = tempBufferLength;
-                b.groupLifetimeGuard = tempGroupOwner;
-
-                a.View.Invalidate();
-                b.View.Invalidate();
-                a.View = new MemoryGroupView<T>(a);
-                b.View = new MemoryGroupView<T>(b);
-            }
-
             // When the MemoryGroup points to multiple buffers via `groupLifetimeGuard`,
             // the lifetime of the individual buffers is managed by the guard.
             // Group buffer IMemoryOwner<T>-s d not manage ownership.

src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs

@@ -6,6 +6,8 @@ using System.Buffers;
 using System.Collections;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading;
 using SixLabors.ImageSharp.Memory.Internals;
 
 namespace SixLabors.ImageSharp.Memory
@@ -21,6 +23,8 @@ namespace SixLabors.ImageSharp.Memory
     {
         private static readonly int ElementSize = Unsafe.SizeOf<T>();
 
+        private MemoryGroupSpanCache memoryGroupSpanCache;
+
         private MemoryGroup(int bufferLength, long totalLength)
         {
             this.BufferLength = bufferLength;
@@ -31,10 +35,10 @@ namespace SixLabors.ImageSharp.Memory
         public abstract int Count { get; }
 
         /// <inheritdoc />
-        public int BufferLength { get; private set; }
+        public int BufferLength { get; }
 
         /// <inheritdoc />
-        public long TotalLength { get; private set; }
+        public long TotalLength { get; }
 
         /// <inheritdoc />
         public bool IsValid { get; private set; } = true;
@@ -241,6 +245,40 @@ namespace SixLabors.ImageSharp.Memory
             return new Owned(source, bufferLength, totalLength, false);
         }
 
+        [MethodImpl(InliningOptions.ShortMethod)]
+        public unsafe Span<T> GetRowSpanCoreUnsafe(int y, int width)
+        {
+            switch (this.memoryGroupSpanCache.Mode)
+            {
+                case SpanCacheMode.SingleArray:
+                {
+                    ref byte b0 = ref MemoryMarshal.GetReference<byte>(this.memoryGroupSpanCache.SingleArray);
+                    ref T e0 = ref Unsafe.As<byte, T>(ref b0);
+                    e0 = ref Unsafe.Add(ref e0, y * width);
+                    return MemoryMarshal.CreateSpan(ref e0, width);
+                }
+
+                case SpanCacheMode.SinglePointer:
+                {
+                    void* start = Unsafe.Add<T>(this.memoryGroupSpanCache.SinglePointer, y * width);
+                    return new Span<T>(start, width);
+                }
+
+                case SpanCacheMode.MultiPointer:
+                {
+                    this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart);
+                    void* start = Unsafe.Add<T>(this.memoryGroupSpanCache.MultiPointer[bufferIdx], bufferStart);
+                    return new Span<T>(start, width);
+                }
+
+                default:
+                {
+                    this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart);
+                    return this[bufferIdx].Span.Slice(bufferStart, width);
+                }
+            }
+        }
+
         public static bool CanSwapContent(MemoryGroup<T> target, MemoryGroup<T> source) =>
             source is Owned { Swappable: true } && target is Owned { Swappable: true };
 
@@ -255,5 +293,13 @@ namespace SixLabors.ImageSharp.Memory
         public virtual void DecreaseRefCounts()
         {
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void GetMultiBufferPosition(int y, int width, out int bufferIdx, out int bufferStart)
+        {
+            long start = y * (long)width;
+            bufferIdx = (int)(start / this.BufferLength);
+            bufferStart = (int)(start % this.BufferLength);
+        }
     }
 }

src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Six Labors.
+// Licensed under the Apache License, Version 2.0.
+
+namespace SixLabors.ImageSharp.Memory
+{
+    internal enum SpanCacheMode
+    {
+        Default = default,
+        SingleArray,
+        SinglePointer,
+        MultiPointer
+    }
+}

tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors.
 // Licensed under the Apache License, Version 2.0.
 
+using System.Runtime.CompilerServices;
 using BenchmarkDotNet.Attributes;
 using SixLabors.ImageSharp.Memory;
 using SixLabors.ImageSharp.PixelFormats;
@@ -9,18 +10,21 @@ namespace SixLabors.ImageSharp.Benchmarks.General
 {
     public class Buffer2D_DangerousGetRowSpan
     {
-        [Params(true, false)]
+        private const int Height = 1024;
-        public bool IsDiscontiguousBuffer { get; set; }
+
+        [Params(0.5, 2.0, 10.0)]
+        public double SizeMegaBytes { get; set; }
 
         private Buffer2D<Rgba32> buffer;
 
         [GlobalSetup]
-        public void Setup()
+        public unsafe void Setup()
         {
+            int totalElements = (int)(1024 * 1024 * this.SizeMegaBytes) / sizeof(Rgba32);
+
+            int width = totalElements / Height;
             MemoryAllocator allocator = Configuration.Default.MemoryAllocator;
-            this.buffer = this.IsDiscontiguousBuffer
+            this.buffer = allocator.Allocate2D<Rgba32>(width, Height);
-                ? allocator.Allocate2D<Rgba32>(4000, 1000)
-                : allocator.Allocate2D<Rgba32>(500, 1000);
         }
 
         [GlobalCleanup]
@@ -29,7 +33,7 @@ namespace SixLabors.ImageSharp.Benchmarks.General
         [Benchmark]
         public int DangerousGetRowSpan() =>
             this.buffer.DangerousGetRowSpan(1).Length +
-            this.buffer.DangerousGetRowSpan(999).Length;
+            this.buffer.DangerousGetRowSpan(Height - 1).Length;
 
         // BenchmarkDotNet=v0.13.0, OS=Windows 10.0.19044
         // Intel Core i9-10900X CPU 3.70GHz, 1 CPU, 20 logical and 10 physical cores

tests/ImageSharp.Tests/Memory/Buffer2DTests.cs

@@ -120,7 +120,7 @@ namespace SixLabors.ImageSharp.Tests.Memory
         [InlineData(200, 100, 30, 1, 0)]
         [InlineData(200, 100, 30, 2, 1)]
         [InlineData(200, 100, 30, 4, 2)]
-        public unsafe void GetRowSpanY(int bufferCapacity, int width, int height, int y, int expectedBufferIndex)
+        public unsafe void DangerousGetRowSpan_TestAllocator(int bufferCapacity, int width, int height, int y, int expectedBufferIndex)
         {
             this.MemoryAllocator.BufferCapacityInBytes = sizeof(TestStructs.Foo) * bufferCapacity;
 
@@ -135,6 +135,57 @@ namespace SixLabors.ImageSharp.Tests.Memory
             }
         }
 
+        [Theory]
+        [InlineData(100, 5)] // Within shared pool
+        [InlineData(77, 11)] // Within shared pool
+        [InlineData(100, 19)] // Single unmanaged pooled buffer
+        [InlineData(103, 17)] // Single unmanaged pooled buffer
+        [InlineData(100, 22)] // 2 unmanaged pooled buffers
+        [InlineData(100, 99)] // 9 unmanaged pooled buffers
+        [InlineData(100, 120)] // 2 unpooled buffers
+        public unsafe void DangerousGetRowSpan_UnmanagedAllocator(int width, int height)
+        {
+            const int sharedPoolThreshold = 1_000;
+            const int poolBufferSize = 2_000;
+            const int maxPoolSize = 10_000;
+            const int unpooledBufferSize = 8_000;
+
+            int elementSize = sizeof(TestStructs.Foo);
+            var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(
+                sharedPoolThreshold * elementSize,
+                poolBufferSize * elementSize,
+                maxPoolSize * elementSize,
+                unpooledBufferSize * elementSize);
+
+            using Buffer2D<TestStructs.Foo> buffer = allocator.Allocate2D<TestStructs.Foo>(width, height);
+
+            var rnd = new Random(42);
+
+            for (int y = 0; y < buffer.Height; y++)
+            {
+                Span<TestStructs.Foo> span = buffer.DangerousGetRowSpan(y);
+                for (int x = 0; x < span.Length; x++)
+                {
+                    ref TestStructs.Foo e = ref span[x];
+                    e.A = rnd.Next();
+                    e.B = rnd.NextDouble();
+                }
+            }
+
+            // Re-seed
+            rnd = new Random(42);
+            for (int y = 0; y < buffer.Height; y++)
+            {
+                Span<TestStructs.Foo> span = buffer.GetSafeRowMemory(y).Span;
+                for (int x = 0; x < span.Length; x++)
+                {
+                    ref TestStructs.Foo e = ref span[x];
+                    Assert.True(rnd.Next() == e.A, $"Mismatch @ y={y} x={x}");
+                    Assert.True(rnd.NextDouble() == e.B, $"Mismatch @ y={y} x={x}");
+                }
+            }
+        }
+
         [Theory]
         [InlineData(10, 0, 0, 0)]
         [InlineData(10, 0, 2, 0)]

tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs

@@ -146,7 +146,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers
         {
             using MemoryGroup<int> group = this.CreateTestGroup(totalLength, bufferLength, true);
 
-            Memory<int> slice = group.GetBoundedSlice(start, length);
+            Memory<int> slice = group.GetBoundedMemorySlice(start, length);
 
             Assert.Equal(length, slice.Length);
 
@@ -172,7 +172,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers
         public void GetBoundedSlice_WhenOverlapsBuffers_Throws(long totalLength, int bufferLength, long start, int length)
         {
             using MemoryGroup<int> group = this.CreateTestGroup(totalLength, bufferLength, true);
-            Assert.ThrowsAny<ArgumentOutOfRangeException>(() => group.GetBoundedSlice(start, length));
+            Assert.ThrowsAny<ArgumentOutOfRangeException>(() => group.GetBoundedMemorySlice(start, length));
         }
 
         [Fact]