retry allocation on OOM

src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs

@@ -26,7 +26,7 @@ namespace SixLabors.ImageSharp.Memory.Internals
         public UnmanagedBuffer(int lengthInElements)
         {
             this.lengthInElements = lengthInElements;
-            this.bufferHandle = new UnmanagedMemoryHandle(lengthInElements * Unsafe.SizeOf<T>());
+            this.bufferHandle = UnmanagedMemoryHandle.Allocate(lengthInElements * Unsafe.SizeOf<T>());
         }
 
         private void* Pointer => (void*)this.bufferHandle.DangerousGetHandle();

src/ImageSharp/Memory/Allocators/Internals/UnmanagedMemoryHandle.cs

@@ -10,16 +10,23 @@ namespace SixLabors.ImageSharp.Memory.Internals
 {
     internal sealed class UnmanagedMemoryHandle : SafeHandle
     {
+        // Number of allocation re-attempts when OutOfMemoryException is thrown.
+        private const int MaxAllocationAttempts = 1000;
+
         private readonly int lengthInBytes;
         private bool resurrected;
 
         // Track allocations for testing purposes:
         private static int totalOutstandingHandles;
 
-        public UnmanagedMemoryHandle(int lengthInBytes)
+        private static long totalOomRetries;
-            : base(IntPtr.Zero, true)
+
+        // A Monitor to wait/signal when we are low on memory.
+        private static object lowMemoryMonitor;
+
+        private UnmanagedMemoryHandle(IntPtr handle, int lengthInBytes)
+            : base(handle, true)
         {
-            this.SetHandle(Marshal.AllocHGlobal(lengthInBytes));
             this.lengthInBytes = lengthInBytes;
             if (lengthInBytes > 0)
             {
@@ -30,10 +37,15 @@ namespace SixLabors.ImageSharp.Memory.Internals
         }
 
         /// <summary>
-        /// Gets a value indicating the total outstanding handle allocations for testing purposes.
+        /// Gets the total outstanding handle allocations for testing purposes.
         /// </summary>
         internal static int TotalOutstandingHandles => totalOutstandingHandles;
 
+        /// <summary>
+        /// Gets the total number <see cref="OutOfMemoryException"/>-s retried.
+        /// </summary>
+        internal static long TotalOomRetries => totalOomRetries;
+
         /// <inheritdoc />
         public override bool IsInvalid => this.handle == IntPtr.Zero;
 
@@ -50,11 +62,59 @@ namespace SixLabors.ImageSharp.Memory.Internals
                 GC.RemoveMemoryPressure(this.lengthInBytes);
             }
 
+            if (lowMemoryMonitor != null)
+            {
+                // We are low on memory. Signal all threads waiting in AllocateHandle().
+                Monitor.Enter(lowMemoryMonitor);
+                Monitor.PulseAll(lowMemoryMonitor);
+                Monitor.Exit(lowMemoryMonitor);
+            }
+
             this.handle = IntPtr.Zero;
             Interlocked.Decrement(ref totalOutstandingHandles);
             return true;
         }
 
+        internal static UnmanagedMemoryHandle Allocate(int lengthInBytes)
+        {
+            IntPtr handle = AllocateHandle(lengthInBytes);
+            return new UnmanagedMemoryHandle(handle, lengthInBytes);
+        }
+
+        private static IntPtr AllocateHandle(int lengthInBytes)
+        {
+            int counter = 0;
+            IntPtr handle = IntPtr.Zero;
+            while (handle == IntPtr.Zero)
+            {
+                try
+                {
+                    handle = Marshal.AllocHGlobal(lengthInBytes);
+                }
+                catch (OutOfMemoryException)
+                {
+                    // We are low on memory, but expect some memory to be freed soon.
+                    // Block the thread & retry to avoid OOM.
+                    if (counter < MaxAllocationAttempts)
+                    {
+                        counter++;
+                        Interlocked.Increment(ref totalOomRetries);
+
+                        Interlocked.CompareExchange(ref lowMemoryMonitor, new object(), null);
+                        Monitor.Enter(lowMemoryMonitor);
+                        Monitor.Wait(lowMemoryMonitor, millisecondsTimeout: 1);
+                        Monitor.Exit(lowMemoryMonitor);
+                    }
+                    else
+                    {
+                        throw;
+                    }
+                }
+            }
+
+            return handle;
+        }
+
         /// <summary>
         /// UnmanagedMemoryHandle's finalizer would release the underlying handle returning the memory to the OS.
         /// We want to prevent this when a finalizable owner (buffer or MemoryGroup) is returning the handle to

tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs

@@ -82,7 +82,7 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox
             }
 
             var stats = new Stats(timer, lrs.Benchmarks.TotalProcessedMegapixels);
-            Console.WriteLine("Total Megapixels: " + stats.TotalMegapixels);
+            Console.WriteLine($"Total Megapixels: {stats.TotalMegapixels}, TotalOomRetries: {UnmanagedMemoryHandle.TotalOomRetries}");
             Console.WriteLine(stats.GetMarkdown());
             if (options?.FileOutput != null)
             {

tests/ImageSharp.Tests/Memory/Allocators/UnmanagedMemoryHandleTests.cs

@@ -12,9 +12,9 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
     public class UnmanagedMemoryHandleTests
     {
         [Fact]
-        public unsafe void Constructor_AllocatesReadWriteMemory()
+        public unsafe void Allocate_AllocatesReadWriteMemory()
         {
-            using var h = new UnmanagedMemoryHandle(128);
+            using var h = UnmanagedMemoryHandle.Allocate(128);
             Assert.False(h.IsClosed);
             Assert.False(h.IsInvalid);
             byte* ptr = (byte*)h.DangerousGetHandle();
@@ -32,7 +32,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
         [Fact]
         public void Dispose_ClosesHandle()
         {
-            var h = new UnmanagedMemoryHandle(128);
+            var h = UnmanagedMemoryHandle.Allocate(128);
             h.Dispose();
             Assert.True(h.IsClosed);
             Assert.True(h.IsInvalid);
@@ -52,7 +52,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
                 for (int i = 0; i < countInner; i++)
                 {
                     Assert.Equal(i, UnmanagedMemoryHandle.TotalOutstandingHandles);
-                    var h = new UnmanagedMemoryHandle(42);
+                    var h = UnmanagedMemoryHandle.Allocate(42);
                     Assert.Equal(i + 1, UnmanagedMemoryHandle.TotalOutstandingHandles);
                     l.Add(h);
                 }
@@ -92,7 +92,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
                 var l = new List<UnmanagedMemoryHandle>();
                 for (int i = 0; i < countInner; i++)
                 {
-                    var h = new UnmanagedMemoryHandle(42);
+                    var h = UnmanagedMemoryHandle.Allocate(42);
                     l.Add(h);
                 }
 
@@ -119,7 +119,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
 
             static void AllocateResurrect()
             {
-                var h = new UnmanagedMemoryHandle(42);
+                var h = UnmanagedMemoryHandle.Allocate(42);
                 h.Resurrect();
             }
         }
@@ -162,7 +162,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators
 
             static void AllocateAndForget()
             {
-                _ = new HandleOwner(new UnmanagedMemoryHandle(42));
+                _ = new HandleOwner(UnmanagedMemoryHandle.Allocate(42));
             }
 
             static void VerifyResurrectedHandle(bool reAssign)