diff --git a/Directory.Build.props b/Directory.Build.props index b3e18e5a5a..4a0fb856da 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -26,4 +26,7 @@ true + + $(DefineConstants);NETCORE31COMPATIBLE + diff --git a/src/ImageSharp/Advanced/AotCompilerTools.cs b/src/ImageSharp/Advanced/AotCompilerTools.cs index be2e964fc0..74b8a85b0e 100644 --- a/src/ImageSharp/Advanced/AotCompilerTools.cs +++ b/src/ImageSharp/Advanced/AotCompilerTools.cs @@ -524,7 +524,7 @@ namespace SixLabors.ImageSharp.Advanced private static void AotCompileMemoryManagers() where TPixel : unmanaged, IPixel { - AotCompileMemoryManager(); + AotCompileMemoryManager(); AotCompileMemoryManager(); } diff --git a/src/ImageSharp/Configuration.cs b/src/ImageSharp/Configuration.cs index 49b7aa79b0..0ac4c29eae 100644 --- a/src/ImageSharp/Configuration.cs +++ b/src/ImageSharp/Configuration.cs @@ -118,7 +118,7 @@ namespace SixLabors.ImageSharp /// /// Gets or sets the that is currently in use. /// - public MemoryAllocator MemoryAllocator { get; set; } = ArrayPoolMemoryAllocator.CreateDefault(); + public MemoryAllocator MemoryAllocator { get; set; } = MemoryAllocator.CreateDefault(); /// /// Gets the maximum header size of all the formats. diff --git a/src/ImageSharp/ImageSharp.csproj b/src/ImageSharp/ImageSharp.csproj index 7719b12420..9e80518921 100644 --- a/src/ImageSharp/ImageSharp.csproj +++ b/src/ImageSharp/ImageSharp.csproj @@ -13,8 +13,8 @@ Image Resize Crop Gif Jpg Jpeg Bitmap Png Tga NetCore A new, fully featured, fully managed, cross-platform, 2D graphics API for .NET Debug;Release;Debug-InnerLoop;Release-InnerLoop + $(DefineConstants);DEBUG - diff --git a/src/ImageSharp/Memory/Allocators/AllocationOptions.cs b/src/ImageSharp/Memory/Allocators/AllocationOptions.cs index 3c865f3578..72c785532b 100644 --- a/src/ImageSharp/Memory/Allocators/AllocationOptions.cs +++ b/src/ImageSharp/Memory/Allocators/AllocationOptions.cs @@ -1,21 +1,30 @@ // Copyright (c) Six Labors. // Licensed under the Apache License, Version 2.0. +using System; + namespace SixLabors.ImageSharp.Memory { /// /// Options for allocating buffers. /// + [Flags] public enum AllocationOptions { /// /// Indicates that the buffer should just be allocated. /// - None, + None = 0, /// /// Indicates that the allocated buffer should be cleaned following allocation. /// - Clean + Clean = 1, + + /// + /// Affects only group allocations. + /// Indicates that the requested or should be made of contiguous blocks up to . + /// + Contiguous = 2 } } diff --git a/src/ImageSharp/Memory/Allocators/AllocationOptionsExtensions.cs b/src/ImageSharp/Memory/Allocators/AllocationOptionsExtensions.cs new file mode 100644 index 0000000000..d3e5bca6ee --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/AllocationOptionsExtensions.cs @@ -0,0 +1,10 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +namespace SixLabors.ImageSharp.Memory +{ + internal static class AllocationOptionsExtensions + { + public static bool Has(this AllocationOptions options, AllocationOptions flag) => (options & flag) == flag; + } +} diff --git a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.Buffer{T}.cs b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.Buffer{T}.cs index 0c35c88286..5200a2793c 100644 --- a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.Buffer{T}.cs +++ b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.Buffer{T}.cs @@ -10,7 +10,7 @@ using SixLabors.ImageSharp.Memory.Internals; namespace SixLabors.ImageSharp.Memory { /// - /// Contains and . + /// Contains . /// public partial class ArrayPoolMemoryAllocator { @@ -87,19 +87,5 @@ namespace SixLabors.ImageSharp.Memory throw new ObjectDisposedException("ArrayPoolMemoryAllocator.Buffer"); } } - - /// - /// The implementation of . - /// - private sealed class ManagedByteBuffer : Buffer, IManagedByteBuffer - { - public ManagedByteBuffer(byte[] data, int length, ArrayPool sourcePool) - : base(data, length, sourcePool) - { - } - - /// - public byte[] Array => this.Data; - } } } diff --git a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.CommonFactoryMethods.cs b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.CommonFactoryMethods.cs index 8aa1b90634..4b06f537b9 100644 --- a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.CommonFactoryMethods.cs +++ b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.CommonFactoryMethods.cs @@ -36,7 +36,7 @@ namespace SixLabors.ImageSharp.Memory /// This is the default. Should be good for most use cases. /// /// The memory manager. - public static ArrayPoolMemoryAllocator CreateDefault() + public static new ArrayPoolMemoryAllocator CreateDefault() { return new ArrayPoolMemoryAllocator( DefaultMaxPooledBufferSizeInBytes, diff --git a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.cs index a79e042a32..eb34b813fe 100644 --- a/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.cs +++ b/src/ImageSharp/Memory/Allocators/ArrayPoolMemoryAllocator.cs @@ -10,6 +10,7 @@ namespace SixLabors.ImageSharp.Memory /// /// Implements by allocating memory from . /// + [Obsolete("ArrayPoolMemoryAllocator is obsolete. Use MemoryAllocator.CreateDefault() instead.")] public sealed partial class ArrayPoolMemoryAllocator : MemoryAllocator { private readonly int maxArraysPerBucketNormalPool; @@ -136,7 +137,7 @@ namespace SixLabors.ImageSharp.Memory byte[] byteArray = pool.Rent(bufferSizeInBytes); var buffer = new Buffer(byteArray, length, pool); - if (options == AllocationOptions.Clean) + if (options.Has(AllocationOptions.Clean)) { buffer.GetSpan().Clear(); } @@ -144,27 +145,7 @@ namespace SixLabors.ImageSharp.Memory return buffer; } - /// - public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None) - { - Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); - - ArrayPool pool = this.GetArrayPool(length); - byte[] byteArray = pool.Rent(length); - - var buffer = new ManagedByteBuffer(byteArray, length, pool); - if (options == AllocationOptions.Clean) - { - buffer.GetSpan().Clear(); - } - - return buffer; - } - - private static int GetLargeBufferThresholdInBytes(int maxPoolSizeInBytes) - { - return maxPoolSizeInBytes / 4; - } + private static int GetLargeBufferThresholdInBytes(int maxPoolSizeInBytes) => maxPoolSizeInBytes / 4; [MethodImpl(InliningOptions.ColdPath)] private static void ThrowInvalidAllocationException(int length, int max) => diff --git a/src/ImageSharp/Memory/Allocators/IManagedByteBuffer.cs b/src/ImageSharp/Memory/Allocators/IManagedByteBuffer.cs deleted file mode 100644 index 8088a2c47c..0000000000 --- a/src/ImageSharp/Memory/Allocators/IManagedByteBuffer.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Apache License, Version 2.0. - -using System.Buffers; - -namespace SixLabors.ImageSharp.Memory -{ - /// - /// Represents a byte buffer backed by a managed array. Useful for interop with classic .NET API-s. - /// - public interface IManagedByteBuffer : IMemoryOwner - { - /// - /// Gets the managed array backing this buffer instance. - /// - byte[] Array { get; } - } -} \ No newline at end of file diff --git a/src/ImageSharp/Memory/Allocators/Internals/BasicArrayBuffer.cs b/src/ImageSharp/Memory/Allocators/Internals/BasicArrayBuffer.cs index 3a3c695b2c..7dbbabff3a 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/BasicArrayBuffer.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/BasicArrayBuffer.cs @@ -2,11 +2,12 @@ // Licensed under the Apache License, Version 2.0. using System; +using System.Buffers; namespace SixLabors.ImageSharp.Memory.Internals { /// - /// Wraps an array as an instance. + /// Wraps an array as an instance. /// /// internal class BasicArrayBuffer : ManagedBufferBase @@ -57,4 +58,4 @@ namespace SixLabors.ImageSharp.Memory.Internals return this.Array; } } -} \ No newline at end of file +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/BasicByteBuffer.cs b/src/ImageSharp/Memory/Allocators/Internals/BasicByteBuffer.cs deleted file mode 100644 index 499a9228c1..0000000000 --- a/src/ImageSharp/Memory/Allocators/Internals/BasicByteBuffer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Apache License, Version 2.0. - -namespace SixLabors.ImageSharp.Memory.Internals -{ - /// - /// Provides an based on . - /// - internal sealed class BasicByteBuffer : BasicArrayBuffer, IManagedByteBuffer - { - /// - /// Initializes a new instance of the class. - /// - /// The byte array. - internal BasicByteBuffer(byte[] array) - : base(array) - { - } - } -} \ No newline at end of file diff --git a/src/ImageSharp/Memory/Allocators/Internals/Gen2GcCallback.cs b/src/ImageSharp/Memory/Allocators/Internals/Gen2GcCallback.cs new file mode 100644 index 0000000000..3a0479359a --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/Gen2GcCallback.cs @@ -0,0 +1,115 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +// Port of BCL internal utility: +// https://github.com/dotnet/runtime/blob/57bfe474518ab5b7cfe6bf7424a79ce3af9d6657/src/libraries/System.Private.CoreLib/src/System/Gen2GcCallback.cs +#if NETCORE31COMPATIBLE +using System; +using System.Runtime.ConstrainedExecution; +using System.Runtime.InteropServices; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + /// + /// Schedules a callback roughly every gen 2 GC (you may see a Gen 0 an Gen 1 but only once) + /// (We can fix this by capturing the Gen 2 count at startup and testing, but I mostly don't care) + /// + internal sealed class Gen2GcCallback : CriticalFinalizerObject + { + private readonly Func callback0; + private readonly Func callback1; + private GCHandle weakTargetObj; + + private Gen2GcCallback(Func callback) + { + this.callback0 = callback; + } + + private Gen2GcCallback(Func callback, object targetObj) + { + this.callback1 = callback; + this.weakTargetObj = GCHandle.Alloc(targetObj, GCHandleType.Weak); + } + + ~Gen2GcCallback() + { + if (this.weakTargetObj.IsAllocated) + { + // Check to see if the target object is still alive. + object targetObj = this.weakTargetObj.Target; + if (targetObj == null) + { + // The target object is dead, so this callback object is no longer needed. + this.weakTargetObj.Free(); + return; + } + + // Execute the callback method. + try + { + if (!this.callback1(targetObj)) + { + // If the callback returns false, this callback object is no longer needed. + this.weakTargetObj.Free(); + return; + } + } + catch + { + // Ensure that we still get a chance to resurrect this object, even if the callback throws an exception. +#if DEBUG + // Except in DEBUG, as we really shouldn't be hitting any exceptions here. + throw; +#endif + } + } + else + { + // Execute the callback method. + try + { + if (!this.callback0()) + { + // If the callback returns false, this callback object is no longer needed. + return; + } + } + catch + { + // Ensure that we still get a chance to resurrect this object, even if the callback throws an exception. +#if DEBUG + // Except in DEBUG, as we really shouldn't be hitting any exceptions here. + throw; +#endif + } + } + + // Resurrect ourselves by re-registering for finalization. + GC.ReRegisterForFinalize(this); + } + + /// + /// Schedule 'callback' to be called in the next GC. If the callback returns true it is + /// rescheduled for the next Gen 2 GC. Otherwise the callbacks stop. + /// + public static void Register(Func callback) + { + // Create a unreachable object that remembers the callback function and target object. + _ = new Gen2GcCallback(callback); + } + + /// + /// Schedule 'callback' to be called in the next GC. If the callback returns true it is + /// rescheduled for the next Gen 2 GC. Otherwise the callbacks stop. + /// + /// NOTE: This callback will be kept alive until either the callback function returns false, + /// or the target object dies. + /// + public static void Register(Func callback, object targetObj) + { + // Create a unreachable object that remembers the callback function and target object. + _ = new Gen2GcCallback(callback, targetObj); + } + } +} +#endif diff --git a/src/ImageSharp/Memory/Allocators/Internals/ManagedBufferBase.cs b/src/ImageSharp/Memory/Allocators/Internals/ManagedBufferBase.cs index 3f54e335e8..654c78647f 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/ManagedBufferBase.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/ManagedBufferBase.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. using System.Buffers; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace SixLabors.ImageSharp.Memory.Internals @@ -23,7 +24,7 @@ namespace SixLabors.ImageSharp.Memory.Internals this.pinHandle = GCHandle.Alloc(this.GetPinnableObject(), GCHandleType.Pinned); } - void* ptr = (void*)this.pinHandle.AddrOfPinnedObject(); + void* ptr = Unsafe.Add((void*)this.pinHandle.AddrOfPinnedObject(), elementIndex); return new MemoryHandle(ptr, this.pinHandle); } @@ -42,4 +43,4 @@ namespace SixLabors.ImageSharp.Memory.Internals /// The pinnable . protected abstract object GetPinnableObject(); } -} \ No newline at end of file +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs new file mode 100644 index 0000000000..c4fed8426d --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs @@ -0,0 +1,40 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + internal class SharedArrayPoolBuffer : ManagedBufferBase + where T : struct + { + private readonly int lengthInBytes; + private byte[] array; + + public SharedArrayPoolBuffer(int lengthInElements) + { + this.lengthInBytes = lengthInElements * Unsafe.SizeOf(); + this.array = ArrayPool.Shared.Rent(this.lengthInBytes); + } + + ~SharedArrayPoolBuffer() => this.Dispose(false); + + protected override void Dispose(bool disposing) + { + if (this.array == null) + { + return; + } + + ArrayPool.Shared.Return(this.array); + this.array = null; + } + + public override Span GetSpan() => MemoryMarshal.Cast(this.array.AsSpan(0, this.lengthInBytes)); + + protected override object GetPinnableObject() => this.array; + } +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.Buffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.Buffer{T}.cs new file mode 100644 index 0000000000..c315aa5484 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.Buffer{T}.cs @@ -0,0 +1,89 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + internal partial class UniformUnmanagedMemoryPool + { + public unsafe class Buffer : MemoryManager + where T : struct + { + private UniformUnmanagedMemoryPool pool; + private readonly int length; + + public Buffer(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle bufferHandle, int length) + { + this.pool = pool; + this.BufferHandle = bufferHandle; + this.length = length; + } + + private void* Pointer => (void*)this.BufferHandle.DangerousGetHandle(); + + protected UnmanagedMemoryHandle BufferHandle { get; private set; } + + public override Span GetSpan() => new Span(this.Pointer, this.length); + + /// + public override MemoryHandle Pin(int elementIndex = 0) + { + // Will be released in Unpin + bool unused = false; + this.BufferHandle.DangerousAddRef(ref unused); + + void* pbData = Unsafe.Add(this.Pointer, elementIndex); + return new MemoryHandle(pbData); + } + + /// + public override void Unpin() => this.BufferHandle.DangerousRelease(); + + /// + protected override void Dispose(bool disposing) + { + if (this.pool == null) + { + return; + } + + this.pool.Return(this.BufferHandle); + this.pool = null; + this.BufferHandle = null; + } + + internal void MarkDisposed() + { + this.pool = null; + this.BufferHandle = null; + } + } + + public class FinalizableBuffer : Buffer + where T : struct + { + public FinalizableBuffer(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle bufferHandle, int length) + : base(pool, bufferHandle, length) + { + bufferHandle.AssignedToNewOwner(); + } + + ~FinalizableBuffer() => this.Dispose(false); + + protected override void Dispose(bool disposing) + { + if (!disposing && this.BufferHandle != null) + { + // We need to prevent handle finalization here. + // See comments on UnmanagedMemoryHandle.Resurrect() + this.BufferHandle.Resurrect(); + } + + base.Dispose(disposing); + } + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs new file mode 100644 index 0000000000..c1a2d90bd5 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs @@ -0,0 +1,323 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Diagnostics; +using System.Threading; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + internal partial class UniformUnmanagedMemoryPool + { + private static readonly Stopwatch Stopwatch = Stopwatch.StartNew(); + + private readonly TrimSettings trimSettings; + private UnmanagedMemoryHandle[] buffers; + private int index; + private Timer trimTimer; + private long lastTrimTimestamp; + + public UniformUnmanagedMemoryPool(int bufferLength, int capacity) + : this(bufferLength, capacity, TrimSettings.Default) + { + } + + public UniformUnmanagedMemoryPool(int bufferLength, int capacity, TrimSettings trimSettings) + { + this.trimSettings = trimSettings; + this.Capacity = capacity; + this.BufferLength = bufferLength; + this.buffers = new UnmanagedMemoryHandle[capacity]; + + if (trimSettings.Enabled) + { + // Invoke the timer callback more frequently, than trimSettings.TrimPeriodMilliseconds, + // and also invoke it on Gen 2 GC. + // We are checking in the callback if enough time passed since the last trimming. If not, we do nothing. + this.trimTimer = new Timer( + s => ((UniformUnmanagedMemoryPool)s)?.Trim(), + this, + this.trimSettings.TrimPeriodMilliseconds / 4, + this.trimSettings.TrimPeriodMilliseconds / 4); + +#if NETCORE31COMPATIBLE + Gen2GcCallback.Register(s => ((UniformUnmanagedMemoryPool)s).Trim(), this); +#endif + } + } + + public int BufferLength { get; } + + public int Capacity { get; } + + public UnmanagedMemoryHandle Rent(AllocationOptions allocationOptions = AllocationOptions.None) + { + UnmanagedMemoryHandle[] buffersLocal = this.buffers; + + // Avoid taking the lock if the pool is released or is over limit: + if (buffersLocal == null || this.index == buffersLocal.Length) + { + return null; + } + + UnmanagedMemoryHandle array; + + lock (buffersLocal) + { + // Check again after taking the lock: + if (this.buffers == null || this.index == buffersLocal.Length) + { + return null; + } + + array = buffersLocal[this.index]; + buffersLocal[this.index++] = null; + } + + if (array == null) + { + array = new UnmanagedMemoryHandle(this.BufferLength); + } + + if (allocationOptions.Has(AllocationOptions.Clean)) + { + this.GetSpan(array).Clear(); + } + + return array; + } + + public UnmanagedMemoryHandle[] Rent(int bufferCount, AllocationOptions allocationOptions = AllocationOptions.None) + { + UnmanagedMemoryHandle[] buffersLocal = this.buffers; + + // Avoid taking the lock if the pool is released or is over limit: + if (buffersLocal == null || this.index + bufferCount >= buffersLocal.Length + 1) + { + return null; + } + + UnmanagedMemoryHandle[] result; + lock (buffersLocal) + { + // Check again after taking the lock: + if (this.buffers == null || this.index + bufferCount >= buffersLocal.Length + 1) + { + return null; + } + + result = new UnmanagedMemoryHandle[bufferCount]; + for (int i = 0; i < bufferCount; i++) + { + result[i] = buffersLocal[this.index]; + buffersLocal[this.index++] = null; + } + } + + for (int i = 0; i < result.Length; i++) + { + if (result[i] == null) + { + result[i] = new UnmanagedMemoryHandle(this.BufferLength); + } + + if (allocationOptions.Has(AllocationOptions.Clean)) + { + this.GetSpan(result[i]).Clear(); + } + } + + return result; + } + + public void Return(UnmanagedMemoryHandle buffer) + { + UnmanagedMemoryHandle[] buffersLocal = this.buffers; + if (buffersLocal == null) + { + buffer.Dispose(); + return; + } + + lock (buffersLocal) + { + // Check again after taking the lock: + if (this.buffers == null) + { + buffer.Dispose(); + return; + } + + if (this.index == 0) + { + ThrowReturnedMoreArraysThanRented(); // DEBUG-only exception + buffer.Dispose(); + return; + } + + this.buffers[--this.index] = buffer; + } + } + + public void Return(Span buffers) + { + UnmanagedMemoryHandle[] buffersLocal = this.buffers; + if (buffersLocal == null) + { + DisposeAll(buffers); + return; + } + + lock (buffersLocal) + { + // Check again after taking the lock: + if (this.buffers == null) + { + DisposeAll(buffers); + return; + } + + if (this.index - buffers.Length + 1 <= 0) + { + ThrowReturnedMoreArraysThanRented(); + DisposeAll(buffers); + return; + } + + for (int i = buffers.Length - 1; i >= 0; i--) + { + buffersLocal[--this.index] = buffers[i]; + } + } + } + + public void Release() + { + this.trimTimer?.Dispose(); + this.trimTimer = null; + UnmanagedMemoryHandle[] oldBuffers = Interlocked.Exchange(ref this.buffers, null); + DebugGuard.NotNull(oldBuffers, nameof(oldBuffers)); + DisposeAll(oldBuffers); + } + + private static void DisposeAll(Span buffers) + { + foreach (UnmanagedMemoryHandle handle in buffers) + { + handle?.Dispose(); + } + } + + private unsafe Span GetSpan(UnmanagedMemoryHandle h) => + new Span((byte*)h.DangerousGetHandle(), this.BufferLength); + + // This indicates a bug in the library, however Return() might be called from a finalizer, + // therefore we should never throw here in production. + [Conditional("DEBUG")] + private static void ThrowReturnedMoreArraysThanRented() => + throw new InvalidMemoryOperationException("Returned more arrays then rented"); + + private bool Trim() + { + UnmanagedMemoryHandle[] buffersLocal = this.buffers; + if (buffersLocal == null) + { + return false; + } + + bool isHighPressure = this.IsHighMemoryPressure(); + + if (isHighPressure) + { + return this.TrimHighPressure(buffersLocal); + } + + long millisecondsSinceLastTrim = Stopwatch.ElapsedMilliseconds - this.lastTrimTimestamp; + if (millisecondsSinceLastTrim > this.trimSettings.TrimPeriodMilliseconds) + { + return this.TrimLowPressure(buffersLocal); + } + + return true; + } + + private bool TrimHighPressure(UnmanagedMemoryHandle[] buffersLocal) + { + lock (buffersLocal) + { + if (this.buffers == null) + { + return false; + } + + // Trim all: + for (int i = this.index; i < buffersLocal.Length && buffersLocal[i] != null; i++) + { + buffersLocal[i] = null; + } + } + + return true; + } + + private bool TrimLowPressure(UnmanagedMemoryHandle[] arraysLocal) + { + lock (arraysLocal) + { + if (this.buffers == null) + { + return false; + } + + // Count the arrays in the pool: + int retainedCount = 0; + for (int i = this.index; i < arraysLocal.Length && arraysLocal[i] != null; i++) + { + retainedCount++; + } + + // Trim 'trimRate' of 'retainedCount': + int trimCount = (int)Math.Ceiling(retainedCount * this.trimSettings.Rate); + int trimStart = this.index + retainedCount - 1; + int trimStop = this.index + retainedCount - trimCount; + for (int i = trimStart; i >= trimStop; i--) + { + arraysLocal[i].Dispose(); + arraysLocal[i] = null; + } + + this.lastTrimTimestamp = Stopwatch.ElapsedMilliseconds; + } + + return true; + } + + private bool IsHighMemoryPressure() + { +#if NETCORE31COMPATIBLE + GCMemoryInfo memoryInfo = GC.GetGCMemoryInfo(); + return memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * this.trimSettings.HighPressureThresholdRate; +#else + // We don't have high pressure detection triggering full trimming on other platforms, + // to counterpart this, the maximum pool size is small. + return false; +#endif + } + + public class TrimSettings + { + // Trim half of the retained pool buffers every minute. + public int TrimPeriodMilliseconds { get; set; } = 60_000; + + public float Rate { get; set; } = 0.5f; + + // Be more strict about high pressure threshold than ArrayPool.Shared. + // A 32 bit process can OOM before reaching HighMemoryLoadThresholdBytes. + public float HighPressureThresholdRate { get; set; } = 0.5f; + + public bool Enabled => this.Rate > 0; + + public static TrimSettings Default => new TrimSettings(); + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs new file mode 100644 index 0000000000..03c5042794 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs @@ -0,0 +1,65 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + /// + /// Allocates and provides an implementation giving + /// access to unmanaged buffers allocated by . + /// + /// The element type. + internal sealed unsafe class UnmanagedBuffer : MemoryManager + where T : struct + { + private readonly UnmanagedMemoryHandle bufferHandle; + private readonly int lengthInElements; + + /// + /// Initializes a new instance of the class. + /// + /// The number of elements to allocate. + public UnmanagedBuffer(int lengthInElements) + { + this.lengthInElements = lengthInElements; + this.bufferHandle = new UnmanagedMemoryHandle(lengthInElements * Unsafe.SizeOf()); + } + + private void* Pointer => (void*)this.bufferHandle.DangerousGetHandle(); + + public override Span GetSpan() + => new Span(this.Pointer, this.lengthInElements); + + /// + public override MemoryHandle Pin(int elementIndex = 0) + { + // Will be released in Unpin + bool unused = false; + this.bufferHandle.DangerousAddRef(ref unused); + + void* pbData = Unsafe.Add(this.Pointer, elementIndex); + return new MemoryHandle(pbData); + } + + /// + public override void Unpin() => this.bufferHandle.DangerousRelease(); + + /// + protected override void Dispose(bool disposing) + { + if (this.bufferHandle.IsInvalid) + { + return; + } + + if (disposing) + { + this.bufferHandle.Dispose(); + } + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedMemoryHandle.cs b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedMemoryHandle.cs new file mode 100644 index 0000000000..216b526b7b --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedMemoryHandle.cs @@ -0,0 +1,80 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Win32.SafeHandles; + +namespace SixLabors.ImageSharp.Memory.Internals +{ + internal sealed class UnmanagedMemoryHandle : SafeHandle + { + private readonly int lengthInBytes; + private bool resurrected; + + // Track allocations for testing purposes: + private static int totalOutstandingHandles; + + public UnmanagedMemoryHandle(int lengthInBytes) + : base(IntPtr.Zero, true) + { + this.SetHandle(Marshal.AllocHGlobal(lengthInBytes)); + this.lengthInBytes = lengthInBytes; + if (lengthInBytes > 0) + { + GC.AddMemoryPressure(lengthInBytes); + } + + Interlocked.Increment(ref totalOutstandingHandles); + } + + /// + /// Gets a value indicating the total outstanding handle allocations for testing purposes. + /// + internal static int TotalOutstandingHandles => totalOutstandingHandles; + + /// + public override bool IsInvalid => this.handle == IntPtr.Zero; + + protected override bool ReleaseHandle() + { + if (this.IsInvalid) + { + return false; + } + + Marshal.FreeHGlobal(this.handle); + if (this.lengthInBytes > 0) + { + GC.RemoveMemoryPressure(this.lengthInBytes); + } + + this.handle = IntPtr.Zero; + Interlocked.Decrement(ref totalOutstandingHandles); + return true; + } + + /// + /// UnmanagedMemoryHandle's finalizer would release the underlying handle returning the memory to the OS. + /// We want to prevent this when a finalizable owner (buffer or MemoryGroup) is returning the handle to + /// in it's finalizer. + /// Since UnmanagedMemoryHandle is CriticalFinalizable, it is guaranteed that the owner's finalizer is called first. + /// + internal void Resurrect() + { + GC.SuppressFinalize(this); + this.resurrected = true; + } + + internal void AssignedToNewOwner() + { + if (this.resurrected) + { + // The handle has been resurrected + GC.ReRegisterForFinalize(this); + this.resurrected = false; + } + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs index af56b99a08..840e07d1ad 100644 --- a/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs +++ b/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs @@ -17,6 +17,21 @@ namespace SixLabors.ImageSharp.Memory /// The length of the largest contiguous buffer that can be handled by this allocator instance. protected internal abstract int GetBufferCapacityInBytes(); + /// + /// Creates a default instance of a optimized for the executing platform. + /// + /// The . + public static MemoryAllocator CreateDefault() => + new UniformUnmanagedMemoryPoolMemoryAllocator(null, null); + + /// + /// Creates the default using the provided options. + /// + /// The . + /// The . + public static MemoryAllocator CreateDefault(MemoryAllocatorOptions options) => + new UniformUnmanagedMemoryPoolMemoryAllocator(options.MaximumPoolSizeMegabytes, options.MinimumContiguousBlockBytes); + /// /// Allocates an , holding a of length . /// @@ -29,16 +44,6 @@ namespace SixLabors.ImageSharp.Memory public abstract IMemoryOwner Allocate(int length, AllocationOptions options = AllocationOptions.None) where T : struct; - /// - /// Allocates an . - /// - /// The requested buffer length. - /// The allocation options. - /// The . - /// When length is zero or negative. - /// When length is over the capacity of the allocator. - public abstract IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None); - /// /// Releases all retained resources not being in use. /// Eg: by resetting array pools and letting GC to free the arrays. @@ -46,5 +51,20 @@ namespace SixLabors.ImageSharp.Memory public virtual void ReleaseRetainedResources() { } + + /// + /// Allocates a . + /// + /// The total length of the buffer. + /// The expected alignment (eg. to make sure image rows fit into single buffers). + /// The . + /// A new . + /// Thrown when 'blockAlignment' converted to bytes is greater than the buffer capacity of the allocator. + internal virtual MemoryGroup AllocateGroup( + long totalLength, + int bufferAlignment, + AllocationOptions options = AllocationOptions.None) + where T : struct + => MemoryGroup.Allocate(this, totalLength, bufferAlignment, options); } } diff --git a/src/ImageSharp/Memory/Allocators/MemoryAllocatorOptions.cs b/src/ImageSharp/Memory/Allocators/MemoryAllocatorOptions.cs new file mode 100644 index 0000000000..8ea7f6f7b9 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/MemoryAllocatorOptions.cs @@ -0,0 +1,57 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +namespace SixLabors.ImageSharp.Memory +{ + /// + /// Defines options for creating the default . + /// + public class MemoryAllocatorOptions + { + private int? maximumPoolSizeMegabytes; + private int? minimumContiguousBlockBytes; + + /// + /// Gets or sets a value defining the maximum size of the 's internal memory pool + /// in Megabytes. means platform default. + /// + public int? MaximumPoolSizeMegabytes + { + get => this.maximumPoolSizeMegabytes; + set + { + if (value.HasValue) + { + Guard.MustBeGreaterThanOrEqualTo(value.Value, 0, nameof(this.MaximumPoolSizeMegabytes)); + } + + this.maximumPoolSizeMegabytes = value; + } + } + + /// + /// Gets or sets a value defining the minimum contiguous block size when allocating buffers for + /// , or . + /// means platform default. + /// + /// + /// Overriding this value is useful for interop scenarios + /// ensuring succeeds. + /// + public int? MinimumContiguousBlockBytes + { + get => this.minimumContiguousBlockBytes; + set + { + if (value.HasValue) + { + // It doesn't make sense to set this to small values in practice. + // Defining an arbitrary minimum of 65536. + Guard.MustBeGreaterThanOrEqualTo(value.Value, 65536, nameof(this.MaximumPoolSizeMegabytes)); + } + + this.minimumContiguousBlockBytes = value; + } + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/SimpleGcMemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/SimpleGcMemoryAllocator.cs index 84494f6856..a53ecbc66e 100644 --- a/src/ImageSharp/Memory/Allocators/SimpleGcMemoryAllocator.cs +++ b/src/ImageSharp/Memory/Allocators/SimpleGcMemoryAllocator.cs @@ -21,13 +21,5 @@ namespace SixLabors.ImageSharp.Memory return new BasicArrayBuffer(new T[length]); } - - /// - public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None) - { - Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); - - return new BasicByteBuffer(new byte[length]); - } } } diff --git a/src/ImageSharp/Memory/Allocators/UniformUnmanagedMemoryPoolMemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/UniformUnmanagedMemoryPoolMemoryAllocator.cs new file mode 100644 index 0000000000..51c20519d6 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/UniformUnmanagedMemoryPoolMemoryAllocator.cs @@ -0,0 +1,149 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using SixLabors.ImageSharp.Memory.Internals; + +namespace SixLabors.ImageSharp.Memory +{ + internal sealed class UniformUnmanagedMemoryPoolMemoryAllocator : MemoryAllocator + { + private const int OneMegabyte = 1 << 20; + private const int DefaultContiguousPoolBlockSizeBytes = 4 * OneMegabyte; + private const int DefaultNonPoolBlockSizeBytes = 32 * OneMegabyte; + private readonly int sharedArrayPoolThresholdInBytes; + private readonly int poolBufferSizeInBytes; + private readonly int poolCapacity; + + private UniformUnmanagedMemoryPool pool; + private readonly UnmanagedMemoryAllocator nonPoolAllocator; + + public UniformUnmanagedMemoryPoolMemoryAllocator(int? maxPoolSizeMegabytes, int? minimumContiguousBlockBytes) + : this( + minimumContiguousBlockBytes.HasValue ? minimumContiguousBlockBytes.Value : DefaultContiguousPoolBlockSizeBytes, + maxPoolSizeMegabytes.HasValue ? (long)maxPoolSizeMegabytes.Value * OneMegabyte : GetDefaultMaxPoolSizeBytes(), + minimumContiguousBlockBytes.HasValue ? Math.Max(minimumContiguousBlockBytes.Value, DefaultNonPoolBlockSizeBytes) : DefaultNonPoolBlockSizeBytes) + { + } + + public UniformUnmanagedMemoryPoolMemoryAllocator( + int poolBufferSizeInBytes, + long maxPoolSizeInBytes, + int unmanagedBufferSizeInBytes) + : this( + OneMegabyte, + poolBufferSizeInBytes, + maxPoolSizeInBytes, + unmanagedBufferSizeInBytes) + { + } + + // Internal constructor allowing to change the shared array pool threshold for testing purposes. + internal UniformUnmanagedMemoryPoolMemoryAllocator( + int sharedArrayPoolThresholdInBytes, + int poolBufferSizeInBytes, + long maxPoolSizeInBytes, + int unmanagedBufferSizeInBytes) + { + this.sharedArrayPoolThresholdInBytes = sharedArrayPoolThresholdInBytes; + this.poolBufferSizeInBytes = poolBufferSizeInBytes; + this.poolCapacity = (int)(maxPoolSizeInBytes / poolBufferSizeInBytes); + this.pool = new UniformUnmanagedMemoryPool(this.poolBufferSizeInBytes, this.poolCapacity); + this.nonPoolAllocator = new UnmanagedMemoryAllocator(unmanagedBufferSizeInBytes); + } + + /// + protected internal override int GetBufferCapacityInBytes() => this.poolBufferSizeInBytes; + + /// + public override IMemoryOwner Allocate( + int length, + AllocationOptions options = AllocationOptions.None) + { + Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); + int lengthInBytes = length * Unsafe.SizeOf(); + + if (lengthInBytes <= this.sharedArrayPoolThresholdInBytes) + { + var buffer = new SharedArrayPoolBuffer(length); + if (options.Has(AllocationOptions.Clean)) + { + buffer.GetSpan().Clear(); + } + + return buffer; + } + + if (lengthInBytes <= this.poolBufferSizeInBytes) + { + UnmanagedMemoryHandle array = this.pool.Rent(options); + if (array != null) + { + return new UniformUnmanagedMemoryPool.FinalizableBuffer(this.pool, array, length); + } + } + + return this.nonPoolAllocator.Allocate(length, options); + } + + /// + internal override MemoryGroup AllocateGroup( + long totalLength, + int bufferAlignment, + AllocationOptions options = AllocationOptions.None) + { + long totalLengthInBytes = totalLength * Unsafe.SizeOf(); + if (totalLengthInBytes <= this.sharedArrayPoolThresholdInBytes) + { + var buffer = new SharedArrayPoolBuffer((int)totalLength); + return MemoryGroup.CreateContiguous(buffer, options.Has(AllocationOptions.Clean)); + } + + if (totalLengthInBytes <= this.poolBufferSizeInBytes) + { + // Optimized path renting single array from the pool + UnmanagedMemoryHandle array = this.pool.Rent(options); + if (array != null) + { + var buffer = new UniformUnmanagedMemoryPool.FinalizableBuffer(this.pool, array, (int)totalLength); + return MemoryGroup.CreateContiguous(buffer, options.Has(AllocationOptions.Clean)); + } + } + + // Attempt to rent the whole group from the pool, allocate a group of unmanaged buffers if the attempt fails: + MemoryGroup poolGroup = options.Has(AllocationOptions.Contiguous) ? + null : + MemoryGroup.Allocate(this.pool, totalLength, bufferAlignment, options); + return poolGroup ?? MemoryGroup.Allocate(this.nonPoolAllocator, totalLength, bufferAlignment, options); + } + + public override void ReleaseRetainedResources() + { + UniformUnmanagedMemoryPool oldPool = Interlocked.Exchange( + ref this.pool, + new UniformUnmanagedMemoryPool(this.poolBufferSizeInBytes, this.poolCapacity)); + oldPool.Release(); + } + + private static long GetDefaultMaxPoolSizeBytes() + { +#if NETCORE31COMPATIBLE + // On .NET Core 3.1+, determine the pool as portion of the total available memory. + // There is a bug in GC.GetGCMemoryInfo() on .NET 5 + 32 bit, making TotalAvailableMemoryBytes unreliable: + // https://github.com/dotnet/runtime/issues/55126#issuecomment-876779327 + if (Environment.Is64BitProcess || !RuntimeInformation.FrameworkDescription.StartsWith(".NET 5.0")) + { + GCMemoryInfo info = GC.GetGCMemoryInfo(); + return info.TotalAvailableMemoryBytes / 8; + } +#endif + + // Stick to a conservative value of 128 Megabytes on other platforms and 32 bit .NET 5.0: + return 128 * OneMegabyte; + } + } +} diff --git a/src/ImageSharp/Memory/Allocators/UnmanagedMemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/UnmanagedMemoryAllocator.cs new file mode 100644 index 0000000000..731c8e0149 --- /dev/null +++ b/src/ImageSharp/Memory/Allocators/UnmanagedMemoryAllocator.cs @@ -0,0 +1,32 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using SixLabors.ImageSharp.Memory.Internals; + +namespace SixLabors.ImageSharp.Memory +{ + internal class UnmanagedMemoryAllocator : MemoryAllocator + { + private readonly int bufferCapacityInBytes; + + public UnmanagedMemoryAllocator(int bufferCapacityInBytes) + { + this.bufferCapacityInBytes = bufferCapacityInBytes; + } + + protected internal override int GetBufferCapacityInBytes() => this.bufferCapacityInBytes; + + public override IMemoryOwner Allocate(int length, AllocationOptions options = AllocationOptions.None) + { + var buffer = new UnmanagedBuffer(length); + if (options.Has(AllocationOptions.Clean)) + { + buffer.GetSpan().Clear(); + } + + return buffer; + } + } +} diff --git a/src/ImageSharp/Memory/Buffer2D{T}.cs b/src/ImageSharp/Memory/Buffer2D{T}.cs index 21c19f5d52..d97f16285a 100644 --- a/src/ImageSharp/Memory/Buffer2D{T}.cs +++ b/src/ImageSharp/Memory/Buffer2D{T}.cs @@ -19,8 +19,6 @@ namespace SixLabors.ImageSharp.Memory public sealed class Buffer2D : IDisposable where T : struct { - private Memory cachedMemory = default; - /// /// Initializes a new instance of the class. /// @@ -32,11 +30,6 @@ namespace SixLabors.ImageSharp.Memory this.FastMemoryGroup = memoryGroup; this.Width = width; this.Height = height; - - if (memoryGroup.Count == 1) - { - this.cachedMemory = memoryGroup[0]; - } } /// @@ -89,11 +82,7 @@ namespace SixLabors.ImageSharp.Memory /// /// Disposes the instance /// - public void Dispose() - { - this.FastMemoryGroup.Dispose(); - this.cachedMemory = default; - } + public void Dispose() => this.FastMemoryGroup.Dispose(); /// /// Gets a to the row 'y' beginning from the pixel at the first pixel on that row. @@ -111,39 +100,14 @@ namespace SixLabors.ImageSharp.Memory DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); - return this.cachedMemory.Length > 0 - ? this.cachedMemory.Span.Slice(y * this.Width, this.Width) - : this.GetRowMemorySlow(y).Span; + return this.GetRowMemoryCore(y).Span; } [MethodImpl(InliningOptions.ShortMethod)] internal ref T GetElementUnsafe(int x, int y) { - if (this.cachedMemory.Length > 0) - { - Span span = this.cachedMemory.Span; - ref T start = ref MemoryMarshal.GetReference(span); - return ref Unsafe.Add(ref start, (y * this.Width) + x); - } - - return ref this.GetElementSlow(x, y); - } - - /// - /// Gets a to the row 'y' beginning from the pixel at the first pixel on that row. - /// This method is intended for internal use only, since it does not use the indirection provided by - /// . - /// - /// The y (row) coordinate. - /// The . - [MethodImpl(InliningOptions.ShortMethod)] - internal Memory GetFastRowMemory(int y) - { - DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); - DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); - return this.cachedMemory.Length > 0 - ? this.cachedMemory.Slice(y * this.Width, this.Width) - : this.GetRowMemorySlow(y); + Span span = this.GetRowMemoryCore(y).Span; + return ref span[x]; } /// @@ -168,11 +132,7 @@ namespace SixLabors.ImageSharp.Memory /// Thrown when the backing group is discontiguous. /// [MethodImpl(InliningOptions.ShortMethod)] - internal Span DangerousGetSingleSpan() - { - // TODO: If we need a public version of this method, we need to cache the non-fast Memory of this.MemoryGroup - return this.cachedMemory.Length != 0 ? this.cachedMemory.Span : this.DangerousGetSingleSpanSlow(); - } + internal Span DangerousGetSingleSpan() => this.FastMemoryGroup.Single().Span; /// /// Gets a to the backing data of if the backing group consists of a single contiguous memory buffer. @@ -183,11 +143,7 @@ namespace SixLabors.ImageSharp.Memory /// Thrown when the backing group is discontiguous. /// [MethodImpl(InliningOptions.ShortMethod)] - internal Memory DangerousGetSingleMemory() - { - // TODO: If we need a public version of this method, we need to cache the non-fast Memory of this.MemoryGroup - return this.cachedMemory.Length != 0 ? this.cachedMemory : this.DangerousGetSingleMemorySlow(); - } + internal Memory DangerousGetSingleMemory() => this.FastMemoryGroup.Single(); /// /// Swaps the contents of 'destination' with 'source' if the buffers are owned (1), @@ -195,27 +151,14 @@ namespace SixLabors.ImageSharp.Memory /// internal static void SwapOrCopyContent(Buffer2D destination, Buffer2D source) { - bool swap = MemoryGroup.SwapOrCopyContent(destination.FastMemoryGroup, source.FastMemoryGroup); - SwapOwnData(destination, source, swap); + MemoryGroup.SwapOrCopyContent(destination.FastMemoryGroup, source.FastMemoryGroup); + SwapOwnData(destination, source); } - [MethodImpl(InliningOptions.ColdPath)] - private Memory GetRowMemorySlow(int y) => this.FastMemoryGroup.GetBoundedSlice(y * (long)this.Width, this.Width); - - [MethodImpl(InliningOptions.ColdPath)] - private Memory DangerousGetSingleMemorySlow() => this.FastMemoryGroup.Single(); - - [MethodImpl(InliningOptions.ColdPath)] - private Span DangerousGetSingleSpanSlow() => this.FastMemoryGroup.Single().Span; - - [MethodImpl(InliningOptions.ColdPath)] - private ref T GetElementSlow(int x, int y) - { - Span span = this.GetRowMemorySlow(y).Span; - return ref span[x]; - } + [MethodImpl(InliningOptions.ShortMethod)] + private Memory GetRowMemoryCore(int y) => this.FastMemoryGroup.GetBoundedSlice(y * (long)this.Width, this.Width); - private static void SwapOwnData(Buffer2D a, Buffer2D b, bool swapCachedMemory) + private static void SwapOwnData(Buffer2D a, Buffer2D b) { Size aSize = a.Size(); Size bSize = b.Size(); @@ -225,13 +168,6 @@ namespace SixLabors.ImageSharp.Memory a.Width = bSize.Width; a.Height = bSize.Height; - - if (swapCachedMemory) - { - Memory aCached = a.cachedMemory; - a.cachedMemory = b.cachedMemory; - b.cachedMemory = aCached; - } } } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Consumed.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Consumed.cs index cc2a2f17c9..2e690ce9b3 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Consumed.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Consumed.cs @@ -50,9 +50,12 @@ namespace SixLabors.ImageSharp.Memory return ((IList>)this.source).GetEnumerator(); } - public override void Dispose() + protected override void Dispose(bool disposing) { - this.View.Invalidate(); + if (disposing) + { + this.View.Invalidate(); + } } } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs index 35290c109e..7c0c3b764d 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs @@ -6,6 +6,7 @@ using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Memory.Internals; namespace SixLabors.ImageSharp.Memory { @@ -17,6 +18,9 @@ namespace SixLabors.ImageSharp.Memory public sealed class Owned : MemoryGroup, IEnumerable> { private IMemoryOwner[] memoryOwners; + private byte[][] pooledArrays; + private UniformUnmanagedMemoryPool unmanagedMemoryPool; + private UnmanagedMemoryHandle[] pooledHandles; public Owned(IMemoryOwner[] memoryOwners, int bufferLength, long totalLength, bool swappable) : base(bufferLength, totalLength) @@ -26,6 +30,15 @@ namespace SixLabors.ImageSharp.Memory this.View = new MemoryGroupView(this); } + public Owned(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle[] pooledArrays, int bufferLength, long totalLength, int sizeOfLastBuffer) + : this(CreateBuffers(pool, pooledArrays, bufferLength, sizeOfLastBuffer), bufferLength, totalLength, true) + { + this.pooledHandles = pooledArrays; + this.unmanagedMemoryPool = pool; + } + + ~Owned() => this.Dispose(false); + public bool Swappable { get; } private bool IsDisposed => this.memoryOwners == null; @@ -49,6 +62,23 @@ namespace SixLabors.ImageSharp.Memory } } + private static IMemoryOwner[] CreateBuffers( + UniformUnmanagedMemoryPool pool, + UnmanagedMemoryHandle[] pooledBuffers, + int bufferLength, + int sizeOfLastBuffer) + { + var result = new IMemoryOwner[pooledBuffers.Length]; + for (int i = 0; i < pooledBuffers.Length - 1; i++) + { + pooledBuffers[i].AssignedToNewOwner(); + result[i] = new UniformUnmanagedMemoryPool.Buffer(pool, pooledBuffers[i], bufferLength); + } + + result[result.Length - 1] = new UniformUnmanagedMemoryPool.Buffer(pool, pooledBuffers[pooledBuffers.Length - 1], sizeOfLastBuffer); + return result; + } + /// [MethodImpl(InliningOptions.ShortMethod)] public override MemoryGroupEnumerator GetEnumerator() @@ -63,7 +93,7 @@ namespace SixLabors.ImageSharp.Memory return this.memoryOwners.Select(mo => mo.Memory).GetEnumerator(); } - public override void Dispose() + protected override void Dispose(bool disposing) { if (this.IsDisposed) { @@ -72,13 +102,37 @@ namespace SixLabors.ImageSharp.Memory this.View.Invalidate(); - foreach (IMemoryOwner memoryOwner in this.memoryOwners) + if (this.unmanagedMemoryPool != null) { - memoryOwner.Dispose(); + this.unmanagedMemoryPool.Return(this.pooledHandles); + if (!disposing) + { + foreach (UnmanagedMemoryHandle handle in this.pooledHandles) + { + // We need to prevent handle finalization here. + // See comments on UnmanagedMemoryHandle.Resurrect() + handle.Resurrect(); + } + } + + foreach (IMemoryOwner memoryOwner in this.memoryOwners) + { + ((UniformUnmanagedMemoryPool.Buffer)memoryOwner).MarkDisposed(); + } + } + else if (disposing) + { + foreach (IMemoryOwner memoryOwner in this.memoryOwners) + { + memoryOwner.Dispose(); + } } this.memoryOwners = null; this.IsValid = false; + this.pooledArrays = null; + this.unmanagedMemoryPool = null; + this.pooledHandles = null; } [MethodImpl(InliningOptions.ShortMethod)] @@ -91,10 +145,7 @@ namespace SixLabors.ImageSharp.Memory } [MethodImpl(MethodImplOptions.NoInlining)] - private static void ThrowObjectDisposedException() - { - throw new ObjectDisposedException(nameof(MemoryGroup)); - } + private static void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(MemoryGroup)); internal static void SwapContents(Owned a, Owned b) { @@ -104,14 +155,23 @@ namespace SixLabors.ImageSharp.Memory IMemoryOwner[] tempOwners = a.memoryOwners; long tempTotalLength = a.TotalLength; int tempBufferLength = a.BufferLength; + byte[][] tempPooledArrays = a.pooledArrays; + UniformUnmanagedMemoryPool tempUnmangedPool = a.unmanagedMemoryPool; + UnmanagedMemoryHandle[] tempPooledHandles = a.pooledHandles; a.memoryOwners = b.memoryOwners; a.TotalLength = b.TotalLength; a.BufferLength = b.BufferLength; + a.pooledArrays = b.pooledArrays; + a.unmanagedMemoryPool = b.unmanagedMemoryPool; + a.pooledHandles = b.pooledHandles; b.memoryOwners = tempOwners; b.TotalLength = tempTotalLength; b.BufferLength = tempBufferLength; + b.pooledArrays = tempPooledArrays; + b.unmanagedMemoryPool = tempUnmangedPool; + b.pooledHandles = tempPooledHandles; a.View.Invalidate(); b.View.Invalidate(); diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs index 451a8f7e39..9b00d9cbf7 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs @@ -6,6 +6,7 @@ using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; +using SixLabors.ImageSharp.Memory.Internals; namespace SixLabors.ImageSharp.Memory { @@ -44,7 +45,13 @@ namespace SixLabors.ImageSharp.Memory public abstract Memory this[int index] { get; } /// - public abstract void Dispose(); + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + protected abstract void Dispose(bool disposing); /// public abstract MemoryGroupEnumerator GetEnumerator(); @@ -67,44 +74,47 @@ namespace SixLabors.ImageSharp.Memory /// Creates a new memory group, allocating it's buffers with the provided allocator. /// /// The to use. - /// The total length of the buffer. - /// The expected alignment (eg. to make sure image rows fit into single buffers). + /// The total length of the buffer. + /// The expected alignment (eg. to make sure image rows fit into single buffers). /// The . /// A new . /// Thrown when 'blockAlignment' converted to bytes is greater than the buffer capacity of the allocator. public static MemoryGroup Allocate( MemoryAllocator allocator, - long totalLength, - int bufferAlignment, + long totalLengthInElements, + int bufferAlignmentInElements, AllocationOptions options = AllocationOptions.None) { + int bufferCapacityInBytes = options.Has(AllocationOptions.Contiguous) ? + int.MaxValue : + allocator.GetBufferCapacityInBytes(); Guard.NotNull(allocator, nameof(allocator)); - Guard.MustBeGreaterThanOrEqualTo(totalLength, 0, nameof(totalLength)); - Guard.MustBeGreaterThanOrEqualTo(bufferAlignment, 0, nameof(bufferAlignment)); + Guard.MustBeGreaterThanOrEqualTo(totalLengthInElements, 0, nameof(totalLengthInElements)); + Guard.MustBeGreaterThanOrEqualTo(bufferAlignmentInElements, 0, nameof(bufferAlignmentInElements)); - int blockCapacityInElements = allocator.GetBufferCapacityInBytes() / ElementSize; + int blockCapacityInElements = bufferCapacityInBytes / ElementSize; - if (bufferAlignment > blockCapacityInElements) + if (bufferAlignmentInElements > blockCapacityInElements) { throw new InvalidMemoryOperationException( - $"The buffer capacity of the provided MemoryAllocator is insufficient for the requested buffer alignment: {bufferAlignment}."); + $"The buffer capacity of the provided MemoryAllocator is insufficient for the requested buffer alignment: {bufferAlignmentInElements}."); } - if (totalLength == 0) + if (totalLengthInElements == 0) { var buffers0 = new IMemoryOwner[1] { allocator.Allocate(0, options) }; return new Owned(buffers0, 0, 0, true); } - int numberOfAlignedSegments = blockCapacityInElements / bufferAlignment; - int bufferLength = numberOfAlignedSegments * bufferAlignment; - if (totalLength > 0 && totalLength < bufferLength) + int numberOfAlignedSegments = blockCapacityInElements / bufferAlignmentInElements; + int bufferLength = numberOfAlignedSegments * bufferAlignmentInElements; + if (totalLengthInElements > 0 && totalLengthInElements < bufferLength) { - bufferLength = (int)totalLength; + bufferLength = (int)totalLengthInElements; } - int sizeOfLastBuffer = (int)(totalLength % bufferLength); - long bufferCount = totalLength / bufferLength; + int sizeOfLastBuffer = (int)(totalLengthInElements % bufferLength); + long bufferCount = totalLengthInElements / bufferLength; if (sizeOfLastBuffer == 0) { @@ -126,7 +136,71 @@ namespace SixLabors.ImageSharp.Memory buffers[buffers.Length - 1] = allocator.Allocate(sizeOfLastBuffer, options); } - return new Owned(buffers, bufferLength, totalLength, true); + return new Owned(buffers, bufferLength, totalLengthInElements, true); + } + + public static MemoryGroup CreateContiguous(IMemoryOwner buffer, bool clear) + { + if (clear) + { + buffer.GetSpan().Clear(); + } + + int length = buffer.Memory.Length; + var buffers = new IMemoryOwner[1] { buffer }; + return new Owned(buffers, length, length, true); + } + + public static MemoryGroup Allocate( + UniformUnmanagedMemoryPool pool, + long totalLengthInElements, + int bufferAlignmentInElements, + AllocationOptions options = AllocationOptions.None) + { + Guard.NotNull(pool, nameof(pool)); + Guard.MustBeGreaterThanOrEqualTo(totalLengthInElements, 0, nameof(totalLengthInElements)); + Guard.MustBeGreaterThanOrEqualTo(bufferAlignmentInElements, 0, nameof(bufferAlignmentInElements)); + + int blockCapacityInElements = pool.BufferLength / ElementSize; + + if (bufferAlignmentInElements > blockCapacityInElements) + { + return null; + } + + if (totalLengthInElements == 0) + { + throw new InvalidMemoryOperationException("Allocating 0 length buffer from UniformByteArrayPool is disallowed"); + } + + int numberOfAlignedSegments = blockCapacityInElements / bufferAlignmentInElements; + int bufferLength = numberOfAlignedSegments * bufferAlignmentInElements; + if (totalLengthInElements > 0 && totalLengthInElements < bufferLength) + { + bufferLength = (int)totalLengthInElements; + } + + int sizeOfLastBuffer = (int)(totalLengthInElements % bufferLength); + int bufferCount = (int)(totalLengthInElements / bufferLength); + + if (sizeOfLastBuffer == 0) + { + sizeOfLastBuffer = bufferLength; + } + else + { + bufferCount++; + } + + UnmanagedMemoryHandle[] arrays = pool.Rent(bufferCount, options); + + if (arrays == null) + { + // Pool is full + return null; + } + + return new Owned(pool, arrays, bufferLength, totalLengthInElements, sizeOfLastBuffer); } public static MemoryGroup Wrap(params Memory[] source) diff --git a/src/ImageSharp/Memory/MemoryAllocatorExtensions.cs b/src/ImageSharp/Memory/MemoryAllocatorExtensions.cs index 2f70ac05e8..9cf1af659f 100644 --- a/src/ImageSharp/Memory/MemoryAllocatorExtensions.cs +++ b/src/ImageSharp/Memory/MemoryAllocatorExtensions.cs @@ -83,22 +83,5 @@ namespace SixLabors.ImageSharp.Memory int length = (width * pixelSizeInBytes) + paddingInBytes; return memoryAllocator.Allocate(length); } - - /// - /// Allocates a . - /// - /// The to use. - /// The total length of the buffer. - /// The expected alignment (eg. to make sure image rows fit into single buffers). - /// The . - /// A new . - /// Thrown when 'blockAlignment' converted to bytes is greater than the buffer capacity of the allocator. - internal static MemoryGroup AllocateGroup( - this MemoryAllocator memoryAllocator, - long totalLength, - int bufferAlignment, - AllocationOptions options = AllocationOptions.None) - where T : struct - => MemoryGroup.Allocate(memoryAllocator, totalLength, bufferAlignment, options); } } diff --git a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs index a58c20f687..220d852507 100644 --- a/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs +++ b/src/ImageSharp/Processing/Processors/Transforms/Resize/ResizeKernelMap.cs @@ -51,7 +51,7 @@ namespace SixLabors.ImageSharp.Processing.Processors.Transforms this.sourceLength = sourceLength; this.DestinationLength = destinationLength; this.MaxDiameter = (radius * 2) + 1; - this.data = memoryAllocator.Allocate2D(this.MaxDiameter, bufferHeight, AllocationOptions.Clean); + this.data = memoryAllocator.Allocate2D(this.MaxDiameter, bufferHeight, AllocationOptions.Clean | AllocationOptions.Contiguous); this.pinHandle = this.data.DangerousGetSingleMemory().Pin(); this.kernels = new ResizeKernel[destinationLength]; this.tempValues = new double[this.MaxDiameter]; diff --git a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj index 84b83ee14a..e69a006edd 100644 --- a/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj +++ b/tests/ImageSharp.Benchmarks/ImageSharp.Benchmarks.csproj @@ -38,9 +38,9 @@ - + - + diff --git a/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressBenchmarks.cs b/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressBenchmarks.cs index f1f7de3dc6..d3191c6955 100644 --- a/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressBenchmarks.cs +++ b/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressBenchmarks.cs @@ -3,6 +3,7 @@ using System; using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave { @@ -16,6 +17,11 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave // private const JpegKind Filter = JpegKind.Progressive; private const JpegKind Filter = JpegKind.Any; +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + private ArrayPoolMemoryAllocator arrayPoolMemoryAllocator; +#pragma warning restore CS0618 + private MemoryAllocator defaultMemoryAllocator; + [GlobalSetup] public void Setup() { @@ -26,6 +32,11 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave }; Console.WriteLine($"ImageCount: {this.runner.ImageCount} Filter: {Filter}"); this.runner.Init(); + this.defaultMemoryAllocator = Configuration.Default.MemoryAllocator; + +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + this.arrayPoolMemoryAllocator = ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore CS0618 } private void ForEachImage(Action action, int maxDegreeOfParallelism) @@ -48,7 +59,19 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave [Benchmark] [ArgumentsSource(nameof(ParallelismValues))] - public void ImageSharp(int maxDegreeOfParallelism) => this.ForEachImage(this.runner.ImageSharpResize, maxDegreeOfParallelism); + public void ImageSharp_DefaultMemoryAllocator(int maxDegreeOfParallelism) + { + Configuration.Default.MemoryAllocator = this.defaultMemoryAllocator; + this.ForEachImage(this.runner.ImageSharpResize, maxDegreeOfParallelism); + } + + [Benchmark] + [ArgumentsSource(nameof(ParallelismValues))] + public void ImageSharp_ArrayPoolMemoryAllocator(int maxDegreeOfParallelism) + { + Configuration.Default.MemoryAllocator = this.arrayPoolMemoryAllocator; + this.ForEachImage(this.runner.ImageSharpResize, maxDegreeOfParallelism); + } [Benchmark] [ArgumentsSource(nameof(ParallelismValues))] diff --git a/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs b/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs index c15f641b4a..7bdc64f02f 100644 --- a/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs +++ b/tests/ImageSharp.Benchmarks/LoadResizeSave/LoadResizeSaveStressRunner.cs @@ -56,7 +56,7 @@ namespace SixLabors.ImageSharp.Benchmarks.LoadResizeSave public int MaxDegreeOfParallelism { get; set; } = -1; - public JpegKind Filter { get; set; } + public JpegKind Filter { get; set; } = JpegKind.Any; private static readonly string[] ProgressiveFiles = { diff --git a/tests/ImageSharp.Tests.ProfilingSandbox/ImageSharp.Tests.ProfilingSandbox.csproj b/tests/ImageSharp.Tests.ProfilingSandbox/ImageSharp.Tests.ProfilingSandbox.csproj index 10deb24c63..b26666da6a 100644 --- a/tests/ImageSharp.Tests.ProfilingSandbox/ImageSharp.Tests.ProfilingSandbox.csproj +++ b/tests/ImageSharp.Tests.ProfilingSandbox/ImageSharp.Tests.ProfilingSandbox.csproj @@ -44,6 +44,7 @@ + diff --git a/tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs b/tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs index 2aadf02eb9..3782e5584e 100644 --- a/tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs +++ b/tests/ImageSharp.Tests.ProfilingSandbox/LoadResizeSaveParallelMemoryStress.cs @@ -3,29 +3,107 @@ using System; using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Runtime; using System.Text; +using System.Threading; +using CommandLine; using SixLabors.ImageSharp.Benchmarks.LoadResizeSave; +using SixLabors.ImageSharp.Memory; namespace SixLabors.ImageSharp.Tests.ProfilingSandbox { // See ImageSharp.Benchmarks/LoadResizeSave/README.md internal class LoadResizeSaveParallelMemoryStress { - private readonly LoadResizeSaveStressRunner benchmarks; - private LoadResizeSaveParallelMemoryStress() { - this.benchmarks = new LoadResizeSaveStressRunner() + this.Benchmarks = new LoadResizeSaveStressRunner() { // MaxDegreeOfParallelism = 10, // Filter = JpegKind.Baseline }; - this.benchmarks.Init(); + this.Benchmarks.Init(); + } + + public LoadResizeSaveStressRunner Benchmarks { get; } + + public static void Run(string[] args) + { + var options = CommandLineOptions.Parse(args); + + var lrs = new LoadResizeSaveParallelMemoryStress(); + if (options != null) + { + lrs.Benchmarks.MaxDegreeOfParallelism = options.MaxDegreeOfParallelism; + } + + Console.WriteLine($"\nEnvironment.ProcessorCount={Environment.ProcessorCount}"); + Stopwatch timer; + + if (options == null || !options.ImageSharp) + { + RunBenchmarkSwitcher(lrs, out timer); + } + else + { + Console.WriteLine("Running ImageSharp with options:"); + Console.WriteLine(options.ToString()); + Configuration.Default.MemoryAllocator = options.CreateMemoryAllocator(); + timer = Stopwatch.StartNew(); + try + { + for (int i = 0; i < options.RepeatCount; i++) + { + lrs.ImageSharpBenchmarkParallel(); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + + timer.Stop(); + + if (options.ReleaseRetainedResourcesAtEnd) + { + Configuration.Default.MemoryAllocator.ReleaseRetainedResources(); + } + + for (int i = 0; i < options.FinalGcCount; i++) + { + GC.Collect(); + GC.WaitForPendingFinalizers(); + Thread.Sleep(1000); + } + } + + var stats = new Stats(timer, lrs.Benchmarks.TotalProcessedMegapixels); + Console.WriteLine(stats.GetMarkdown()); + if (options?.FileOutput != null) + { + PrintFileOutput(options.FileOutput, stats); + } + + if (options != null && options.PauseAtEnd) + { + Console.WriteLine("Press ENTER"); + Console.ReadLine(); + } } - private double TotalProcessedMegapixels => this.benchmarks.TotalProcessedMegapixels; + private static void PrintFileOutput(string fileOutput, Stats stats) + { + string[] ss = fileOutput.Split(';'); + string fileName = ss[0]; + string content = ss[1] + .Replace("TotalSeconds", stats.TotalSeconds.ToString(CultureInfo.InvariantCulture)) + .Replace("EOL", Environment.NewLine, StringComparison.OrdinalIgnoreCase); + File.AppendAllText(fileName, content); + } - public static void Run() + private static void RunBenchmarkSwitcher(LoadResizeSaveParallelMemoryStress lrs, out Stopwatch timer) { Console.WriteLine(@"Choose a library for image resizing stress test: @@ -41,48 +119,34 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox if (key < ConsoleKey.D1 || key > ConsoleKey.D6) { Console.WriteLine("Unrecognized command."); - return; + Environment.Exit(-1); } - try - { - var lrs = new LoadResizeSaveParallelMemoryStress(); + timer = Stopwatch.StartNew(); - Console.WriteLine($"\nEnvironment.ProcessorCount={Environment.ProcessorCount}"); - Console.WriteLine($"Running with MaxDegreeOfParallelism={lrs.benchmarks.MaxDegreeOfParallelism} ..."); - var timer = Stopwatch.StartNew(); - - switch (key) - { - case ConsoleKey.D1: - lrs.SystemDrawingBenchmarkParallel(); - break; - case ConsoleKey.D2: - lrs.ImageSharpBenchmarkParallel(); - break; - case ConsoleKey.D3: - lrs.MagicScalerBenchmarkParallel(); - break; - case ConsoleKey.D4: - lrs.SkiaBitmapBenchmarkParallel(); - break; - case ConsoleKey.D5: - lrs.NetVipsBenchmarkParallel(); - break; - case ConsoleKey.D6: - lrs.MagickBenchmarkParallel(); - break; - } - - timer.Stop(); - var stats = new Stats(timer, lrs.TotalProcessedMegapixels); - Console.WriteLine("Done. TotalProcessedMegapixels: " + lrs.TotalProcessedMegapixels); - Console.WriteLine(stats.GetMarkdown()); - } - catch (Exception ex) + switch (key) { - Console.WriteLine(ex.ToString()); + case ConsoleKey.D1: + lrs.SystemDrawingBenchmarkParallel(); + break; + case ConsoleKey.D2: + lrs.ImageSharpBenchmarkParallel(); + break; + case ConsoleKey.D3: + lrs.MagicScalerBenchmarkParallel(); + break; + case ConsoleKey.D4: + lrs.SkiaBitmapBenchmarkParallel(); + break; + case ConsoleKey.D5: + lrs.NetVipsBenchmarkParallel(); + break; + case ConsoleKey.D6: + lrs.MagickBenchmarkParallel(); + break; } + + timer.Stop(); } private struct Stats @@ -125,18 +189,95 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox } } - private void ForEachImage(Action action) => this.benchmarks.ForEachImageParallel(action); + private enum AllocatorKind + { + Classic, + Unmanaged + } + + private class CommandLineOptions + { + [Option('i', "imagesharp", Required = false, Default = false)] + public bool ImageSharp { get; set; } + + [Option('a', "allocator", Required = false, Default = AllocatorKind.Unmanaged)] + public AllocatorKind Allocator { get; set; } + + [Option('m', "max-contiguous", Required = false, Default = 4)] + public int MaxContiguousPoolBufferMegaBytes { get; set; } = 4; + + [Option('s', "poolsize", Required = false, Default = 4096)] + public int MaxPoolSizeMegaBytes { get; set; } = 4096; + + [Option('u', "max-unmg", Required = false, Default = 32)] + public int MaxCapacityOfUnmanagedBuffersMegaBytes { get; set; } = 32; + + [Option('p', "parallelism", Required = false, Default = -1)] + public int MaxDegreeOfParallelism { get; set; } = -1; + + [Option('r', "repeat-count", Required = false, Default = 1)] + public int RepeatCount { get; set; } = 1; + + // This is to test trimming and virtual memory decommit + [Option('g', "final-gc-count", Required = false, Default = 0)] + public int FinalGcCount { get; set; } + + [Option('e', "release-at-end", Required = false, Default = false)] + public bool ReleaseRetainedResourcesAtEnd { get; set; } + + [Option('w', "pause", Required = false, Default = false)] + public bool PauseAtEnd { get; set; } + + [Option('f', "file", Required = false, Default = null)] + public string FileOutput { get; set; } + + public static CommandLineOptions Parse(string[] args) + { + CommandLineOptions result = null; + Parser.Default.ParseArguments(args).WithParsed(o => + { + result = o; + }); + return result; + } + + public override string ToString() => + $"p({this.MaxDegreeOfParallelism})_i({this.ImageSharp})_a({this.Allocator})_m({this.MaxContiguousPoolBufferMegaBytes})_s({this.MaxPoolSizeMegaBytes})_u({this.MaxCapacityOfUnmanagedBuffersMegaBytes})_r({this.RepeatCount})_g({this.FinalGcCount})_e({this.ReleaseRetainedResourcesAtEnd})"; + + public MemoryAllocator CreateMemoryAllocator() + { + switch (this.Allocator) + { + case AllocatorKind.Classic: +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + return ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore CS0618 + case AllocatorKind.Unmanaged: + return new UniformUnmanagedMemoryPoolMemoryAllocator( + 1024 * 1024, + (int)B(this.MaxContiguousPoolBufferMegaBytes), + B(this.MaxPoolSizeMegaBytes), + (int)B(this.MaxCapacityOfUnmanagedBuffersMegaBytes)); + default: + throw new ArgumentOutOfRangeException(); + } + } + + private static long B(double megaBytes) => (long)(megaBytes * 1024 * 1024); + } + + private void ForEachImage(Action action) => this.Benchmarks.ForEachImageParallel(action); - private void SystemDrawingBenchmarkParallel() => this.ForEachImage(this.benchmarks.SystemDrawingResize); + private void SystemDrawingBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SystemDrawingResize); - private void ImageSharpBenchmarkParallel() => this.ForEachImage(this.benchmarks.ImageSharpResize); + private void ImageSharpBenchmarkParallel() => this.ForEachImage(this.Benchmarks.ImageSharpResize); - private void MagickBenchmarkParallel() => this.ForEachImage(this.benchmarks.MagickResize); + private void MagickBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagickResize); - private void MagicScalerBenchmarkParallel() => this.ForEachImage(this.benchmarks.MagicScalerResize); + private void MagicScalerBenchmarkParallel() => this.ForEachImage(this.Benchmarks.MagicScalerResize); - private void SkiaBitmapBenchmarkParallel() => this.ForEachImage(this.benchmarks.SkiaBitmapResize); + private void SkiaBitmapBenchmarkParallel() => this.ForEachImage(this.Benchmarks.SkiaBitmapResize); - private void NetVipsBenchmarkParallel() => this.ForEachImage(this.benchmarks.NetVipsResize); + private void NetVipsBenchmarkParallel() => this.ForEachImage(this.Benchmarks.NetVipsResize); } } diff --git a/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs b/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs index e6e82b9810..a1bca9e6a7 100644 --- a/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs +++ b/tests/ImageSharp.Tests.ProfilingSandbox/Program.cs @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. using System; +using System.Threading; +using SixLabors.ImageSharp.Memory.Internals; using SixLabors.ImageSharp.Tests.Formats.Jpg; using SixLabors.ImageSharp.Tests.PixelFormats.PixelOperations; using SixLabors.ImageSharp.Tests.ProfilingBenchmarks; @@ -31,7 +33,8 @@ namespace SixLabors.ImageSharp.Tests.ProfilingSandbox /// public static void Main(string[] args) { - LoadResizeSaveParallelMemoryStress.Run(); + LoadResizeSaveParallelMemoryStress.Run(args); + // TrimPoolTest(); // RunJpegEncoderProfilingTests(); // RunJpegColorProfilingTests(); // RunDecodeJpegProfilingTests(); diff --git a/tests/ImageSharp.Tests/Formats/GeneralFormatTests.cs b/tests/ImageSharp.Tests/Formats/GeneralFormatTests.cs index c0843a51bb..09513010b0 100644 --- a/tests/ImageSharp.Tests/Formats/GeneralFormatTests.cs +++ b/tests/ImageSharp.Tests/Formats/GeneralFormatTests.cs @@ -98,8 +98,6 @@ namespace SixLabors.ImageSharp.Tests.Formats public void QuantizeImageShouldPreserveMaximumColorPrecision(TestImageProvider provider, string quantizerName) where TPixel : unmanaged, IPixel { - provider.Configuration.MemoryAllocator = ArrayPoolMemoryAllocator.CreateWithModeratePooling(); - IQuantizer quantizer = GetQuantizer(quantizerName); using (Image image = provider.GetImage()) diff --git a/tests/ImageSharp.Tests/Formats/Jpg/SpectralJpegTests.cs b/tests/ImageSharp.Tests/Formats/Jpg/SpectralJpegTests.cs index 40b9e68677..aaf0f13f82 100644 --- a/tests/ImageSharp.Tests/Formats/Jpg/SpectralJpegTests.cs +++ b/tests/ImageSharp.Tests/Formats/Jpg/SpectralJpegTests.cs @@ -46,7 +46,6 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg public static readonly string[] AllTestJpegs = BaselineTestJpegs.Concat(ProgressiveTestJpegs).ToArray(); [Theory(Skip = "Debug only, enable manually!")] - //[Theory] [WithFileCollection(nameof(AllTestJpegs), PixelTypes.Rgba32)] public void Decoder_ParseStream_SaveSpectralResult(TestImageProvider provider) where TPixel : unmanaged, IPixel @@ -126,7 +125,8 @@ namespace SixLabors.ImageSharp.Tests.Formats.Jpg this.Output.WriteLine($"Component{i}: [total: {total} | average: {average}]"); averageDifference += average; totalDifference += total; - tolerance += libJpegComponent.SpectralBlocks.DangerousGetSingleSpan().Length; + Size s = libJpegComponent.SpectralBlocks.Size(); + tolerance += s.Width * s.Height; } averageDifference /= componentCount; diff --git a/tests/ImageSharp.Tests/Formats/Tiff/Compression/PackBitsTiffCompressionTests.cs b/tests/ImageSharp.Tests/Formats/Tiff/Compression/PackBitsTiffCompressionTests.cs index b67cb83254..066264c330 100644 --- a/tests/ImageSharp.Tests/Formats/Tiff/Compression/PackBitsTiffCompressionTests.cs +++ b/tests/ImageSharp.Tests/Formats/Tiff/Compression/PackBitsTiffCompressionTests.cs @@ -29,7 +29,7 @@ namespace SixLabors.ImageSharp.Tests.Formats.Tiff.Compression var stream = new BufferedReadStream(Configuration.Default, new MemoryStream(inputData)); var buffer = new byte[expectedResult.Length]; - using var decompressor = new PackBitsTiffCompression(new ArrayPoolMemoryAllocator(), default, default); + using var decompressor = new PackBitsTiffCompression(MemoryAllocator.CreateDefault(), default, default); decompressor.Decompress(stream, 0, (uint)inputData.Length, buffer); Assert.Equal(expectedResult, buffer); diff --git a/tests/ImageSharp.Tests/Formats/Tiff/TiffEncoderHeaderTests.cs b/tests/ImageSharp.Tests/Formats/Tiff/TiffEncoderHeaderTests.cs index acbe2b4896..3cc52aa1a0 100644 --- a/tests/ImageSharp.Tests/Formats/Tiff/TiffEncoderHeaderTests.cs +++ b/tests/ImageSharp.Tests/Formats/Tiff/TiffEncoderHeaderTests.cs @@ -14,7 +14,7 @@ namespace SixLabors.ImageSharp.Tests.Formats.Tiff [Trait("Format", "Tiff")] public class TiffEncoderHeaderTests { - private static readonly MemoryAllocator MemoryAllocator = new ArrayPoolMemoryAllocator(); + private static readonly MemoryAllocator MemoryAllocator = MemoryAllocator.CreateDefault(); private static readonly Configuration Configuration = Configuration.Default; private static readonly ITiffEncoderOptions Options = new TiffEncoder(); diff --git a/tests/ImageSharp.Tests/Helpers/ParallelRowIteratorTests.cs b/tests/ImageSharp.Tests/Helpers/ParallelRowIteratorTests.cs index 7d4f2da425..f856b1b120 100644 --- a/tests/ImageSharp.Tests/Helpers/ParallelRowIteratorTests.cs +++ b/tests/ImageSharp.Tests/Helpers/ParallelRowIteratorTests.cs @@ -361,7 +361,7 @@ namespace SixLabors.ImageSharp.Tests.Helpers in operation); // Assert: - TestImageExtensions.CompareBuffers(expected.DangerousGetSingleSpan(), actual.DangerousGetSingleSpan()); + TestImageExtensions.CompareBuffers(expected, actual); } } diff --git a/tests/ImageSharp.Tests/Image/ImageFrameTests.cs b/tests/ImageSharp.Tests/Image/ImageFrameTests.cs index d4aef75387..bbe1a2335e 100644 --- a/tests/ImageSharp.Tests/Image/ImageFrameTests.cs +++ b/tests/ImageSharp.Tests/Image/ImageFrameTests.cs @@ -16,8 +16,12 @@ namespace SixLabors.ImageSharp.Tests private void LimitBufferCapacity(int bufferCapacityInBytes) { - var allocator = (ArrayPoolMemoryAllocator)this.configuration.MemoryAllocator; + // TODO: Create a test-only MemoryAllocator for this +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + var allocator = ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore CS0618 allocator.BufferCapacityInBytes = bufferCapacityInBytes; + this.configuration.MemoryAllocator = allocator; } [Theory] diff --git a/tests/ImageSharp.Tests/Image/ImageTests.cs b/tests/ImageSharp.Tests/Image/ImageTests.cs index 1296f26c47..671fc2b909 100644 --- a/tests/ImageSharp.Tests/Image/ImageTests.cs +++ b/tests/ImageSharp.Tests/Image/ImageTests.cs @@ -99,8 +99,12 @@ namespace SixLabors.ImageSharp.Tests private void LimitBufferCapacity(int bufferCapacityInBytes) { - var allocator = (ArrayPoolMemoryAllocator)this.configuration.MemoryAllocator; + // TODO: Create a test-only MemoryAllocator for this +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + var allocator = ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore CS0618 allocator.BufferCapacityInBytes = bufferCapacityInBytes; + this.configuration.MemoryAllocator = allocator; } [Theory] diff --git a/tests/ImageSharp.Tests/ImageSharp.Tests.csproj b/tests/ImageSharp.Tests/ImageSharp.Tests.csproj index 30bd544fa3..88b6b88134 100644 --- a/tests/ImageSharp.Tests/ImageSharp.Tests.csproj +++ b/tests/ImageSharp.Tests/ImageSharp.Tests.csproj @@ -7,6 +7,7 @@ AnyCPU;x64;x86 SixLabors.ImageSharp.Tests Debug;Release;Debug-InnerLoop;Release-InnerLoop + $(DefineConstants);NETCORE31COMPATIBLE @@ -21,7 +22,7 @@ - + diff --git a/tests/ImageSharp.Tests/Memory/Allocators/ArrayPoolMemoryAllocatorTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/ArrayPoolMemoryAllocatorTests.cs index 50ec09ce3f..b3e3718ec8 100644 --- a/tests/ImageSharp.Tests/Memory/Allocators/ArrayPoolMemoryAllocatorTests.cs +++ b/tests/ImageSharp.Tests/Memory/Allocators/ArrayPoolMemoryAllocatorTests.cs @@ -11,6 +11,7 @@ using Xunit; namespace SixLabors.ImageSharp.Tests.Memory.Allocators { +#pragma warning disable CS0618 public class ArrayPoolMemoryAllocatorTests { private const int MaxPooledBufferSizeInBytes = 2048; @@ -223,15 +224,6 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators Assert.Equal(0, buffer.Memory.Length); } - [Theory] - [InlineData(-1)] - public void AllocateManagedByteBuffer_IncorrectAmount_ThrowsCorrect_ArgumentOutOfRangeException(int length) - { - ArgumentOutOfRangeException ex = Assert.Throws(() => - this.LocalFixture.MemoryAllocator.AllocateManagedByteBuffer(length)); - Assert.Equal("length", ex.ParamName); - } - private class MemoryAllocatorFixture { public ArrayPoolMemoryAllocator MemoryAllocator { get; set; } = @@ -268,4 +260,5 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators { } } +#pragma warning restore CS0618 } diff --git a/tests/ImageSharp.Tests/Memory/Allocators/BufferTestSuite.cs b/tests/ImageSharp.Tests/Memory/Allocators/BufferTestSuite.cs index 1cadf16536..c3c3d40b9d 100644 --- a/tests/ImageSharp.Tests/Memory/Allocators/BufferTestSuite.cs +++ b/tests/ImageSharp.Tests/Memory/Allocators/BufferTestSuite.cs @@ -96,8 +96,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators [MemberData(nameof(LengthValues))] public void CanAllocateCleanBuffer_byte(int desiredLength) { - this.TestCanAllocateCleanBuffer(desiredLength, false); - this.TestCanAllocateCleanBuffer(desiredLength, true); + this.TestCanAllocateCleanBuffer(desiredLength); } [Theory] @@ -114,30 +113,14 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators this.TestCanAllocateCleanBuffer(desiredLength); } - private IMemoryOwner Allocate(int desiredLength, AllocationOptions options, bool managedByteBuffer) - where T : struct - { - if (managedByteBuffer) - { - if (!(this.MemoryAllocator.AllocateManagedByteBuffer(desiredLength, options) is IMemoryOwner buffer)) - { - throw new InvalidOperationException("typeof(T) != typeof(byte)"); - } - - return buffer; - } - - return this.MemoryAllocator.Allocate(desiredLength, options); - } - - private void TestCanAllocateCleanBuffer(int desiredLength, bool testManagedByteBuffer = false) + private void TestCanAllocateCleanBuffer(int desiredLength) where T : struct, IEquatable { ReadOnlySpan expected = new T[desiredLength]; for (int i = 0; i < 10; i++) { - using (IMemoryOwner buffer = this.Allocate(desiredLength, AllocationOptions.Clean, testManagedByteBuffer)) + using (IMemoryOwner buffer = this.MemoryAllocator.Allocate(desiredLength, AllocationOptions.Clean)) { Assert.True(buffer.GetSpan().SequenceEqual(expected)); } @@ -155,14 +138,13 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators [MemberData(nameof(LengthValues))] public void SpanPropertyIsAlwaysTheSame_byte(int desiredLength) { - this.TestSpanPropertyIsAlwaysTheSame(desiredLength, false); - this.TestSpanPropertyIsAlwaysTheSame(desiredLength, true); + this.TestSpanPropertyIsAlwaysTheSame(desiredLength); } - private void TestSpanPropertyIsAlwaysTheSame(int desiredLength, bool testManagedByteBuffer = false) + private void TestSpanPropertyIsAlwaysTheSame(int desiredLength) where T : struct { - using (IMemoryOwner buffer = this.Allocate(desiredLength, AllocationOptions.None, testManagedByteBuffer)) + using (IMemoryOwner buffer = this.MemoryAllocator.Allocate(desiredLength, AllocationOptions.None)) { ref T a = ref MemoryMarshal.GetReference(buffer.GetSpan()); ref T b = ref MemoryMarshal.GetReference(buffer.GetSpan()); @@ -184,14 +166,13 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators [MemberData(nameof(LengthValues))] public void WriteAndReadElements_byte(int desiredLength) { - this.TestWriteAndReadElements(desiredLength, x => (byte)(x + 1), false); - this.TestWriteAndReadElements(desiredLength, x => (byte)(x + 1), true); + this.TestWriteAndReadElements(desiredLength, x => (byte)(x + 1)); } - private void TestWriteAndReadElements(int desiredLength, Func getExpectedValue, bool testManagedByteBuffer = false) + private void TestWriteAndReadElements(int desiredLength, Func getExpectedValue) where T : struct { - using (IMemoryOwner buffer = this.Allocate(desiredLength, AllocationOptions.None, testManagedByteBuffer)) + using (IMemoryOwner buffer = this.MemoryAllocator.Allocate(desiredLength)) { var expectedVals = new T[buffer.Length()]; @@ -214,8 +195,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators [MemberData(nameof(LengthValues))] public void IndexingSpan_WhenOutOfRange_Throws_byte(int desiredLength) { - this.TestIndexOutOfRangeShouldThrow(desiredLength, false); - this.TestIndexOutOfRangeShouldThrow(desiredLength, true); + this.TestIndexOutOfRangeShouldThrow(desiredLength); } [Theory] @@ -232,12 +212,12 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators this.TestIndexOutOfRangeShouldThrow(desiredLength); } - private T TestIndexOutOfRangeShouldThrow(int desiredLength, bool testManagedByteBuffer = false) + private T TestIndexOutOfRangeShouldThrow(int desiredLength) where T : struct, IEquatable { var dummy = default(T); - using (IMemoryOwner buffer = this.Allocate(desiredLength, AllocationOptions.None, testManagedByteBuffer)) + using (IMemoryOwner buffer = this.MemoryAllocator.Allocate(desiredLength)) { Assert.ThrowsAny( () => @@ -264,23 +244,6 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators return dummy; } - [Theory] - [InlineData(1)] - [InlineData(7)] - [InlineData(1024)] - [InlineData(6666)] - public void ManagedByteBuffer_ArrayIsCorrect(int desiredLength) - { - using (IManagedByteBuffer buffer = this.MemoryAllocator.AllocateManagedByteBuffer(desiredLength)) - { - ref byte array0 = ref buffer.Array[0]; - ref byte span0 = ref buffer.GetReference(); - - Assert.True(Unsafe.AreSame(ref span0, ref array0)); - Assert.True(buffer.Array.Length >= buffer.GetSpan().Length); - } - } - [Fact] public void GetMemory_ReturnsValidMemory() { diff --git a/tests/ImageSharp.Tests/Memory/Allocators/SimpleGcMemoryAllocatorTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/SimpleGcMemoryAllocatorTests.cs index 8e7b305672..40d3bfe536 100644 --- a/tests/ImageSharp.Tests/Memory/Allocators/SimpleGcMemoryAllocatorTests.cs +++ b/tests/ImageSharp.Tests/Memory/Allocators/SimpleGcMemoryAllocatorTests.cs @@ -28,17 +28,9 @@ namespace SixLabors.ImageSharp.Tests.Memory.Allocators Assert.Equal("length", ex.ParamName); } - [Theory] - [InlineData(-1)] - public void AllocateManagedByteBuffer_IncorrectAmount_ThrowsCorrect_ArgumentOutOfRangeException(int length) - { - ArgumentOutOfRangeException ex = Assert.Throws(() => this.MemoryAllocator.AllocateManagedByteBuffer(length)); - Assert.Equal("length", ex.ParamName); - } - [StructLayout(LayoutKind.Explicit, Size = 512)] private struct BigStruct { } } -} \ No newline at end of file +} diff --git a/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.Trim.cs b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.Trim.cs new file mode 100644 index 0000000000..9726c9c585 --- /dev/null +++ b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.Trim.cs @@ -0,0 +1,101 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using Microsoft.DotNet.RemoteExecutor; +using SixLabors.ImageSharp.Memory.Internals; +using Xunit; + +namespace SixLabors.ImageSharp.Tests.Memory.Allocators +{ + public partial class UniformUnmanagedMemoryPoolTests + { + [CollectionDefinition(nameof(NonParallelTests), DisableParallelization = true)] + public class NonParallelTests + { + } + + [Collection(nameof(NonParallelTests))] + public class Trim + { + [Fact] + public void TrimPeriodElapsed_TrimsHalfOfUnusedArrays() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + static void RunTest() + { + var trimSettings = new UniformUnmanagedMemoryPool.TrimSettings { TrimPeriodMilliseconds = 5_000 }; + var pool = new UniformUnmanagedMemoryPool(128, 256, trimSettings); + + UnmanagedMemoryHandle[] a = pool.Rent(64); + UnmanagedMemoryHandle[] b = pool.Rent(64); + pool.Return(a); + Assert.Equal(128, UnmanagedMemoryHandle.TotalOutstandingHandles); + Thread.Sleep(15_000); + + // We expect at least 2 Trim actions, first trim 32, then 16 arrays. + // 128 - 32 - 16 = 80 + Assert.True( + UnmanagedMemoryHandle.TotalOutstandingHandles <= 80, + $"UnmanagedMemoryHandle.TotalOutstandingHandles={UnmanagedMemoryHandle.TotalOutstandingHandles} > 80"); + pool.Return(b); + } + } + +#if NETCORE31COMPATIBLE + public static readonly bool Is32BitProcess = !Environment.Is64BitProcess; + private static readonly List PressureArrays = new List(); + + [ConditionalFact(nameof(Is32BitProcess))] + public static void GC_Collect_OnHighLoad_TrimsEntirePool() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + static void RunTest() + { + Assert.False(Environment.Is64BitProcess); + const int OneMb = 1024 * 1024; + + var trimSettings = new UniformUnmanagedMemoryPool.TrimSettings { HighPressureThresholdRate = 0.2f }; + + GCMemoryInfo memInfo = GC.GetGCMemoryInfo(); + int highLoadThreshold = (int)(memInfo.HighMemoryLoadThresholdBytes / OneMb); + highLoadThreshold = (int)(trimSettings.HighPressureThresholdRate * highLoadThreshold); + + var pool = new UniformUnmanagedMemoryPool(OneMb, 16, trimSettings); + pool.Return(pool.Rent(16)); + Assert.Equal(16, UnmanagedMemoryHandle.TotalOutstandingHandles); + + for (int i = 0; i < highLoadThreshold; i++) + { + byte[] array = new byte[OneMb]; + TouchPage(array); + PressureArrays.Add(array); + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + GC.WaitForPendingFinalizers(); // The pool should be fully trimmed after this point + + Assert.Equal(0, UnmanagedMemoryHandle.TotalOutstandingHandles); + + static void TouchPage(byte[] b) + { + uint size = (uint)b.Length; + const uint pageSize = 4096; + uint numPages = size / pageSize; + + for (uint i = 0; i < numPages; i++) + { + b[i * pageSize] = (byte)(i % 256); + } + } + } + } +#endif + } + } +} diff --git a/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.cs new file mode 100644 index 0000000000..86bbff1812 --- /dev/null +++ b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedMemoryPoolTests.cs @@ -0,0 +1,292 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.RemoteExecutor; +using SixLabors.ImageSharp.Memory.Internals; +using Xunit; +using Xunit.Abstractions; + +namespace SixLabors.ImageSharp.Tests.Memory.Allocators +{ + public partial class UniformUnmanagedMemoryPoolTests + { + private readonly ITestOutputHelper output; + + public UniformUnmanagedMemoryPoolTests(ITestOutputHelper output) + { + this.output = output; + } + + private static unsafe Span GetSpan(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle h) => + new Span((void*)h.DangerousGetHandle(), pool.BufferLength); + + [Theory] + [InlineData(3, 11)] + [InlineData(7, 4)] + public void Constructor_InitializesProperties(int arrayLength, int capacity) + { + var pool = new UniformUnmanagedMemoryPool(arrayLength, capacity); + Assert.Equal(arrayLength, pool.BufferLength); + Assert.Equal(capacity, pool.Capacity); + } + + [Theory] + [InlineData(1, 3)] + [InlineData(8, 10)] + public void Rent_SingleBuffer_ReturnsCorrectBuffer(int length, int capacity) + { + var pool = new UniformUnmanagedMemoryPool(length, capacity); + for (int i = 0; i < capacity; i++) + { + UnmanagedMemoryHandle h = pool.Rent(); + CheckBuffer(length, pool, h); + } + } + + [Fact] + public void Return_DoesNotDeallocateMemory() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + + static void RunTest() + { + var pool = new UniformUnmanagedMemoryPool(16, 16); + UnmanagedMemoryHandle a = pool.Rent(); + UnmanagedMemoryHandle[] b = pool.Rent(2); + + Assert.Equal(3, UnmanagedMemoryHandle.TotalOutstandingHandles); + pool.Return(a); + pool.Return(b); + Assert.Equal(3, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + } + + private static void CheckBuffer(int length, UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle h) + { + Assert.NotNull(h); + Assert.False(h.IsClosed); + Span span = GetSpan(pool, h); + span.Fill(123); + + byte[] expected = new byte[length]; + expected.AsSpan().Fill(123); + Assert.True(span.SequenceEqual(expected)); + } + + [Theory] + [InlineData(1, 1)] + [InlineData(1, 5)] + [InlineData(42, 7)] + [InlineData(5, 10)] + public void Rent_MultiBuffer_ReturnsCorrectBuffers(int length, int bufferCount) + { + var pool = new UniformUnmanagedMemoryPool(length, 10); + UnmanagedMemoryHandle[] handles = pool.Rent(bufferCount); + Assert.NotNull(handles); + Assert.Equal(bufferCount, handles.Length); + + foreach (UnmanagedMemoryHandle h in handles) + { + CheckBuffer(length, pool, h); + } + } + + [Fact] + public void Rent_MultipleTimesWithoutReturn_ReturnsDifferentHandles() + { + var pool = new UniformUnmanagedMemoryPool(128, 10); + UnmanagedMemoryHandle[] a = pool.Rent(2); + UnmanagedMemoryHandle b = pool.Rent(); + + Assert.NotEqual(a[0].DangerousGetHandle(), a[1].DangerousGetHandle()); + Assert.NotEqual(a[0].DangerousGetHandle(), b.DangerousGetHandle()); + Assert.NotEqual(a[1].DangerousGetHandle(), b.DangerousGetHandle()); + } + + [Theory] + [InlineData(4, 2, 10)] + [InlineData(5, 1, 6)] + [InlineData(12, 4, 12)] + public void RentReturnRent_SameBuffers(int totalCount, int rentUnit, int capacity) + { + var pool = new UniformUnmanagedMemoryPool(128, capacity); + var allHandles = new HashSet(); + var handleUnits = new List(); + + UnmanagedMemoryHandle[] handles; + for (int i = 0; i < totalCount; i += rentUnit) + { + handles = pool.Rent(rentUnit); + Assert.NotNull(handles); + handleUnits.Add(handles); + foreach (UnmanagedMemoryHandle array in handles) + { + allHandles.Add(array); + } + } + + foreach (UnmanagedMemoryHandle[] arrayUnit in handleUnits) + { + if (arrayUnit.Length == 1) + { + // Test single-array return: + pool.Return(arrayUnit.Single()); + } + else + { + pool.Return(arrayUnit); + } + } + + handles = pool.Rent(totalCount); + + Assert.NotNull(handles); + + foreach (UnmanagedMemoryHandle array in handles) + { + Assert.Contains(array, allHandles); + } + } + + [Fact] + public void Rent_SingleBuffer_OverCapacity_ReturnsNull() + { + var pool = new UniformUnmanagedMemoryPool(7, 1000); + Assert.NotNull(pool.Rent(1000)); + Assert.Null(pool.Rent()); + } + + [Theory] + [InlineData(0, 6, 5)] + [InlineData(5, 1, 5)] + [InlineData(4, 7, 10)] + public void Rent_MultiBuffer_OverCapacity_ReturnsNull(int initialRent, int attempt, int capacity) + { + var pool = new UniformUnmanagedMemoryPool(128, capacity); + Assert.NotNull(pool.Rent(initialRent)); + Assert.Null(pool.Rent(attempt)); + } + + [Theory] + [InlineData(0, 5, 5)] + [InlineData(5, 1, 6)] + [InlineData(4, 7, 11)] + [InlineData(3, 3, 7)] + public void Rent_MultiBuff_BelowCapacity_Succeeds(int initialRent, int attempt, int capacity) + { + var pool = new UniformUnmanagedMemoryPool(128, capacity); + Assert.NotNull(pool.Rent(initialRent)); + Assert.NotNull(pool.Rent(attempt)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Release_SubsequentRentReturnsNull(bool multiple) + { + var pool = new UniformUnmanagedMemoryPool(16, 16); + pool.Rent(); // Dummy rent + pool.Release(); + if (multiple) + { + UnmanagedMemoryHandle b = pool.Rent(); + Assert.Null(b); + } + else + { + UnmanagedMemoryHandle[] b = pool.Rent(2); + Assert.Null(b); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Release_SubsequentReturnClosesHandle(bool multiple) + { + var pool = new UniformUnmanagedMemoryPool(16, 16); + if (multiple) + { + UnmanagedMemoryHandle[] b = pool.Rent(2); + pool.Release(); + + Assert.False(b[0].IsClosed); + Assert.False(b[1].IsClosed); + + pool.Return(b); + + Assert.True(b[0].IsClosed); + Assert.True(b[1].IsClosed); + } + else + { + UnmanagedMemoryHandle b = pool.Rent(); + pool.Release(); + Assert.False(b.IsClosed); + pool.Return(b); + Assert.True(b.IsClosed); + } + } + + [Fact] + public void Release_ShouldFreeRetainedMemory() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + + static void RunTest() + { + var pool = new UniformUnmanagedMemoryPool(16, 16); + UnmanagedMemoryHandle a = pool.Rent(); + UnmanagedMemoryHandle[] b = pool.Rent(2); + pool.Return(a); + pool.Return(b); + + Assert.Equal(3, UnmanagedMemoryHandle.TotalOutstandingHandles); + pool.Release(); + Assert.Equal(0, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + } + + [Fact] + public void RentReturn_IsThreadSafe() + { + int count = Environment.ProcessorCount * 200; + var pool = new UniformUnmanagedMemoryPool(8, count); + var rnd = new Random(0); + + Parallel.For(0, Environment.ProcessorCount, (int i) => + { + var allArrays = new List(); + int pauseAt = rnd.Next(100); + for (int j = 0; j < 100; j++) + { + UnmanagedMemoryHandle[] data = pool.Rent(2); + + GetSpan(pool, data[0]).Fill((byte)i); + GetSpan(pool, data[1]).Fill((byte)i); + allArrays.Add(data[0]); + allArrays.Add(data[1]); + + if (j == pauseAt) + { + Thread.Sleep(15); + } + } + + Span expected = new byte[8]; + expected.Fill((byte)i); + + foreach (UnmanagedMemoryHandle array in allArrays) + { + Assert.True(expected.SequenceEqual(GetSpan(pool, array))); + pool.Return(new[] { array }); + } + }); + } + } +} diff --git a/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs new file mode 100644 index 0000000000..a872deef05 --- /dev/null +++ b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs @@ -0,0 +1,289 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using Microsoft.DotNet.RemoteExecutor; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.Memory.Internals; +using SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers; +using Xunit; + +namespace SixLabors.ImageSharp.Tests.Memory.Allocators +{ + public class UniformUnmanagedPoolMemoryAllocatorTests + { + public class BufferTests1 : BufferTestSuite + { + private static MemoryAllocator CreateMemoryAllocator() => + new UniformUnmanagedMemoryPoolMemoryAllocator( + sharedArrayPoolThresholdInBytes: 1024, + poolBufferSizeInBytes: 2048, + maxPoolSizeInBytes: 2048 * 4, + unmanagedBufferSizeInBytes: 4096); + + public BufferTests1() + : base(CreateMemoryAllocator()) + { + } + } + + public class BufferTests2 : BufferTestSuite + { + private static MemoryAllocator CreateMemoryAllocator() => + new UniformUnmanagedMemoryPoolMemoryAllocator( + sharedArrayPoolThresholdInBytes: 512, + poolBufferSizeInBytes: 1024, + maxPoolSizeInBytes: 1024 * 4, + unmanagedBufferSizeInBytes: 2048); + + public BufferTests2() + : base(CreateMemoryAllocator()) + { + } + } + + public static TheoryData AllocateData = + new TheoryData() + { + { default(S4), 16, 256, 256, 1024, 64, 64, 1, -1, 64 }, + { default(S4), 16, 256, 256, 1024, 256, 256, 1, -1, 256 }, + { default(S4), 16, 256, 256, 1024, 250, 256, 1, -1, 250 }, + { default(S4), 16, 256, 256, 1024, 248, 250, 1, -1, 248 }, + { default(S4), 16, 1024, 2048, 4096, 512, 256, 2, 256, 256 }, + { default(S4), 16, 1024, 1024, 1024, 511, 256, 2, 256, 255 }, + }; + + [Theory] + [MemberData(nameof(AllocateData))] + public void AllocateGroup_BufferSizesAreCorrect( + T dummy, + int sharedArrayPoolThresholdInBytes, + int maxContiguousPoolBufferInBytes, + int maxPoolSizeInBytes, + int maxCapacityOfUnmanagedBuffers, + long allocationLengthInElements, + int bufferAlignmentInElements, + int expectedNumberOfBuffers, + int expectedBufferSize, + int expectedSizeOfLastBuffer) + where T : struct + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator( + sharedArrayPoolThresholdInBytes, + maxContiguousPoolBufferInBytes, + maxPoolSizeInBytes, + maxCapacityOfUnmanagedBuffers); + + using MemoryGroup g = allocator.AllocateGroup(allocationLengthInElements, bufferAlignmentInElements); + MemoryGroupTests.Allocate.ValidateAllocateMemoryGroup( + expectedNumberOfBuffers, + expectedBufferSize, + expectedSizeOfLastBuffer, + g); + } + + [Fact] + public void AllocateGroup_MultipleTimes_ExceedPoolLimit() + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator( + 64, + 128, + 1024, + 1024); + + var groups = new List>(); + for (int i = 0; i < 16; i++) + { + int lengthInElements = 128 / Unsafe.SizeOf(); + MemoryGroup g = allocator.AllocateGroup(lengthInElements, 32); + MemoryGroupTests.Allocate.ValidateAllocateMemoryGroup(1, -1, lengthInElements, g); + groups.Add(g); + } + + foreach (MemoryGroup g in groups) + { + g.Dispose(); + } + } + + [Theory] + [InlineData(512)] + [InlineData(2048)] + [InlineData(8192)] + [InlineData(65536)] + public void AllocateGroup_OptionsContiguous_AllocatesContiguousBuffer(int lengthInBytes) + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator( + 128, + 1024, + 2048, + 4096); + int length = lengthInBytes / Unsafe.SizeOf(); + using MemoryGroup g = allocator.AllocateGroup(length, 32, AllocationOptions.Contiguous); + Assert.Equal(length, g.BufferLength); + Assert.Equal(length, g.TotalLength); + Assert.Equal(1, g.Count); + } + + [Fact] + public void MemoryAllocator_CreateDefault_WithoutOptions_AllocatesDiscontiguousMemory() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + + static void RunTest() + { + var allocator = MemoryAllocator.CreateDefault(); + long sixteenMegabytes = 16 * (1 << 20); + + // Should allocate 4 times 4MB discontiguos blocks: + MemoryGroup g = allocator.AllocateGroup(sixteenMegabytes, 1024); + Assert.Equal(4, g.Count); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void MemoryAllocator_CreateDefault_WithOptions_CanForceContiguousAllocation(bool poolAllocation) + { + RemoteExecutor.Invoke(RunTest, poolAllocation.ToString()).Dispose(); + + static void RunTest(string poolAllocationStr) + { + int fortyEightMegabytes = 48 * (1 << 20); + var allocator = MemoryAllocator.CreateDefault(new MemoryAllocatorOptions() + { + MaximumPoolSizeMegabytes = bool.Parse(poolAllocationStr) ? 64 : 0, + MinimumContiguousBlockBytes = fortyEightMegabytes + }); + + MemoryGroup g = allocator.AllocateGroup(fortyEightMegabytes, 1024); + Assert.Equal(1, g.Count); + Assert.Equal(fortyEightMegabytes, g.TotalLength); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BufferDisposal_ReturnsToPool(bool shared) + { + RemoteExecutor.Invoke(RunTest, shared.ToString()).Dispose(); + + static void RunTest(string sharedStr) + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(512, 1024, 16 * 1024, 1024); + IMemoryOwner buffer0 = allocator.Allocate(bool.Parse(sharedStr) ? 300 : 600); + buffer0.GetSpan()[0] = 42; + buffer0.Dispose(); + using IMemoryOwner buffer1 = allocator.Allocate(bool.Parse(sharedStr) ? 300 : 600); + Assert.Equal(42, buffer1.GetSpan()[0]); + } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MemoryGroupDisposal_ReturnsToPool(bool shared) + { + RemoteExecutor.Invoke(RunTest, shared.ToString()).Dispose(); + + static void RunTest(string sharedStr) + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(512, 1024, 16 * 1024, 1024); + MemoryGroup g0 = allocator.AllocateGroup(bool.Parse(sharedStr) ? 300 : 600, 100); + g0.Single().Span[0] = 42; + g0.Dispose(); + using MemoryGroup g1 = allocator.AllocateGroup(bool.Parse(sharedStr) ? 300 : 600, 100); + Assert.Equal(42, g1.Single().Span[0]); + } + } + + [Fact] + public void ReleaseRetainedResources_ShouldFreePooledMemory() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + static void RunTest() + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(128, 512, 16 * 512, 1024); + MemoryGroup g = allocator.AllocateGroup(2048, 128); + g.Dispose(); + Assert.Equal(4, UnmanagedMemoryHandle.TotalOutstandingHandles); + allocator.ReleaseRetainedResources(); + Assert.Equal(0, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + } + + [Fact] + public void ReleaseRetainedResources_DoesNotFreeOutstandingBuffers() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + static void RunTest() + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(128, 512, 16 * 512, 1024); + IMemoryOwner b = allocator.Allocate(256); + MemoryGroup g = allocator.AllocateGroup(2048, 128); + Assert.Equal(5, UnmanagedMemoryHandle.TotalOutstandingHandles); + allocator.ReleaseRetainedResources(); + Assert.Equal(5, UnmanagedMemoryHandle.TotalOutstandingHandles); + b.Dispose(); + g.Dispose(); + Assert.Equal(0, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + } + + [Theory] + [InlineData(300)] + [InlineData(600)] + [InlineData(1200)] + public void MemoryGroupFinalizer_ReturnsToPool(int length) + { + // RunTest(length.ToString()); + RemoteExecutor.Invoke(RunTest, length.ToString()).Dispose(); + + static void RunTest(string lengthStr) + { + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator(512, 1024, 16 * 1024, 1024); + int lengthInner = int.Parse(lengthStr); + + AllocateGroupAndForget(allocator, lengthInner); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + + AllocateGroupAndForget(allocator, lengthInner, true); + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + GC.WaitForPendingFinalizers(); + + using MemoryGroup g = allocator.AllocateGroup(lengthInner, 100); + Assert.Equal(42, g.First().Span[0]); + } + } + + private static void AllocateGroupAndForget(UniformUnmanagedMemoryPoolMemoryAllocator allocator, int length, bool check = false) + { + MemoryGroup g = allocator.AllocateGroup(length, 100); + if (check) + { + Assert.Equal(42, g.First().Span[0]); + } + + g.First().Span[0] = 42; + + if (length < 512) + { + // For ArrayPool.Shared, first array will be returned to the TLS storage of the finalizer thread, + // repeat rental to make sure per-core buckets are also utilized. + MemoryGroup g1 = allocator.AllocateGroup(length, 100); + g1.First().Span[0] = 42; + } + } + } +} diff --git a/tests/ImageSharp.Tests/Memory/Allocators/UnmanagedMemoryHandleTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/UnmanagedMemoryHandleTests.cs new file mode 100644 index 0000000000..5744f81ec3 --- /dev/null +++ b/tests/ImageSharp.Tests/Memory/Allocators/UnmanagedMemoryHandleTests.cs @@ -0,0 +1,184 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Collections.Generic; +using Microsoft.DotNet.RemoteExecutor; +using SixLabors.ImageSharp.Memory.Internals; +using Xunit; + +namespace SixLabors.ImageSharp.Tests.Memory.Allocators +{ + public class UnmanagedMemoryHandleTests + { + [Fact] + public unsafe void Constructor_AllocatesReadWriteMemory() + { + using var h = new UnmanagedMemoryHandle(128); + Assert.False(h.IsClosed); + Assert.False(h.IsInvalid); + byte* ptr = (byte*)h.DangerousGetHandle(); + for (int i = 0; i < 128; i++) + { + ptr[i] = (byte)i; + } + + for (int i = 0; i < 128; i++) + { + Assert.Equal((byte)i, ptr[i]); + } + } + + [Fact] + public void Dispose_ClosesHandle() + { + var h = new UnmanagedMemoryHandle(128); + h.Dispose(); + Assert.True(h.IsClosed); + Assert.True(h.IsInvalid); + } + + [Theory] + [InlineData(1)] + [InlineData(13)] + public void CreateDispose_TracksAllocations(int count) + { + RemoteExecutor.Invoke(RunTest, count.ToString()).Dispose(); + + static void RunTest(string countStr) + { + int countInner = int.Parse(countStr); + var l = new List(); + for (int i = 0; i < countInner; i++) + { + Assert.Equal(i, UnmanagedMemoryHandle.TotalOutstandingHandles); + var h = new UnmanagedMemoryHandle(42); + Assert.Equal(i + 1, UnmanagedMemoryHandle.TotalOutstandingHandles); + l.Add(h); + } + + for (int i = 0; i < countInner; i++) + { + Assert.Equal(countInner - i, UnmanagedMemoryHandle.TotalOutstandingHandles); + l[i].Dispose(); + Assert.Equal(countInner - i - 1, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + } + } + + [Theory] + [InlineData(2)] + [InlineData(12)] + public void CreateFinalize_TracksAllocations(int count) + { + RemoteExecutor.Invoke(RunTest, count.ToString()).Dispose(); + + static void RunTest(string countStr) + { + int countInner = int.Parse(countStr); + List l = FillList(countInner); + + l.RemoveRange(0, l.Count / 2); + + GC.Collect(); + GC.WaitForPendingFinalizers(); + + Assert.Equal(countInner / 2, l.Count); // This is here to prevent eager finalization of the list's elements + Assert.Equal(countInner / 2, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + + static List FillList(int countInner) + { + var l = new List(); + for (int i = 0; i < countInner; i++) + { + var h = new UnmanagedMemoryHandle(42); + l.Add(h); + } + + return l; + } + } + + [Fact] + public void Resurrect_PreventsFinalization() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + + static void RunTest() + { + AllocateResurrect(); + Assert.Equal(1, UnmanagedMemoryHandle.TotalOutstandingHandles); + GC.Collect(); + GC.WaitForPendingFinalizers(); + Assert.Equal(1, UnmanagedMemoryHandle.TotalOutstandingHandles); + GC.Collect(); + GC.WaitForPendingFinalizers(); + Assert.Equal(1, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + + static void AllocateResurrect() + { + var h = new UnmanagedMemoryHandle(42); + h.Resurrect(); + } + } + + private static UnmanagedMemoryHandle resurrectedHandle; + + private class HandleOwner + { + private UnmanagedMemoryHandle handle; + + public HandleOwner(UnmanagedMemoryHandle handle) => this.handle = handle; + + ~HandleOwner() + { + resurrectedHandle = this.handle; + this.handle.Resurrect(); + } + } + + [Fact] + public void AssignedToNewOwner_ReRegistersForFinalization() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + + static void RunTest() + { + AllocateAndForget(); + Assert.Equal(1, UnmanagedMemoryHandle.TotalOutstandingHandles); + GC.Collect(); + GC.WaitForPendingFinalizers(); + VerifyResurrectedHandle(true); + GC.Collect(); + GC.WaitForPendingFinalizers(); + VerifyResurrectedHandle(false); + GC.Collect(); + GC.WaitForPendingFinalizers(); + + Assert.Equal(0, UnmanagedMemoryHandle.TotalOutstandingHandles); + } + + static void AllocateAndForget() + { + _ = new HandleOwner(new UnmanagedMemoryHandle(42)); + } + + static void VerifyResurrectedHandle(bool reAssign) + { + Assert.NotNull(resurrectedHandle); + Assert.Equal(1, UnmanagedMemoryHandle.TotalOutstandingHandles); + Assert.False(resurrectedHandle.IsClosed); + Assert.False(resurrectedHandle.IsInvalid); + resurrectedHandle.AssignedToNewOwner(); + if (reAssign) + { + _ = new HandleOwner(resurrectedHandle); + } + + resurrectedHandle = null; + } + } + } +} diff --git a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs index 015b3617bc..23e0ba5f14 100644 --- a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs +++ b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs @@ -227,13 +227,11 @@ namespace SixLabors.ImageSharp.Tests.Memory int actual1 = dest.GetRowSpan(0)[0]; int actual2 = dest.GetRowSpan(0)[0]; int actual3 = dest.GetSafeRowMemory(0).Span[0]; - int actual4 = dest.GetFastRowMemory(0).Span[0]; int actual5 = dest[0, 0]; Assert.Equal(1, actual1); Assert.Equal(1, actual2); Assert.Equal(1, actual3); - Assert.Equal(1, actual4); Assert.Equal(1, actual5); } diff --git a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.Allocate.cs b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.Allocate.cs index e9094fcca4..07b99584d5 100644 --- a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.Allocate.cs +++ b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.Allocate.cs @@ -2,10 +2,15 @@ // Licensed under the Apache License, Version 2.0. using System; +using System.Buffers; using System.Collections.Generic; using System.Linq; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.Memory.Internals; using Xunit; +using Xunit.Abstractions; namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers { @@ -39,6 +44,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers { default(S4), 50, 7, 21, 3, 7, 7 }, { default(S4), 50, 7, 23, 4, 7, 2 }, { default(S4), 50, 6, 13, 2, 12, 1 }, + { default(S4), 1024, 20, 800, 4, 240, 80 }, { default(short), 200, 50, 49, 1, 49, 49 }, { default(short), 200, 50, 1, 1, 1, 1 }, @@ -47,7 +53,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers [Theory] [MemberData(nameof(AllocateData))] - public void BufferSizesAreCorrect( + public void Allocate_FromMemoryAllocator_BufferSizesAreCorrect( T dummy, int bufferCapacity, int bufferAlignment, @@ -63,6 +69,94 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers using var g = MemoryGroup.Allocate(this.MemoryAllocator, totalLength, bufferAlignment); // Assert: + ValidateAllocateMemoryGroup(expectedNumberOfBuffers, expectedBufferSize, expectedSizeOfLastBuffer, g); + } + + [Theory] + [MemberData(nameof(AllocateData))] + public void Allocate_FromPool_BufferSizesAreCorrect( + T dummy, + int bufferCapacity, + int bufferAlignment, + long totalLength, + int expectedNumberOfBuffers, + int expectedBufferSize, + int expectedSizeOfLastBuffer) + where T : struct + { + if (totalLength == 0) + { + // Invalid case for UniformByteArrayPool allocations + return; + } + + var pool = new UniformUnmanagedMemoryPool(bufferCapacity, expectedNumberOfBuffers); + + // Act: + using var g = MemoryGroup.Allocate(pool, totalLength, bufferAlignment); + + // Assert: + ValidateAllocateMemoryGroup(expectedNumberOfBuffers, expectedBufferSize, expectedSizeOfLastBuffer, g); + } + + private static unsafe Span GetSpan(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle h) => + new Span((void*)h.DangerousGetHandle(), pool.BufferLength); + + [Theory] + [InlineData(AllocationOptions.None)] + [InlineData(AllocationOptions.Clean)] + public void Allocate_FromPool_AllocationOptionsAreApplied(AllocationOptions options) + { + var pool = new UniformUnmanagedMemoryPool(10, 5); + UnmanagedMemoryHandle[] buffers = pool.Rent(5); + foreach (UnmanagedMemoryHandle b in buffers) + { + GetSpan(pool, b).Fill(42); + } + + pool.Return(buffers); + + using var g = MemoryGroup.Allocate(pool, 50, 10, options); + Span expected = stackalloc byte[10]; + expected.Fill((byte)(options == AllocationOptions.Clean ? 0 : 42)); + foreach (Memory memory in g) + { + Assert.True(expected.SequenceEqual(memory.Span)); + } + } + + [Theory] + [InlineData(64, 4, 60, 240, false)] + [InlineData(64, 4, 60, 244, true)] + public void Allocate_FromPool_AroundLimit( + int bufferCapacityBytes, + int poolCapacity, + int alignmentBytes, + int requestBytes, + bool shouldReturnNull) + { + var pool = new UniformUnmanagedMemoryPool(bufferCapacityBytes, poolCapacity); + int alignmentElements = alignmentBytes / Unsafe.SizeOf(); + int requestElements = requestBytes / Unsafe.SizeOf(); + + using var g = MemoryGroup.Allocate(pool, requestElements, alignmentElements); + if (shouldReturnNull) + { + Assert.Null(g); + } + else + { + Assert.NotNull(g); + } + } + + internal static void ValidateAllocateMemoryGroup( + int expectedNumberOfBuffers, + int expectedBufferSize, + int expectedSizeOfLastBuffer, + MemoryGroup g) + where T : struct + { Assert.Equal(expectedNumberOfBuffers, g.Count); if (expectedBufferSize >= 0) @@ -123,6 +217,31 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers Assert.Equal(expectedBlockCount, this.MemoryAllocator.ReturnLog.Count); Assert.True(bufferHashes.SetEquals(this.MemoryAllocator.ReturnLog.Select(l => l.HashCodeOfBuffer))); } + + [Theory] + [InlineData(128)] + [InlineData(1024)] + public void Allocate_OptionsContiguous_AllocatesContiguousBuffer(int lengthInBytes) + { + this.MemoryAllocator.BufferCapacityInBytes = 256; + int length = lengthInBytes / Unsafe.SizeOf(); + using var g = MemoryGroup.Allocate(this.MemoryAllocator, length, 32, AllocationOptions.Contiguous); + Assert.Equal(length, g.BufferLength); + Assert.Equal(length, g.TotalLength); + Assert.Equal(1, g.Count); + } } } + + [StructLayout(LayoutKind.Sequential, Size = 5)] + internal struct S5 + { + public override string ToString() => "S5"; + } + + [StructLayout(LayoutKind.Sequential, Size = 4)] + internal struct S4 + { + public override string ToString() => "S4"; + } } diff --git a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs index 3ab5797ddb..a93dbbeb39 100644 --- a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs +++ b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs @@ -229,17 +229,5 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers target[k] = source[k] * 2; } } - - [StructLayout(LayoutKind.Sequential, Size = 5)] - private struct S5 - { - public override string ToString() => "S5"; - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - private struct S4 - { - public override string ToString() => "S4"; - } } } diff --git a/tests/ImageSharp.Tests/Processing/Processors/Dithering/DitherTests.cs b/tests/ImageSharp.Tests/Processing/Processors/Dithering/DitherTests.cs index 0465cae940..7ae85c1974 100644 --- a/tests/ImageSharp.Tests/Processing/Processors/Dithering/DitherTests.cs +++ b/tests/ImageSharp.Tests/Processing/Processors/Dithering/DitherTests.cs @@ -61,7 +61,7 @@ namespace SixLabors.ImageSharp.Tests.Processing.Processors.Dithering /// but it is very different because of floating point inaccuracies. /// private static readonly bool SkipAllDitherTests = - !TestEnvironment.Is64BitProcess && string.IsNullOrEmpty(TestEnvironment.NetCoreVersion); + !TestEnvironment.Is64BitProcess && TestEnvironment.NetCoreVersion == null; [Theory] [WithFile(TestImages.Png.CalliphoraPartial, PixelTypes.Rgba32)] diff --git a/tests/ImageSharp.Tests/Processing/Processors/Transforms/ResizeTests.cs b/tests/ImageSharp.Tests/Processing/Processors/Transforms/ResizeTests.cs index 42cf1e3c12..93e68a3e04 100644 --- a/tests/ImageSharp.Tests/Processing/Processors/Transforms/ResizeTests.cs +++ b/tests/ImageSharp.Tests/Processing/Processors/Transforms/ResizeTests.cs @@ -342,7 +342,7 @@ namespace SixLabors.ImageSharp.Tests.Processing.Processors.Transforms // Resize_WorksWithAllResamplers_TestPattern301x1180_NearestNeighbor-300x480.png // TODO: Should we investigate this? bool allowHigherInaccuracy = !TestEnvironment.Is64BitProcess - && string.IsNullOrEmpty(TestEnvironment.NetCoreVersion) + && TestEnvironment.NetCoreVersion == null && sampler is NearestNeighborResampler; var comparer = ImageComparer.TolerantPercentage(allowHigherInaccuracy ? 0.3f : 0.017f); diff --git a/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs b/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs index f57c19f12a..4b27360099 100644 --- a/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs +++ b/tests/ImageSharp.Tests/TestUtilities/ImageProviders/FileProvider.cs @@ -8,6 +8,7 @@ using System.IO; using System.Reflection; using System.Threading.Tasks; using SixLabors.ImageSharp.Formats; +using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; using Xunit.Abstractions; @@ -157,7 +158,14 @@ namespace SixLabors.ImageSharp.Tests return this.LoadImage(decoder); } - int bufferCapacity = this.Configuration.MemoryAllocator.GetBufferCapacityInBytes(); + int bufferCapacity = -1; +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + if (this.Configuration.MemoryAllocator is ArrayPoolMemoryAllocator arrayPoolMemoryAllocator) +#pragma warning restore CS0618 + { + bufferCapacity = arrayPoolMemoryAllocator.BufferCapacityInBytes; + } + var key = new Key(this.PixelType, this.FilePath, bufferCapacity, decoder); Image cachedImage = Cache.GetOrAdd(key, _ => this.LoadImage(decoder)); diff --git a/tests/ImageSharp.Tests/TestUtilities/TestEnvironment.cs b/tests/ImageSharp.Tests/TestUtilities/TestEnvironment.cs index b14c2bf782..00633b959e 100644 --- a/tests/ImageSharp.Tests/TestUtilities/TestEnvironment.cs +++ b/tests/ImageSharp.Tests/TestUtilities/TestEnvironment.cs @@ -24,14 +24,14 @@ namespace SixLabors.ImageSharp.Tests private static readonly Lazy SolutionDirectoryFullPathLazy = new Lazy(GetSolutionDirectoryFullPathImpl); - private static readonly Lazy NetCoreVersionLazy = new Lazy(GetNetCoreVersion); + private static readonly Lazy NetCoreVersionLazy = new Lazy(GetNetCoreVersion); static TestEnvironment() => PrepareRemoteExecutor(); /// /// Gets the .NET Core version, if running on .NET Core, otherwise returns an empty string. /// - internal static string NetCoreVersion => NetCoreVersionLazy.Value; + internal static Version NetCoreVersion => NetCoreVersionLazy.Value; // ReSharper disable once InconsistentNaming @@ -118,7 +118,7 @@ namespace SixLabors.ImageSharp.Tests internal static bool Is64BitProcess => IntPtr.Size == 8; - internal static bool IsFramework => string.IsNullOrEmpty(NetCoreVersion); + internal static bool IsFramework => NetCoreVersion == null; /// /// A dummy operation to enforce the execution of the static constructor. @@ -262,17 +262,17 @@ namespace SixLabors.ImageSharp.Tests /// Solution borrowed from: /// https://github.com/dotnet/BenchmarkDotNet/issues/448#issuecomment-308424100 /// - private static string GetNetCoreVersion() + private static Version GetNetCoreVersion() { Assembly assembly = typeof(System.Runtime.GCSettings).GetTypeInfo().Assembly; string[] assemblyPath = assembly.Location.Split(new[] { '/', '\\' }, StringSplitOptions.RemoveEmptyEntries); int netCoreAppIndex = Array.IndexOf(assemblyPath, "Microsoft.NETCore.App"); if (netCoreAppIndex > 0 && netCoreAppIndex < assemblyPath.Length - 2) { - return assemblyPath[netCoreAppIndex + 1]; + return Version.Parse(assemblyPath[netCoreAppIndex + 1]); } - return string.Empty; + return null; } } } diff --git a/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs b/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs index 26378796bd..3338070795 100644 --- a/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs +++ b/tests/ImageSharp.Tests/TestUtilities/TestImageExtensions.cs @@ -416,6 +416,27 @@ namespace SixLabors.ImageSharp.Tests } } + public static void CompareBuffers(Buffer2D expected, Buffer2D actual) + where T : struct, IEquatable + { + Assert.True(expected.Size() == actual.Size(), "Buffer sizes are not equal!"); + + for (int y = 0; y < expected.Height; y++) + { + Span expectedRow = expected.GetRowSpan(y); + Span actualRow = actual.GetRowSpan(y); + for (int x = 0; x < expectedRow.Length; x++) + { + T expectedVal = expectedRow[x]; + T actualVal = actualRow[x]; + + Assert.True( + expectedVal.Equals(actualVal), + $"Buffers differ at position ({x},{y})! Expected: {expectedVal} | Actual: {actualVal}"); + } + } + } + /// /// All pixels in all frames should be exactly equal to 'expectedPixel'. /// @@ -663,7 +684,11 @@ namespace SixLabors.ImageSharp.Tests this TestImageProvider provider) where TPixel : unmanaged, IPixel { - var allocator = (ArrayPoolMemoryAllocator)provider.Configuration.MemoryAllocator; + // TODO: Use a test-only allocator for this. +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete + var allocator = ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore + provider.Configuration.MemoryAllocator = allocator; return new AllocatorBufferCapacityConfigurator(allocator, Unsafe.SizeOf()); } @@ -746,6 +771,7 @@ namespace SixLabors.ImageSharp.Tests internal class AllocatorBufferCapacityConfigurator { +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete private readonly ArrayPoolMemoryAllocator allocator; private readonly int pixelSizeInBytes; @@ -754,6 +780,7 @@ namespace SixLabors.ImageSharp.Tests this.allocator = allocator; this.pixelSizeInBytes = pixelSizeInBytes; } +#pragma warning restore CS0618 public void InBytes(int totalBytes) => this.allocator.BufferCapacityInBytes = totalBytes; diff --git a/tests/ImageSharp.Tests/TestUtilities/TestMemoryAllocator.cs b/tests/ImageSharp.Tests/TestUtilities/TestMemoryAllocator.cs index ab9611d2fb..89bc787bf4 100644 --- a/tests/ImageSharp.Tests/TestUtilities/TestMemoryAllocator.cs +++ b/tests/ImageSharp.Tests/TestUtilities/TestMemoryAllocator.cs @@ -40,12 +40,6 @@ namespace SixLabors.ImageSharp.Tests.Memory return new BasicArrayBuffer(array, length, this); } - public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None) - { - byte[] array = this.AllocateArray(length, options); - return new ManagedByteBuffer(array, this); - } - private T[] AllocateArray(int length, AllocationOptions options) where T : struct { @@ -171,7 +165,7 @@ namespace SixLabors.ImageSharp.Tests.Memory } } - private class ManagedByteBuffer : BasicArrayBuffer, IManagedByteBuffer + private class ManagedByteBuffer : BasicArrayBuffer, IMemoryOwner { public ManagedByteBuffer(byte[] array, TestMemoryAllocator allocator) : base(array, allocator) diff --git a/tests/ImageSharp.Tests/TestUtilities/TestUtils.cs b/tests/ImageSharp.Tests/TestUtilities/TestUtils.cs index 32b5eaf182..8c03e8deb1 100644 --- a/tests/ImageSharp.Tests/TestUtilities/TestUtils.cs +++ b/tests/ImageSharp.Tests/TestUtilities/TestUtils.cs @@ -165,7 +165,10 @@ namespace SixLabors.ImageSharp.Tests int width = expected.Width; expected.Mutate(process); + // TODO: Use a test-only allocator for this +#pragma warning disable CS0618 // 'ArrayPoolMemoryAllocator' is obsolete var allocator = ArrayPoolMemoryAllocator.CreateDefault(); +#pragma warning restore CS0618 provider.Configuration.MemoryAllocator = allocator; allocator.BufferCapacityInBytes = bufferCapacityInPixelRows * width * Unsafe.SizeOf();