mirror of https://github.com/SixLabors/ImageSharp
Dmitry Pentin
4 years ago
192 changed files with 5919 additions and 1808 deletions
@ -1,21 +1,24 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Options for allocating buffers.
|
|||
/// </summary>
|
|||
[Flags] |
|||
public enum AllocationOptions |
|||
{ |
|||
/// <summary>
|
|||
/// Indicates that the buffer should just be allocated.
|
|||
/// </summary>
|
|||
None, |
|||
None = 0, |
|||
|
|||
/// <summary>
|
|||
/// Indicates that the allocated buffer should be cleaned following allocation.
|
|||
/// </summary>
|
|||
Clean |
|||
Clean = 1 |
|||
} |
|||
} |
|||
|
|||
@ -0,0 +1,10 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
internal static class AllocationOptionsExtensions |
|||
{ |
|||
public static bool Has(this AllocationOptions options, AllocationOptions flag) => (options & flag) == flag; |
|||
} |
|||
} |
|||
@ -1,105 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using SixLabors.ImageSharp.Memory.Internals; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Contains <see cref="Buffer{T}"/> and <see cref="ManagedByteBuffer"/>.
|
|||
/// </summary>
|
|||
public partial class ArrayPoolMemoryAllocator |
|||
{ |
|||
/// <summary>
|
|||
/// The buffer implementation of <see cref="ArrayPoolMemoryAllocator"/>.
|
|||
/// </summary>
|
|||
private class Buffer<T> : ManagedBufferBase<T> |
|||
where T : struct |
|||
{ |
|||
/// <summary>
|
|||
/// The length of the buffer.
|
|||
/// </summary>
|
|||
private readonly int length; |
|||
|
|||
/// <summary>
|
|||
/// A weak reference to the source pool.
|
|||
/// </summary>
|
|||
/// <remarks>
|
|||
/// By using a weak reference here, we are making sure that array pools and their retained arrays are always GC-ed
|
|||
/// after a call to <see cref="ArrayPoolMemoryAllocator.ReleaseRetainedResources"/>, regardless of having buffer instances still being in use.
|
|||
/// </remarks>
|
|||
private WeakReference<ArrayPool<byte>> sourcePoolReference; |
|||
|
|||
public Buffer(byte[] data, int length, ArrayPool<byte> sourcePool) |
|||
{ |
|||
this.Data = data; |
|||
this.length = length; |
|||
this.sourcePoolReference = new WeakReference<ArrayPool<byte>>(sourcePool); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Gets the buffer as a byte array.
|
|||
/// </summary>
|
|||
protected byte[] Data { get; private set; } |
|||
|
|||
/// <inheritdoc />
|
|||
public override Span<T> GetSpan() |
|||
{ |
|||
if (this.Data is null) |
|||
{ |
|||
ThrowObjectDisposedException(); |
|||
} |
|||
#if SUPPORTS_CREATESPAN
|
|||
ref byte r0 = ref MemoryMarshal.GetReference<byte>(this.Data); |
|||
return MemoryMarshal.CreateSpan(ref Unsafe.As<byte, T>(ref r0), this.length); |
|||
#else
|
|||
return MemoryMarshal.Cast<byte, T>(this.Data.AsSpan()).Slice(0, this.length); |
|||
#endif
|
|||
|
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
if (!disposing || this.Data is null || this.sourcePoolReference is null) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
if (this.sourcePoolReference.TryGetTarget(out ArrayPool<byte> pool)) |
|||
{ |
|||
pool.Return(this.Data); |
|||
} |
|||
|
|||
this.sourcePoolReference = null; |
|||
this.Data = null; |
|||
} |
|||
|
|||
protected override object GetPinnableObject() => this.Data; |
|||
|
|||
[MethodImpl(InliningOptions.ColdPath)] |
|||
private static void ThrowObjectDisposedException() |
|||
{ |
|||
throw new ObjectDisposedException("ArrayPoolMemoryAllocator.Buffer<T>"); |
|||
} |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// The <see cref="IManagedByteBuffer"/> implementation of <see cref="ArrayPoolMemoryAllocator"/>.
|
|||
/// </summary>
|
|||
private sealed class ManagedByteBuffer : Buffer<byte>, IManagedByteBuffer |
|||
{ |
|||
public ManagedByteBuffer(byte[] data, int length, ArrayPool<byte> sourcePool) |
|||
: base(data, length, sourcePool) |
|||
{ |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public byte[] Array => this.Data; |
|||
} |
|||
} |
|||
} |
|||
@ -1,76 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Contains common factory methods and configuration constants.
|
|||
/// </summary>
|
|||
public partial class ArrayPoolMemoryAllocator |
|||
{ |
|||
/// <summary>
|
|||
/// The default value for: maximum size of pooled arrays in bytes.
|
|||
/// Currently set to 24MB, which is equivalent to 8 megapixels of raw RGBA32 data.
|
|||
/// </summary>
|
|||
internal const int DefaultMaxPooledBufferSizeInBytes = 24 * 1024 * 1024; |
|||
|
|||
/// <summary>
|
|||
/// The value for: The threshold to pool arrays in <see cref="largeArrayPool"/> which has less buckets for memory safety.
|
|||
/// </summary>
|
|||
private const int DefaultBufferSelectorThresholdInBytes = 8 * 1024 * 1024; |
|||
|
|||
/// <summary>
|
|||
/// The default bucket count for <see cref="largeArrayPool"/>.
|
|||
/// </summary>
|
|||
private const int DefaultLargePoolBucketCount = 6; |
|||
|
|||
/// <summary>
|
|||
/// The default bucket count for <see cref="normalArrayPool"/>.
|
|||
/// </summary>
|
|||
private const int DefaultNormalPoolBucketCount = 16; |
|||
|
|||
// TODO: This value should be determined by benchmarking
|
|||
private const int DefaultBufferCapacityInBytes = int.MaxValue / 4; |
|||
|
|||
/// <summary>
|
|||
/// This is the default. Should be good for most use cases.
|
|||
/// </summary>
|
|||
/// <returns>The memory manager.</returns>
|
|||
public static ArrayPoolMemoryAllocator CreateDefault() |
|||
{ |
|||
return new ArrayPoolMemoryAllocator( |
|||
DefaultMaxPooledBufferSizeInBytes, |
|||
DefaultBufferSelectorThresholdInBytes, |
|||
DefaultLargePoolBucketCount, |
|||
DefaultNormalPoolBucketCount, |
|||
DefaultBufferCapacityInBytes); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// For environments with very limited memory capabilities, only small buffers like image rows are pooled.
|
|||
/// </summary>
|
|||
/// <returns>The memory manager.</returns>
|
|||
public static ArrayPoolMemoryAllocator CreateWithMinimalPooling() |
|||
{ |
|||
return new ArrayPoolMemoryAllocator(64 * 1024, 32 * 1024, 8, 24); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// For environments with limited memory capabilities, only small array requests are pooled, which can result in reduced throughput.
|
|||
/// </summary>
|
|||
/// <returns>The memory manager.</returns>
|
|||
public static ArrayPoolMemoryAllocator CreateWithModeratePooling() |
|||
{ |
|||
return new ArrayPoolMemoryAllocator(1024 * 1024, 32 * 1024, 16, 24); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// For environments where memory capabilities are not an issue, the maximum amount of array requests are pooled which results in optimal throughput.
|
|||
/// </summary>
|
|||
/// <returns>The memory manager.</returns>
|
|||
public static ArrayPoolMemoryAllocator CreateWithAggressivePooling() |
|||
{ |
|||
return new ArrayPoolMemoryAllocator(128 * 1024 * 1024, 32 * 1024 * 1024, 16, 32); |
|||
} |
|||
} |
|||
} |
|||
@ -1,185 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using System.Runtime.CompilerServices; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Implements <see cref="MemoryAllocator"/> by allocating memory from <see cref="ArrayPool{T}"/>.
|
|||
/// </summary>
|
|||
public sealed partial class ArrayPoolMemoryAllocator : MemoryAllocator |
|||
{ |
|||
private readonly int maxArraysPerBucketNormalPool; |
|||
|
|||
private readonly int maxArraysPerBucketLargePool; |
|||
|
|||
/// <summary>
|
|||
/// The <see cref="ArrayPool{T}"/> for small-to-medium buffers which is not kept clean.
|
|||
/// </summary>
|
|||
private ArrayPool<byte> normalArrayPool; |
|||
|
|||
/// <summary>
|
|||
/// The <see cref="ArrayPool{T}"/> for huge buffers, which is not kept clean.
|
|||
/// </summary>
|
|||
private ArrayPool<byte> largeArrayPool; |
|||
|
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="ArrayPoolMemoryAllocator"/> class.
|
|||
/// </summary>
|
|||
public ArrayPoolMemoryAllocator() |
|||
: this(DefaultMaxPooledBufferSizeInBytes, DefaultBufferSelectorThresholdInBytes) |
|||
{ |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="ArrayPoolMemoryAllocator"/> class.
|
|||
/// </summary>
|
|||
/// <param name="maxPoolSizeInBytes">The maximum size of pooled arrays. Arrays over the thershold are gonna be always allocated.</param>
|
|||
public ArrayPoolMemoryAllocator(int maxPoolSizeInBytes) |
|||
: this(maxPoolSizeInBytes, GetLargeBufferThresholdInBytes(maxPoolSizeInBytes)) |
|||
{ |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="ArrayPoolMemoryAllocator"/> class.
|
|||
/// </summary>
|
|||
/// <param name="maxPoolSizeInBytes">The maximum size of pooled arrays. Arrays over the thershold are gonna be always allocated.</param>
|
|||
/// <param name="poolSelectorThresholdInBytes">Arrays over this threshold will be pooled in <see cref="largeArrayPool"/> which has less buckets for memory safety.</param>
|
|||
public ArrayPoolMemoryAllocator(int maxPoolSizeInBytes, int poolSelectorThresholdInBytes) |
|||
: this(maxPoolSizeInBytes, poolSelectorThresholdInBytes, DefaultLargePoolBucketCount, DefaultNormalPoolBucketCount) |
|||
{ |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="ArrayPoolMemoryAllocator"/> class.
|
|||
/// </summary>
|
|||
/// <param name="maxPoolSizeInBytes">The maximum size of pooled arrays. Arrays over the thershold are gonna be always allocated.</param>
|
|||
/// <param name="poolSelectorThresholdInBytes">The threshold to pool arrays in <see cref="largeArrayPool"/> which has less buckets for memory safety.</param>
|
|||
/// <param name="maxArraysPerBucketLargePool">Max arrays per bucket for the large array pool.</param>
|
|||
/// <param name="maxArraysPerBucketNormalPool">Max arrays per bucket for the normal array pool.</param>
|
|||
public ArrayPoolMemoryAllocator( |
|||
int maxPoolSizeInBytes, |
|||
int poolSelectorThresholdInBytes, |
|||
int maxArraysPerBucketLargePool, |
|||
int maxArraysPerBucketNormalPool) |
|||
: this( |
|||
maxPoolSizeInBytes, |
|||
poolSelectorThresholdInBytes, |
|||
maxArraysPerBucketLargePool, |
|||
maxArraysPerBucketNormalPool, |
|||
DefaultBufferCapacityInBytes) |
|||
{ |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="ArrayPoolMemoryAllocator"/> class.
|
|||
/// </summary>
|
|||
/// <param name="maxPoolSizeInBytes">The maximum size of pooled arrays. Arrays over the thershold are gonna be always allocated.</param>
|
|||
/// <param name="poolSelectorThresholdInBytes">The threshold to pool arrays in <see cref="largeArrayPool"/> which has less buckets for memory safety.</param>
|
|||
/// <param name="maxArraysPerBucketLargePool">Max arrays per bucket for the large array pool.</param>
|
|||
/// <param name="maxArraysPerBucketNormalPool">Max arrays per bucket for the normal array pool.</param>
|
|||
/// <param name="bufferCapacityInBytes">The length of the largest contiguous buffer that can be handled by this allocator instance.</param>
|
|||
public ArrayPoolMemoryAllocator( |
|||
int maxPoolSizeInBytes, |
|||
int poolSelectorThresholdInBytes, |
|||
int maxArraysPerBucketLargePool, |
|||
int maxArraysPerBucketNormalPool, |
|||
int bufferCapacityInBytes) |
|||
{ |
|||
Guard.MustBeGreaterThan(maxPoolSizeInBytes, 0, nameof(maxPoolSizeInBytes)); |
|||
Guard.MustBeLessThanOrEqualTo(poolSelectorThresholdInBytes, maxPoolSizeInBytes, nameof(poolSelectorThresholdInBytes)); |
|||
|
|||
this.MaxPoolSizeInBytes = maxPoolSizeInBytes; |
|||
this.PoolSelectorThresholdInBytes = poolSelectorThresholdInBytes; |
|||
this.BufferCapacityInBytes = bufferCapacityInBytes; |
|||
this.maxArraysPerBucketLargePool = maxArraysPerBucketLargePool; |
|||
this.maxArraysPerBucketNormalPool = maxArraysPerBucketNormalPool; |
|||
|
|||
this.InitArrayPools(); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Gets the maximum size of pooled arrays in bytes.
|
|||
/// </summary>
|
|||
public int MaxPoolSizeInBytes { get; } |
|||
|
|||
/// <summary>
|
|||
/// Gets the threshold to pool arrays in <see cref="largeArrayPool"/> which has less buckets for memory safety.
|
|||
/// </summary>
|
|||
public int PoolSelectorThresholdInBytes { get; } |
|||
|
|||
/// <summary>
|
|||
/// Gets the length of the largest contiguous buffer that can be handled by this allocator instance.
|
|||
/// </summary>
|
|||
public int BufferCapacityInBytes { get; internal set; } // Setter is internal for easy configuration in tests
|
|||
|
|||
/// <inheritdoc />
|
|||
public override void ReleaseRetainedResources() |
|||
{ |
|||
this.InitArrayPools(); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected internal override int GetBufferCapacityInBytes() => this.BufferCapacityInBytes; |
|||
|
|||
/// <inheritdoc />
|
|||
public override IMemoryOwner<T> Allocate<T>(int length, AllocationOptions options = AllocationOptions.None) |
|||
{ |
|||
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); |
|||
int itemSizeBytes = Unsafe.SizeOf<T>(); |
|||
int bufferSizeInBytes = length * itemSizeBytes; |
|||
|
|||
ArrayPool<byte> pool = this.GetArrayPool(bufferSizeInBytes); |
|||
byte[] byteArray = pool.Rent(bufferSizeInBytes); |
|||
|
|||
var buffer = new Buffer<T>(byteArray, length, pool); |
|||
if (options == AllocationOptions.Clean) |
|||
{ |
|||
buffer.GetSpan().Clear(); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public override IManagedByteBuffer AllocateManagedByteBuffer(int length, AllocationOptions options = AllocationOptions.None) |
|||
{ |
|||
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); |
|||
|
|||
ArrayPool<byte> pool = this.GetArrayPool(length); |
|||
byte[] byteArray = pool.Rent(length); |
|||
|
|||
var buffer = new ManagedByteBuffer(byteArray, length, pool); |
|||
if (options == AllocationOptions.Clean) |
|||
{ |
|||
buffer.GetSpan().Clear(); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
|
|||
private static int GetLargeBufferThresholdInBytes(int maxPoolSizeInBytes) |
|||
{ |
|||
return maxPoolSizeInBytes / 4; |
|||
} |
|||
|
|||
[MethodImpl(InliningOptions.ColdPath)] |
|||
private static void ThrowInvalidAllocationException<T>(int length, int max) => |
|||
throw new InvalidMemoryOperationException( |
|||
$"Requested allocation: '{length}' elements of '{typeof(T).Name}' is over the capacity in bytes '{max}' of the MemoryAllocator."); |
|||
|
|||
private ArrayPool<byte> GetArrayPool(int bufferSizeInBytes) |
|||
{ |
|||
return bufferSizeInBytes <= this.PoolSelectorThresholdInBytes ? this.normalArrayPool : this.largeArrayPool; |
|||
} |
|||
|
|||
private void InitArrayPools() |
|||
{ |
|||
this.largeArrayPool = ArrayPool<byte>.Create(this.MaxPoolSizeInBytes, this.maxArraysPerBucketLargePool); |
|||
this.normalArrayPool = ArrayPool<byte>.Create(this.PoolSelectorThresholdInBytes, this.maxArraysPerBucketNormalPool); |
|||
} |
|||
} |
|||
} |
|||
@ -1,18 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System.Buffers; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Represents a byte buffer backed by a managed array. Useful for interop with classic .NET API-s.
|
|||
/// </summary>
|
|||
public interface IManagedByteBuffer : IMemoryOwner<byte> |
|||
{ |
|||
/// <summary>
|
|||
/// Gets the managed array backing this buffer instance.
|
|||
/// </summary>
|
|||
byte[] Array { get; } |
|||
} |
|||
} |
|||
@ -1,20 +0,0 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Provides an <see cref="IManagedByteBuffer"/> based on <see cref="BasicArrayBuffer{T}"/>.
|
|||
/// </summary>
|
|||
internal sealed class BasicByteBuffer : BasicArrayBuffer<byte>, IManagedByteBuffer |
|||
{ |
|||
/// <summary>
|
|||
/// Initializes a new instance of the <see cref="BasicByteBuffer"/> class.
|
|||
/// </summary>
|
|||
/// <param name="array">The byte array.</param>
|
|||
internal BasicByteBuffer(byte[] array) |
|||
: base(array) |
|||
{ |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,115 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
// Port of BCL internal utility:
|
|||
// https://github.com/dotnet/runtime/blob/57bfe474518ab5b7cfe6bf7424a79ce3af9d6657/src/libraries/System.Private.CoreLib/src/System/Gen2GcCallback.cs
|
|||
#if NETCOREAPP3_1_OR_GREATER
|
|||
using System; |
|||
using System.Runtime.ConstrainedExecution; |
|||
using System.Runtime.InteropServices; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Schedules a callback roughly every gen 2 GC (you may see a Gen 0 an Gen 1 but only once)
|
|||
/// (We can fix this by capturing the Gen 2 count at startup and testing, but I mostly don't care)
|
|||
/// </summary>
|
|||
internal sealed class Gen2GcCallback : CriticalFinalizerObject |
|||
{ |
|||
private readonly Func<bool> callback0; |
|||
private readonly Func<object, bool> callback1; |
|||
private GCHandle weakTargetObj; |
|||
|
|||
private Gen2GcCallback(Func<bool> callback) |
|||
{ |
|||
this.callback0 = callback; |
|||
} |
|||
|
|||
private Gen2GcCallback(Func<object, bool> callback, object targetObj) |
|||
{ |
|||
this.callback1 = callback; |
|||
this.weakTargetObj = GCHandle.Alloc(targetObj, GCHandleType.Weak); |
|||
} |
|||
|
|||
~Gen2GcCallback() |
|||
{ |
|||
if (this.weakTargetObj.IsAllocated) |
|||
{ |
|||
// Check to see if the target object is still alive.
|
|||
object targetObj = this.weakTargetObj.Target; |
|||
if (targetObj == null) |
|||
{ |
|||
// The target object is dead, so this callback object is no longer needed.
|
|||
this.weakTargetObj.Free(); |
|||
return; |
|||
} |
|||
|
|||
// Execute the callback method.
|
|||
try |
|||
{ |
|||
if (!this.callback1(targetObj)) |
|||
{ |
|||
// If the callback returns false, this callback object is no longer needed.
|
|||
this.weakTargetObj.Free(); |
|||
return; |
|||
} |
|||
} |
|||
catch |
|||
{ |
|||
// Ensure that we still get a chance to resurrect this object, even if the callback throws an exception.
|
|||
#if DEBUG
|
|||
// Except in DEBUG, as we really shouldn't be hitting any exceptions here.
|
|||
throw; |
|||
#endif
|
|||
} |
|||
} |
|||
else |
|||
{ |
|||
// Execute the callback method.
|
|||
try |
|||
{ |
|||
if (!this.callback0()) |
|||
{ |
|||
// If the callback returns false, this callback object is no longer needed.
|
|||
return; |
|||
} |
|||
} |
|||
catch |
|||
{ |
|||
// Ensure that we still get a chance to resurrect this object, even if the callback throws an exception.
|
|||
#if DEBUG
|
|||
// Except in DEBUG, as we really shouldn't be hitting any exceptions here.
|
|||
throw; |
|||
#endif
|
|||
} |
|||
} |
|||
|
|||
// Resurrect ourselves by re-registering for finalization.
|
|||
GC.ReRegisterForFinalize(this); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Schedule 'callback' to be called in the next GC. If the callback returns true it is
|
|||
/// rescheduled for the next Gen 2 GC. Otherwise the callbacks stop.
|
|||
/// </summary>
|
|||
public static void Register(Func<bool> callback) |
|||
{ |
|||
// Create a unreachable object that remembers the callback function and target object.
|
|||
_ = new Gen2GcCallback(callback); |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Schedule 'callback' to be called in the next GC. If the callback returns true it is
|
|||
/// rescheduled for the next Gen 2 GC. Otherwise the callbacks stop.
|
|||
///
|
|||
/// NOTE: This callback will be kept alive until either the callback function returns false,
|
|||
/// or the target object dies.
|
|||
/// </summary>
|
|||
public static void Register(Func<object, bool> callback, object targetObj) |
|||
{ |
|||
// Create a unreachable object that remembers the callback function and target object.
|
|||
_ = new Gen2GcCallback(callback, targetObj); |
|||
} |
|||
} |
|||
} |
|||
#endif
|
|||
@ -0,0 +1,21 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Defines an common interface for ref-counted objects.
|
|||
/// </summary>
|
|||
internal interface IRefCounted |
|||
{ |
|||
/// <summary>
|
|||
/// Increments the reference counter.
|
|||
/// </summary>
|
|||
void AddRef(); |
|||
|
|||
/// <summary>
|
|||
/// Decrements the reference counter.
|
|||
/// </summary>
|
|||
void ReleaseRef(); |
|||
} |
|||
} |
|||
@ -0,0 +1,56 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Runtime.InteropServices; |
|||
using System.Threading; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Implements reference counting lifetime guard mechanism similar to the one provided by <see cref="SafeHandle"/>,
|
|||
/// but without the restriction of the guarded object being a handle.
|
|||
/// </summary>
|
|||
internal abstract class RefCountedLifetimeGuard : IDisposable |
|||
{ |
|||
private int refCount = 1; |
|||
private int disposed; |
|||
private int released; |
|||
|
|||
~RefCountedLifetimeGuard() |
|||
{ |
|||
Interlocked.Exchange(ref this.disposed, 1); |
|||
this.ReleaseRef(); |
|||
} |
|||
|
|||
public bool IsDisposed => this.disposed == 1; |
|||
|
|||
public void AddRef() => Interlocked.Increment(ref this.refCount); |
|||
|
|||
public void ReleaseRef() |
|||
{ |
|||
Interlocked.Decrement(ref this.refCount); |
|||
if (this.refCount == 0) |
|||
{ |
|||
int wasReleased = Interlocked.Exchange(ref this.released, 1); |
|||
|
|||
if (wasReleased == 0) |
|||
{ |
|||
this.Release(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
int wasDisposed = Interlocked.Exchange(ref this.disposed, 1); |
|||
if (wasDisposed == 0) |
|||
{ |
|||
this.ReleaseRef(); |
|||
GC.SuppressFinalize(this); |
|||
} |
|||
} |
|||
|
|||
protected abstract void Release(); |
|||
} |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using System.Diagnostics; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
internal class SharedArrayPoolBuffer<T> : ManagedBufferBase<T>, IRefCounted |
|||
where T : struct |
|||
{ |
|||
private readonly int lengthInBytes; |
|||
private byte[] array; |
|||
private LifetimeGuard lifetimeGuard; |
|||
|
|||
public SharedArrayPoolBuffer(int lengthInElements) |
|||
{ |
|||
this.lengthInBytes = lengthInElements * Unsafe.SizeOf<T>(); |
|||
this.array = ArrayPool<byte>.Shared.Rent(this.lengthInBytes); |
|||
this.lifetimeGuard = new LifetimeGuard(this.array); |
|||
} |
|||
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
if (this.array == null) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
this.lifetimeGuard.Dispose(); |
|||
this.array = null; |
|||
} |
|||
|
|||
public override Span<T> GetSpan() |
|||
{ |
|||
this.CheckDisposed(); |
|||
return MemoryMarshal.Cast<byte, T>(this.array.AsSpan(0, this.lengthInBytes)); |
|||
} |
|||
|
|||
protected override object GetPinnableObject() => this.array; |
|||
|
|||
public void AddRef() |
|||
{ |
|||
this.CheckDisposed(); |
|||
this.lifetimeGuard.AddRef(); |
|||
} |
|||
|
|||
public void ReleaseRef() => this.lifetimeGuard.ReleaseRef(); |
|||
|
|||
[Conditional("DEBUG")] |
|||
private void CheckDisposed() |
|||
{ |
|||
if (this.array == null) |
|||
{ |
|||
throw new ObjectDisposedException("SharedArrayPoolBuffer"); |
|||
} |
|||
} |
|||
|
|||
private sealed class LifetimeGuard : RefCountedLifetimeGuard |
|||
{ |
|||
private byte[] array; |
|||
|
|||
public LifetimeGuard(byte[] array) => this.array = array; |
|||
|
|||
protected override void Release() |
|||
{ |
|||
// If this is called by a finalizer, we will end storing the first array of this bucket
|
|||
// on the thread local storage of the finalizer thread.
|
|||
// This is not ideal, but subsequent leaks will end up returning arrays to per-cpu buckets,
|
|||
// meaning likely a different bucket than it was rented from,
|
|||
// but this is PROBABLY better than not returning the arrays at all.
|
|||
ArrayPool<byte>.Shared.Return(this.array); |
|||
this.array = null; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
internal partial class UniformUnmanagedMemoryPool |
|||
{ |
|||
public UnmanagedBuffer<T> CreateGuardedBuffer<T>( |
|||
UnmanagedMemoryHandle handle, |
|||
int lengthInElements, |
|||
bool clear) |
|||
where T : struct |
|||
{ |
|||
var buffer = new UnmanagedBuffer<T>(lengthInElements, new ReturnToPoolBufferLifetimeGuard(this, handle)); |
|||
if (clear) |
|||
{ |
|||
buffer.Clear(); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
|
|||
public RefCountedLifetimeGuard CreateGroupLifetimeGuard(UnmanagedMemoryHandle[] handles) => new GroupLifetimeGuard(this, handles); |
|||
|
|||
private sealed class GroupLifetimeGuard : RefCountedLifetimeGuard |
|||
{ |
|||
private readonly UniformUnmanagedMemoryPool pool; |
|||
private readonly UnmanagedMemoryHandle[] handles; |
|||
|
|||
public GroupLifetimeGuard(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle[] handles) |
|||
{ |
|||
this.pool = pool; |
|||
this.handles = handles; |
|||
} |
|||
|
|||
protected override void Release() |
|||
{ |
|||
if (!this.pool.Return(this.handles)) |
|||
{ |
|||
foreach (UnmanagedMemoryHandle handle in this.handles) |
|||
{ |
|||
handle.Free(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private sealed class ReturnToPoolBufferLifetimeGuard : UnmanagedBufferLifetimeGuard |
|||
{ |
|||
private readonly UniformUnmanagedMemoryPool pool; |
|||
|
|||
public ReturnToPoolBufferLifetimeGuard(UniformUnmanagedMemoryPool pool, UnmanagedMemoryHandle handle) |
|||
: base(handle) => |
|||
this.pool = pool; |
|||
|
|||
protected override void Release() |
|||
{ |
|||
if (!this.pool.Return(this.Handle)) |
|||
{ |
|||
this.Handle.Free(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,356 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Diagnostics; |
|||
using System.Threading; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
internal partial class UniformUnmanagedMemoryPool |
|||
#if !NETSTANDARD1_3
|
|||
// In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers,
|
|||
// but we should not rely on this.
|
|||
: System.Runtime.ConstrainedExecution.CriticalFinalizerObject |
|||
#endif
|
|||
{ |
|||
private static int minTrimPeriodMilliseconds = int.MaxValue; |
|||
private static readonly List<WeakReference<UniformUnmanagedMemoryPool>> AllPools = new(); |
|||
private static Timer trimTimer; |
|||
|
|||
private static readonly Stopwatch Stopwatch = Stopwatch.StartNew(); |
|||
|
|||
private readonly TrimSettings trimSettings; |
|||
private readonly UnmanagedMemoryHandle[] buffers; |
|||
private int index; |
|||
private long lastTrimTimestamp; |
|||
private int finalized; |
|||
|
|||
public UniformUnmanagedMemoryPool(int bufferLength, int capacity) |
|||
: this(bufferLength, capacity, TrimSettings.Default) |
|||
{ |
|||
} |
|||
|
|||
public UniformUnmanagedMemoryPool(int bufferLength, int capacity, TrimSettings trimSettings) |
|||
{ |
|||
this.trimSettings = trimSettings; |
|||
this.Capacity = capacity; |
|||
this.BufferLength = bufferLength; |
|||
this.buffers = new UnmanagedMemoryHandle[capacity]; |
|||
|
|||
if (trimSettings.Enabled) |
|||
{ |
|||
UpdateTimer(trimSettings, this); |
|||
#if NETCOREAPP3_1_OR_GREATER
|
|||
Gen2GcCallback.Register(s => ((UniformUnmanagedMemoryPool)s).Trim(), this); |
|||
#endif
|
|||
this.lastTrimTimestamp = Stopwatch.ElapsedMilliseconds; |
|||
} |
|||
} |
|||
|
|||
// We don't want UniformUnmanagedMemoryPool and MemoryAllocator to be IDisposable,
|
|||
// since the types don't really match Disposable semantics.
|
|||
// If a user wants to drop a MemoryAllocator after they finished using it, they should call allocator.ReleaseRetainedResources(),
|
|||
// which normally should free the already returned (!) buffers.
|
|||
// However in case if this doesn't happen, we need the retained memory to be freed by the finalizer.
|
|||
~UniformUnmanagedMemoryPool() |
|||
{ |
|||
Interlocked.Exchange(ref this.finalized, 1); |
|||
this.TrimAll(this.buffers); |
|||
} |
|||
|
|||
public int BufferLength { get; } |
|||
|
|||
public int Capacity { get; } |
|||
|
|||
private bool Finalized => this.finalized == 1; |
|||
|
|||
/// <summary>
|
|||
/// Rent a single buffer. If the pool is full, return <see cref="UnmanagedMemoryHandle.NullHandle"/>.
|
|||
/// </summary>
|
|||
public UnmanagedMemoryHandle Rent() |
|||
{ |
|||
UnmanagedMemoryHandle[] buffersLocal = this.buffers; |
|||
|
|||
// Avoid taking the lock if the pool is is over it's limit:
|
|||
if (this.index == buffersLocal.Length || this.Finalized) |
|||
{ |
|||
return UnmanagedMemoryHandle.NullHandle; |
|||
} |
|||
|
|||
UnmanagedMemoryHandle buffer; |
|||
lock (buffersLocal) |
|||
{ |
|||
// Check again after taking the lock:
|
|||
if (this.index == buffersLocal.Length || this.Finalized) |
|||
{ |
|||
return UnmanagedMemoryHandle.NullHandle; |
|||
} |
|||
|
|||
buffer = buffersLocal[this.index]; |
|||
buffersLocal[this.index++] = default; |
|||
} |
|||
|
|||
if (buffer.IsInvalid) |
|||
{ |
|||
buffer = UnmanagedMemoryHandle.Allocate(this.BufferLength); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
|
|||
/// <summary>
|
|||
/// Rent <paramref name="bufferCount"/> buffers or return 'null' if the pool is full.
|
|||
/// </summary>
|
|||
public UnmanagedMemoryHandle[] Rent(int bufferCount) |
|||
{ |
|||
UnmanagedMemoryHandle[] buffersLocal = this.buffers; |
|||
|
|||
// Avoid taking the lock if the pool is is over it's limit:
|
|||
if (this.index + bufferCount >= buffersLocal.Length + 1 || this.Finalized) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
UnmanagedMemoryHandle[] result; |
|||
lock (buffersLocal) |
|||
{ |
|||
// Check again after taking the lock:
|
|||
if (this.index + bufferCount >= buffersLocal.Length + 1 || this.Finalized) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
result = new UnmanagedMemoryHandle[bufferCount]; |
|||
for (int i = 0; i < bufferCount; i++) |
|||
{ |
|||
result[i] = buffersLocal[this.index]; |
|||
buffersLocal[this.index++] = UnmanagedMemoryHandle.NullHandle; |
|||
} |
|||
} |
|||
|
|||
for (int i = 0; i < result.Length; i++) |
|||
{ |
|||
if (result[i].IsInvalid) |
|||
{ |
|||
result[i] = UnmanagedMemoryHandle.Allocate(this.BufferLength); |
|||
} |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
|
|||
// The Return methods return false if and only if:
|
|||
// (1) More buffers are returned than rented OR
|
|||
// (2) The pool has been finalized.
|
|||
// This is defensive programming, since neither of the cases should happen normally
|
|||
// (case 1 would be a programming mistake in the library, case 2 should be prevented by the CriticalFinalizerObject contract),
|
|||
// so we throw in Debug instead of returning false.
|
|||
// In Release, the caller should Free() the handles if false is returned to avoid memory leaks.
|
|||
public bool Return(UnmanagedMemoryHandle bufferHandle) |
|||
{ |
|||
Guard.IsTrue(bufferHandle.IsValid, nameof(bufferHandle), "Returning NullHandle to the pool is not allowed."); |
|||
lock (this.buffers) |
|||
{ |
|||
if (this.Finalized || this.index == 0) |
|||
{ |
|||
this.DebugThrowInvalidReturn(); |
|||
return false; |
|||
} |
|||
|
|||
this.buffers[--this.index] = bufferHandle; |
|||
} |
|||
|
|||
return true; |
|||
} |
|||
|
|||
public bool Return(Span<UnmanagedMemoryHandle> bufferHandles) |
|||
{ |
|||
lock (this.buffers) |
|||
{ |
|||
if (this.Finalized || this.index - bufferHandles.Length + 1 <= 0) |
|||
{ |
|||
this.DebugThrowInvalidReturn(); |
|||
return false; |
|||
} |
|||
|
|||
for (int i = bufferHandles.Length - 1; i >= 0; i--) |
|||
{ |
|||
ref UnmanagedMemoryHandle h = ref bufferHandles[i]; |
|||
Guard.IsTrue(h.IsValid, nameof(bufferHandles), "Returning NullHandle to the pool is not allowed."); |
|||
this.buffers[--this.index] = h; |
|||
} |
|||
} |
|||
|
|||
return true; |
|||
} |
|||
|
|||
public void Release() |
|||
{ |
|||
lock (this.buffers) |
|||
{ |
|||
for (int i = this.index; i < this.buffers.Length; i++) |
|||
{ |
|||
ref UnmanagedMemoryHandle buffer = ref this.buffers[i]; |
|||
if (buffer.IsInvalid) |
|||
{ |
|||
break; |
|||
} |
|||
|
|||
buffer.Free(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
[Conditional("DEBUG")] |
|||
private void DebugThrowInvalidReturn() |
|||
{ |
|||
if (this.Finalized) |
|||
{ |
|||
throw new ObjectDisposedException( |
|||
nameof(UniformUnmanagedMemoryPool), |
|||
"Invalid handle return to the pool! The pool has been finalized."); |
|||
} |
|||
|
|||
throw new InvalidOperationException( |
|||
"Invalid handle return to the pool! Returning more buffers than rented."); |
|||
} |
|||
|
|||
private static void UpdateTimer(TrimSettings settings, UniformUnmanagedMemoryPool pool) |
|||
{ |
|||
lock (AllPools) |
|||
{ |
|||
AllPools.Add(new WeakReference<UniformUnmanagedMemoryPool>(pool)); |
|||
|
|||
// Invoke the timer callback more frequently, than trimSettings.TrimPeriodMilliseconds.
|
|||
// We are checking in the callback if enough time passed since the last trimming. If not, we do nothing.
|
|||
int period = settings.TrimPeriodMilliseconds / 4; |
|||
if (trimTimer == null) |
|||
{ |
|||
trimTimer = new Timer(_ => TimerCallback(), null, period, period); |
|||
} |
|||
else if (settings.TrimPeriodMilliseconds < minTrimPeriodMilliseconds) |
|||
{ |
|||
trimTimer.Change(period, period); |
|||
} |
|||
|
|||
minTrimPeriodMilliseconds = Math.Min(minTrimPeriodMilliseconds, settings.TrimPeriodMilliseconds); |
|||
} |
|||
} |
|||
|
|||
private static void TimerCallback() |
|||
{ |
|||
lock (AllPools) |
|||
{ |
|||
// Remove lost references from the list:
|
|||
for (int i = AllPools.Count - 1; i >= 0; i--) |
|||
{ |
|||
if (!AllPools[i].TryGetTarget(out _)) |
|||
{ |
|||
AllPools.RemoveAt(i); |
|||
} |
|||
} |
|||
|
|||
foreach (WeakReference<UniformUnmanagedMemoryPool> weakPoolRef in AllPools) |
|||
{ |
|||
if (weakPoolRef.TryGetTarget(out UniformUnmanagedMemoryPool pool)) |
|||
{ |
|||
pool.Trim(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private bool Trim() |
|||
{ |
|||
if (this.Finalized) |
|||
{ |
|||
return false; |
|||
} |
|||
|
|||
UnmanagedMemoryHandle[] buffersLocal = this.buffers; |
|||
|
|||
bool isHighPressure = this.IsHighMemoryPressure(); |
|||
|
|||
if (isHighPressure) |
|||
{ |
|||
this.TrimAll(buffersLocal); |
|||
return true; |
|||
} |
|||
|
|||
long millisecondsSinceLastTrim = Stopwatch.ElapsedMilliseconds - this.lastTrimTimestamp; |
|||
if (millisecondsSinceLastTrim > this.trimSettings.TrimPeriodMilliseconds) |
|||
{ |
|||
return this.TrimLowPressure(buffersLocal); |
|||
} |
|||
|
|||
return true; |
|||
} |
|||
|
|||
private void TrimAll(UnmanagedMemoryHandle[] buffersLocal) |
|||
{ |
|||
lock (buffersLocal) |
|||
{ |
|||
// Trim all:
|
|||
for (int i = this.index; i < buffersLocal.Length && buffersLocal[i].IsValid; i++) |
|||
{ |
|||
buffersLocal[i].Free(); |
|||
} |
|||
} |
|||
} |
|||
|
|||
private bool TrimLowPressure(UnmanagedMemoryHandle[] buffersLocal) |
|||
{ |
|||
lock (buffersLocal) |
|||
{ |
|||
// Count the buffers in the pool:
|
|||
int retainedCount = 0; |
|||
for (int i = this.index; i < buffersLocal.Length && buffersLocal[i].IsValid; i++) |
|||
{ |
|||
retainedCount++; |
|||
} |
|||
|
|||
// Trim 'trimRate' of 'retainedCount':
|
|||
int trimCount = (int)Math.Ceiling(retainedCount * this.trimSettings.Rate); |
|||
int trimStart = this.index + retainedCount - 1; |
|||
int trimStop = this.index + retainedCount - trimCount; |
|||
for (int i = trimStart; i >= trimStop; i--) |
|||
{ |
|||
buffersLocal[i].Free(); |
|||
} |
|||
|
|||
this.lastTrimTimestamp = Stopwatch.ElapsedMilliseconds; |
|||
} |
|||
|
|||
return true; |
|||
} |
|||
|
|||
private bool IsHighMemoryPressure() |
|||
{ |
|||
#if NETCOREAPP3_1_OR_GREATER
|
|||
GCMemoryInfo memoryInfo = GC.GetGCMemoryInfo(); |
|||
return memoryInfo.MemoryLoadBytes >= memoryInfo.HighMemoryLoadThresholdBytes * this.trimSettings.HighPressureThresholdRate; |
|||
#else
|
|||
// We don't have high pressure detection triggering full trimming on other platforms,
|
|||
// to counterpart this, the maximum pool size is small.
|
|||
return false; |
|||
#endif
|
|||
} |
|||
|
|||
public class TrimSettings |
|||
{ |
|||
// Trim half of the retained pool buffers every minute.
|
|||
public int TrimPeriodMilliseconds { get; set; } = 60_000; |
|||
|
|||
public float Rate { get; set; } = 0.5f; |
|||
|
|||
// Be more strict about high pressure on 32 bit.
|
|||
public unsafe float HighPressureThresholdRate { get; set; } = sizeof(IntPtr) == 8 ? 0.9f : 0.6f; |
|||
|
|||
public bool Enabled => this.Rate > 0; |
|||
|
|||
public static TrimSettings Default => new TrimSettings(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,27 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Defines a strategy for managing unmanaged memory ownership.
|
|||
/// </summary>
|
|||
internal abstract class UnmanagedBufferLifetimeGuard : RefCountedLifetimeGuard |
|||
{ |
|||
private UnmanagedMemoryHandle handle; |
|||
|
|||
protected UnmanagedBufferLifetimeGuard(UnmanagedMemoryHandle handle) => this.handle = handle; |
|||
|
|||
public ref UnmanagedMemoryHandle Handle => ref this.handle; |
|||
|
|||
public sealed class FreeHandle : UnmanagedBufferLifetimeGuard |
|||
{ |
|||
public FreeHandle(UnmanagedMemoryHandle handle) |
|||
: base(handle) |
|||
{ |
|||
} |
|||
|
|||
protected override void Release() => this.Handle.Free(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Threading; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Allocates and provides an <see cref="IMemoryOwner{T}"/> implementation giving
|
|||
/// access to unmanaged buffers allocated by <see cref="Marshal.AllocHGlobal(int)"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="T">The element type.</typeparam>
|
|||
internal sealed unsafe class UnmanagedBuffer<T> : MemoryManager<T>, IRefCounted |
|||
where T : struct |
|||
{ |
|||
private readonly int lengthInElements; |
|||
|
|||
private readonly UnmanagedBufferLifetimeGuard lifetimeGuard; |
|||
|
|||
private int disposed; |
|||
|
|||
public UnmanagedBuffer(int lengthInElements, UnmanagedBufferLifetimeGuard lifetimeGuard) |
|||
{ |
|||
DebugGuard.NotNull(lifetimeGuard, nameof(lifetimeGuard)); |
|||
|
|||
this.lengthInElements = lengthInElements; |
|||
this.lifetimeGuard = lifetimeGuard; |
|||
} |
|||
|
|||
private void* Pointer => this.lifetimeGuard.Handle.Pointer; |
|||
|
|||
public override Span<T> GetSpan() |
|||
{ |
|||
DebugGuard.NotDisposed(this.disposed == 1, this.GetType().Name); |
|||
DebugGuard.NotDisposed(this.lifetimeGuard.IsDisposed, this.lifetimeGuard.GetType().Name); |
|||
return new(this.Pointer, this.lengthInElements); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public override MemoryHandle Pin(int elementIndex = 0) |
|||
{ |
|||
DebugGuard.NotDisposed(this.disposed == 1, this.GetType().Name); |
|||
DebugGuard.NotDisposed(this.lifetimeGuard.IsDisposed, this.lifetimeGuard.GetType().Name); |
|||
|
|||
// Will be released in Unpin
|
|||
this.lifetimeGuard.AddRef(); |
|||
|
|||
void* pbData = Unsafe.Add<T>(this.Pointer, elementIndex); |
|||
return new MemoryHandle(pbData, pinnable: this); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected override void Dispose(bool disposing) |
|||
{ |
|||
DebugGuard.IsTrue(disposing, nameof(disposing), "Unmanaged buffers should not have finalizer!"); |
|||
|
|||
if (Interlocked.Exchange(ref this.disposed, 1) == 1) |
|||
{ |
|||
// Already disposed
|
|||
return; |
|||
} |
|||
|
|||
this.lifetimeGuard.Dispose(); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
public override void Unpin() => this.lifetimeGuard.ReleaseRef(); |
|||
|
|||
public void AddRef() => this.lifetimeGuard.AddRef(); |
|||
|
|||
public void ReleaseRef() => this.lifetimeGuard.ReleaseRef(); |
|||
|
|||
public static UnmanagedBuffer<T> Allocate(int lengthInElements) => |
|||
new(lengthInElements, new UnmanagedBufferLifetimeGuard.FreeHandle(UnmanagedMemoryHandle.Allocate(lengthInElements * Unsafe.SizeOf<T>()))); |
|||
} |
|||
} |
|||
@ -0,0 +1,140 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Threading; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory.Internals |
|||
{ |
|||
/// <summary>
|
|||
/// Encapsulates the functionality around allocating and releasing unmanaged memory. NOT a <see cref="SafeHandle"/>.
|
|||
/// </summary>
|
|||
internal struct UnmanagedMemoryHandle : IEquatable<UnmanagedMemoryHandle> |
|||
{ |
|||
// Number of allocation re-attempts when detecting OutOfMemoryException.
|
|||
private const int MaxAllocationAttempts = 1000; |
|||
|
|||
// Track allocations for testing purposes:
|
|||
private static int totalOutstandingHandles; |
|||
|
|||
private static long totalOomRetries; |
|||
|
|||
// A Monitor to wait/signal when we are low on memory.
|
|||
private static object lowMemoryMonitor; |
|||
|
|||
public static readonly UnmanagedMemoryHandle NullHandle = default; |
|||
|
|||
private IntPtr handle; |
|||
private int lengthInBytes; |
|||
|
|||
private UnmanagedMemoryHandle(IntPtr handle, int lengthInBytes) |
|||
{ |
|||
this.handle = handle; |
|||
this.lengthInBytes = lengthInBytes; |
|||
|
|||
if (lengthInBytes > 0) |
|||
{ |
|||
GC.AddMemoryPressure(lengthInBytes); |
|||
} |
|||
|
|||
Interlocked.Increment(ref totalOutstandingHandles); |
|||
} |
|||
|
|||
public IntPtr Handle => this.handle; |
|||
|
|||
public bool IsInvalid => this.Handle == IntPtr.Zero; |
|||
|
|||
public bool IsValid => this.Handle != IntPtr.Zero; |
|||
|
|||
public unsafe void* Pointer => (void*)this.Handle; |
|||
|
|||
/// <summary>
|
|||
/// Gets the total outstanding handle allocations for testing purposes.
|
|||
/// </summary>
|
|||
internal static int TotalOutstandingHandles => totalOutstandingHandles; |
|||
|
|||
/// <summary>
|
|||
/// Gets the total number <see cref="OutOfMemoryException"/>-s retried.
|
|||
/// </summary>
|
|||
internal static long TotalOomRetries => totalOomRetries; |
|||
|
|||
public static bool operator ==(UnmanagedMemoryHandle a, UnmanagedMemoryHandle b) => a.Equals(b); |
|||
|
|||
public static bool operator !=(UnmanagedMemoryHandle a, UnmanagedMemoryHandle b) => !a.Equals(b); |
|||
|
|||
public static UnmanagedMemoryHandle Allocate(int lengthInBytes) |
|||
{ |
|||
IntPtr handle = AllocateHandle(lengthInBytes); |
|||
return new UnmanagedMemoryHandle(handle, lengthInBytes); |
|||
} |
|||
|
|||
private static IntPtr AllocateHandle(int lengthInBytes) |
|||
{ |
|||
int counter = 0; |
|||
IntPtr handle = IntPtr.Zero; |
|||
while (handle == IntPtr.Zero) |
|||
{ |
|||
try |
|||
{ |
|||
handle = Marshal.AllocHGlobal(lengthInBytes); |
|||
} |
|||
catch (OutOfMemoryException) |
|||
{ |
|||
// We are low on memory, but expect some memory to be freed soon.
|
|||
// Block the thread & retry to avoid OOM.
|
|||
if (counter < MaxAllocationAttempts) |
|||
{ |
|||
counter++; |
|||
Interlocked.Increment(ref totalOomRetries); |
|||
|
|||
Interlocked.CompareExchange(ref lowMemoryMonitor, new object(), null); |
|||
Monitor.Enter(lowMemoryMonitor); |
|||
Monitor.Wait(lowMemoryMonitor, millisecondsTimeout: 1); |
|||
Monitor.Exit(lowMemoryMonitor); |
|||
} |
|||
else |
|||
{ |
|||
throw; |
|||
} |
|||
} |
|||
} |
|||
|
|||
return handle; |
|||
} |
|||
|
|||
public void Free() |
|||
{ |
|||
IntPtr h = Interlocked.Exchange(ref this.handle, IntPtr.Zero); |
|||
|
|||
if (h == IntPtr.Zero) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
Marshal.FreeHGlobal(h); |
|||
Interlocked.Decrement(ref totalOutstandingHandles); |
|||
if (this.lengthInBytes > 0) |
|||
{ |
|||
GC.RemoveMemoryPressure(this.lengthInBytes); |
|||
} |
|||
|
|||
if (Volatile.Read(ref lowMemoryMonitor) != null) |
|||
{ |
|||
// We are low on memory. Signal all threads waiting in AllocateHandle().
|
|||
Monitor.Enter(lowMemoryMonitor); |
|||
Monitor.PulseAll(lowMemoryMonitor); |
|||
Monitor.Exit(lowMemoryMonitor); |
|||
} |
|||
|
|||
this.lengthInBytes = 0; |
|||
} |
|||
|
|||
public bool Equals(UnmanagedMemoryHandle other) => this.handle.Equals(other.handle); |
|||
|
|||
public override bool Equals(object obj) => obj is UnmanagedMemoryHandle other && this.Equals(other); |
|||
|
|||
public override int GetHashCode() => this.handle.GetHashCode(); |
|||
} |
|||
} |
|||
@ -0,0 +1,31 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// Defines options for creating the default <see cref="MemoryAllocator"/>.
|
|||
/// </summary>
|
|||
public struct MemoryAllocatorOptions |
|||
{ |
|||
private int? maximumPoolSizeMegabytes; |
|||
|
|||
/// <summary>
|
|||
/// Gets or sets a value defining the maximum size of the <see cref="MemoryAllocator"/>'s internal memory pool
|
|||
/// in Megabytes. <see langword="null"/> means platform default.
|
|||
/// </summary>
|
|||
public int? MaximumPoolSizeMegabytes |
|||
{ |
|||
get => this.maximumPoolSizeMegabytes; |
|||
set |
|||
{ |
|||
if (value.HasValue) |
|||
{ |
|||
Guard.MustBeGreaterThanOrEqualTo(value.Value, 0, nameof(this.MaximumPoolSizeMegabytes)); |
|||
} |
|||
|
|||
this.maximumPoolSizeMegabytes = value; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,164 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using System.Runtime.CompilerServices; |
|||
using System.Runtime.InteropServices; |
|||
using System.Threading; |
|||
using SixLabors.ImageSharp.Memory.Internals; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
internal sealed class UniformUnmanagedMemoryPoolMemoryAllocator : MemoryAllocator |
|||
{ |
|||
private const int OneMegabyte = 1 << 20; |
|||
|
|||
// 4 MB seemed to perform slightly better in benchmarks than 2MB or higher values:
|
|||
private const int DefaultContiguousPoolBlockSizeBytes = 4 * OneMegabyte; |
|||
private const int DefaultNonPoolBlockSizeBytes = 32 * OneMegabyte; |
|||
private readonly int sharedArrayPoolThresholdInBytes; |
|||
private readonly int poolBufferSizeInBytes; |
|||
private readonly int poolCapacity; |
|||
private readonly UniformUnmanagedMemoryPool.TrimSettings trimSettings; |
|||
|
|||
private UniformUnmanagedMemoryPool pool; |
|||
private readonly UnmanagedMemoryAllocator nonPoolAllocator; |
|||
|
|||
public UniformUnmanagedMemoryPoolMemoryAllocator(int? maxPoolSizeMegabytes) |
|||
: this( |
|||
DefaultContiguousPoolBlockSizeBytes, |
|||
maxPoolSizeMegabytes.HasValue ? (long)maxPoolSizeMegabytes.Value * OneMegabyte : GetDefaultMaxPoolSizeBytes(), |
|||
DefaultNonPoolBlockSizeBytes) |
|||
{ |
|||
} |
|||
|
|||
public UniformUnmanagedMemoryPoolMemoryAllocator( |
|||
int poolBufferSizeInBytes, |
|||
long maxPoolSizeInBytes, |
|||
int unmanagedBufferSizeInBytes) |
|||
: this( |
|||
OneMegabyte, |
|||
poolBufferSizeInBytes, |
|||
maxPoolSizeInBytes, |
|||
unmanagedBufferSizeInBytes) |
|||
{ |
|||
} |
|||
|
|||
internal UniformUnmanagedMemoryPoolMemoryAllocator( |
|||
int sharedArrayPoolThresholdInBytes, |
|||
int poolBufferSizeInBytes, |
|||
long maxPoolSizeInBytes, |
|||
int unmanagedBufferSizeInBytes) |
|||
: this( |
|||
sharedArrayPoolThresholdInBytes, |
|||
poolBufferSizeInBytes, |
|||
maxPoolSizeInBytes, |
|||
unmanagedBufferSizeInBytes, |
|||
UniformUnmanagedMemoryPool.TrimSettings.Default) |
|||
{ |
|||
} |
|||
|
|||
internal UniformUnmanagedMemoryPoolMemoryAllocator( |
|||
int sharedArrayPoolThresholdInBytes, |
|||
int poolBufferSizeInBytes, |
|||
long maxPoolSizeInBytes, |
|||
int unmanagedBufferSizeInBytes, |
|||
UniformUnmanagedMemoryPool.TrimSettings trimSettings) |
|||
{ |
|||
this.sharedArrayPoolThresholdInBytes = sharedArrayPoolThresholdInBytes; |
|||
this.poolBufferSizeInBytes = poolBufferSizeInBytes; |
|||
this.poolCapacity = (int)(maxPoolSizeInBytes / poolBufferSizeInBytes); |
|||
this.trimSettings = trimSettings; |
|||
this.pool = new UniformUnmanagedMemoryPool(this.poolBufferSizeInBytes, this.poolCapacity, this.trimSettings); |
|||
this.nonPoolAllocator = new UnmanagedMemoryAllocator(unmanagedBufferSizeInBytes); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
protected internal override int GetBufferCapacityInBytes() => this.poolBufferSizeInBytes; |
|||
|
|||
/// <inheritdoc />
|
|||
public override IMemoryOwner<T> Allocate<T>( |
|||
int length, |
|||
AllocationOptions options = AllocationOptions.None) |
|||
{ |
|||
Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); |
|||
int lengthInBytes = length * Unsafe.SizeOf<T>(); |
|||
|
|||
if (lengthInBytes <= this.sharedArrayPoolThresholdInBytes) |
|||
{ |
|||
var buffer = new SharedArrayPoolBuffer<T>(length); |
|||
if (options.Has(AllocationOptions.Clean)) |
|||
{ |
|||
buffer.GetSpan().Clear(); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
|
|||
if (lengthInBytes <= this.poolBufferSizeInBytes) |
|||
{ |
|||
UnmanagedMemoryHandle mem = this.pool.Rent(); |
|||
if (mem.IsValid) |
|||
{ |
|||
UnmanagedBuffer<T> buffer = this.pool.CreateGuardedBuffer<T>(mem, length, options.Has(AllocationOptions.Clean)); |
|||
return buffer; |
|||
} |
|||
} |
|||
|
|||
return this.nonPoolAllocator.Allocate<T>(length, options); |
|||
} |
|||
|
|||
/// <inheritdoc />
|
|||
internal override MemoryGroup<T> AllocateGroup<T>( |
|||
long totalLength, |
|||
int bufferAlignment, |
|||
AllocationOptions options = AllocationOptions.None) |
|||
{ |
|||
long totalLengthInBytes = totalLength * Unsafe.SizeOf<T>(); |
|||
if (totalLengthInBytes <= this.sharedArrayPoolThresholdInBytes) |
|||
{ |
|||
var buffer = new SharedArrayPoolBuffer<T>((int)totalLength); |
|||
return MemoryGroup<T>.CreateContiguous(buffer, options.Has(AllocationOptions.Clean)); |
|||
} |
|||
|
|||
if (totalLengthInBytes <= this.poolBufferSizeInBytes) |
|||
{ |
|||
// Optimized path renting single array from the pool
|
|||
UnmanagedMemoryHandle mem = this.pool.Rent(); |
|||
if (mem.IsValid) |
|||
{ |
|||
UnmanagedBuffer<T> buffer = this.pool.CreateGuardedBuffer<T>(mem, (int)totalLength, options.Has(AllocationOptions.Clean)); |
|||
return MemoryGroup<T>.CreateContiguous(buffer, options.Has(AllocationOptions.Clean)); |
|||
} |
|||
} |
|||
|
|||
// Attempt to rent the whole group from the pool, allocate a group of unmanaged buffers if the attempt fails:
|
|||
if (MemoryGroup<T>.TryAllocate(this.pool, totalLength, bufferAlignment, options, out MemoryGroup<T> poolGroup)) |
|||
{ |
|||
return poolGroup; |
|||
} |
|||
|
|||
return MemoryGroup<T>.Allocate(this.nonPoolAllocator, totalLength, bufferAlignment, options); |
|||
} |
|||
|
|||
public override void ReleaseRetainedResources() => this.pool.Release(); |
|||
|
|||
private static long GetDefaultMaxPoolSizeBytes() |
|||
{ |
|||
#if NETCOREAPP3_1_OR_GREATER
|
|||
// On 64 bit .NET Core 3.1+, set the pool size to a portion of the total available memory.
|
|||
// There is a bug in GC.GetGCMemoryInfo() on .NET 5 + 32 bit, making TotalAvailableMemoryBytes unreliable:
|
|||
// https://github.com/dotnet/runtime/issues/55126#issuecomment-876779327
|
|||
if (Environment.Is64BitProcess || !RuntimeInformation.FrameworkDescription.StartsWith(".NET 5.0")) |
|||
{ |
|||
GCMemoryInfo info = GC.GetGCMemoryInfo(); |
|||
return info.TotalAvailableMemoryBytes / 8; |
|||
} |
|||
#endif
|
|||
|
|||
// Stick to a conservative value of 128 Megabytes on other platforms and 32 bit .NET 5.0:
|
|||
return 128 * OneMegabyte; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using System.Buffers; |
|||
using SixLabors.ImageSharp.Memory.Internals; |
|||
|
|||
namespace SixLabors.ImageSharp.Memory |
|||
{ |
|||
/// <summary>
|
|||
/// A <see cref="MemoryAllocator"/> implementation that allocates memory on the unmanaged heap
|
|||
/// without any pooling.
|
|||
/// </summary>
|
|||
internal class UnmanagedMemoryAllocator : MemoryAllocator |
|||
{ |
|||
private readonly int bufferCapacityInBytes; |
|||
|
|||
public UnmanagedMemoryAllocator(int bufferCapacityInBytes) => this.bufferCapacityInBytes = bufferCapacityInBytes; |
|||
|
|||
protected internal override int GetBufferCapacityInBytes() => this.bufferCapacityInBytes; |
|||
|
|||
public override IMemoryOwner<T> Allocate<T>(int length, AllocationOptions options = AllocationOptions.None) |
|||
{ |
|||
var buffer = UnmanagedBuffer<T>.Allocate(length); |
|||
if (options.Has(AllocationOptions.Clean)) |
|||
{ |
|||
buffer.GetSpan().Clear(); |
|||
} |
|||
|
|||
return buffer; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,72 @@ |
|||
// Copyright (c) Six Labors.
|
|||
// Licensed under the Apache License, Version 2.0.
|
|||
|
|||
using System; |
|||
using SixLabors.ImageSharp.Memory; |
|||
using SixLabors.ImageSharp.PixelFormats; |
|||
|
|||
namespace SixLabors.ImageSharp |
|||
{ |
|||
/// <summary>
|
|||
/// A delegate to be executed on a <see cref="PixelAccessor{TPixel}"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="TPixel">The pixel type.</typeparam>
|
|||
public delegate void PixelAccessorAction<TPixel>(PixelAccessor<TPixel> pixelAccessor) |
|||
where TPixel : unmanaged, IPixel<TPixel>; |
|||
|
|||
/// <summary>
|
|||
/// A delegate to be executed on two instances of <see cref="PixelAccessor{TPixel}"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="TPixel1">The first pixel type.</typeparam>
|
|||
/// <typeparam name="TPixel2">The second pixel type.</typeparam>
|
|||
public delegate void PixelAccessorAction<TPixel1, TPixel2>( |
|||
PixelAccessor<TPixel1> pixelAccessor1, |
|||
PixelAccessor<TPixel2> pixelAccessor2) |
|||
where TPixel1 : unmanaged, IPixel<TPixel1> |
|||
where TPixel2 : unmanaged, IPixel<TPixel2>; |
|||
|
|||
/// <summary>
|
|||
/// A delegate to be executed on three instances of <see cref="PixelAccessor{TPixel}"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="TPixel1">The first pixel type.</typeparam>
|
|||
/// <typeparam name="TPixel2">The second pixel type.</typeparam>
|
|||
/// <typeparam name="TPixel3">The third pixel type.</typeparam>
|
|||
public delegate void PixelAccessorAction<TPixel1, TPixel2, TPixel3>( |
|||
PixelAccessor<TPixel1> pixelAccessor1, |
|||
PixelAccessor<TPixel2> pixelAccessor2, |
|||
PixelAccessor<TPixel3> pixelAccessor3) |
|||
where TPixel1 : unmanaged, IPixel<TPixel1> |
|||
where TPixel2 : unmanaged, IPixel<TPixel2> |
|||
where TPixel3 : unmanaged, IPixel<TPixel3>; |
|||
|
|||
/// <summary>
|
|||
/// Provides efficient access the pixel buffers of an <see cref="Image{TPixel}"/>.
|
|||
/// </summary>
|
|||
/// <typeparam name="TPixel">The pixel type.</typeparam>
|
|||
public ref struct PixelAccessor<TPixel> |
|||
where TPixel : unmanaged, IPixel<TPixel> |
|||
{ |
|||
private Buffer2D<TPixel> buffer; |
|||
|
|||
internal PixelAccessor(Buffer2D<TPixel> buffer) => this.buffer = buffer; |
|||
|
|||
/// <summary>
|
|||
/// Gets the width of the backing <see cref="Image{TPixel}"/>.
|
|||
/// </summary>
|
|||
public int Width => this.buffer.Width; |
|||
|
|||
/// <summary>
|
|||
/// Gets the height of the backing <see cref="Image{TPixel}"/>.
|
|||
/// </summary>
|
|||
public int Height => this.buffer.Height; |
|||
|
|||
/// <summary>
|
|||
/// Gets the representation of the pixels as a <see cref="Span{T}"/> of contiguous memory
|
|||
/// at row <paramref name="rowIndex"/> beginning from the first pixel on that row.
|
|||
/// </summary>
|
|||
/// <param name="rowIndex">The row index.</param>
|
|||
/// <returns>The <see cref="Span{TPixel}"/>.</returns>
|
|||
/// <exception cref="ArgumentOutOfRangeException">Thrown when row index is out of range.</exception>
|
|||
public Span<TPixel> GetRowSpan(int rowIndex) => this.buffer.DangerousGetRowSpan(rowIndex); |
|||
} |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Loading…
Reference in new issue