diff --git a/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs index 21673215ae..9302c67e7d 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs @@ -13,34 +13,35 @@ namespace SixLabors.ImageSharp.Memory.Internals where T : struct { private readonly int lengthInBytes; - private byte[] array; private LifetimeGuard lifetimeGuard; public SharedArrayPoolBuffer(int lengthInElements) { this.lengthInBytes = lengthInElements * Unsafe.SizeOf(); - this.array = ArrayPool.Shared.Rent(this.lengthInBytes); - this.lifetimeGuard = new LifetimeGuard(this.array); + this.Array = ArrayPool.Shared.Rent(this.lengthInBytes); + this.lifetimeGuard = new LifetimeGuard(this.Array); } + public byte[] Array { get; private set; } + protected override void Dispose(bool disposing) { - if (this.array == null) + if (this.Array == null) { return; } this.lifetimeGuard.Dispose(); - this.array = null; + this.Array = null; } public override Span GetSpan() { this.CheckDisposed(); - return MemoryMarshal.Cast(this.array.AsSpan(0, this.lengthInBytes)); + return MemoryMarshal.Cast(this.Array.AsSpan(0, this.lengthInBytes)); } - protected override object GetPinnableObject() => this.array; + protected override object GetPinnableObject() => this.Array; public void AddRef() { @@ -53,7 +54,7 @@ namespace SixLabors.ImageSharp.Memory.Internals [Conditional("DEBUG")] private void CheckDisposed() { - if (this.array == null) + if (this.Array == null) { throw new ObjectDisposedException("SharedArrayPoolBuffer"); } diff --git a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs index 6504787a84..584de44648 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs @@ -8,12 +8,10 @@ using System.Threading; namespace SixLabors.ImageSharp.Memory.Internals { - internal partial class UniformUnmanagedMemoryPool -#if !NETSTANDARD1_3 - // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers, - // but we should not rely on this. - : System.Runtime.ConstrainedExecution.CriticalFinalizerObject -#endif + // CriticalFinalizerObject: + // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers, + // but we should not rely on this. + internal partial class UniformUnmanagedMemoryPool : System.Runtime.ConstrainedExecution.CriticalFinalizerObject { private static int minTrimPeriodMilliseconds = int.MaxValue; private static readonly List> AllPools = new(); diff --git a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs index 5d0c6dd613..a948bf8487 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs @@ -31,7 +31,7 @@ namespace SixLabors.ImageSharp.Memory.Internals this.lifetimeGuard = lifetimeGuard; } - private void* Pointer => this.lifetimeGuard.Handle.Pointer; + public void* Pointer => this.lifetimeGuard.Handle.Pointer; public override Span GetSpan() { diff --git a/src/ImageSharp/Memory/Buffer2D{T}.cs b/src/ImageSharp/Memory/Buffer2D{T}.cs index 42dce2def5..ea4b8031e4 100644 --- a/src/ImageSharp/Memory/Buffer2D{T}.cs +++ b/src/ImageSharp/Memory/Buffer2D{T}.cs @@ -97,10 +97,12 @@ namespace SixLabors.ImageSharp.Memory [MethodImpl(InliningOptions.ShortMethod)] public Span DangerousGetRowSpan(int y) { - DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); - DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); + if (y < 0 || y >= this.Height) + { + this.ThrowYOutOfRangeException(y); + } - return this.GetRowMemoryCore(y).Span; + return this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width); } internal bool TryGetPaddedRowSpan(int y, int padding, out Span paddedSpan) @@ -125,7 +127,7 @@ namespace SixLabors.ImageSharp.Memory [MethodImpl(InliningOptions.ShortMethod)] internal ref T GetElementUnsafe(int x, int y) { - Span span = this.GetRowMemoryCore(y).Span; + Span span = this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width); return ref span[x]; } @@ -139,7 +141,7 @@ namespace SixLabors.ImageSharp.Memory { DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); - return this.FastMemoryGroup.View.GetBoundedSlice(y * (long)this.Width, this.Width); + return this.FastMemoryGroup.View.GetBoundedMemorySlice(y * (long)this.Width, this.Width); } /// @@ -195,7 +197,9 @@ namespace SixLabors.ImageSharp.Memory return swapped; } - [MethodImpl(InliningOptions.ShortMethod)] - private Memory GetRowMemoryCore(int y) => this.FastMemoryGroup.GetBoundedSlice(y * (long)this.Width, this.Width); + [MethodImpl(InliningOptions.ColdPath)] + private void ThrowYOutOfRangeException(int y) => + throw new ArgumentOutOfRangeException( + $"DangerousGetRowSpan({y}). Y was out of range. Height={this.Height}"); } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs index 8e6f38d145..571ffc98d6 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs @@ -29,7 +29,7 @@ namespace SixLabors.ImageSharp.Memory /// Returns a slice that is expected to be within the bounds of a single buffer. /// Otherwise is thrown. /// - internal static Memory GetBoundedSlice(this IMemoryGroup group, long start, int length) + internal static Memory GetBoundedMemorySlice(this IMemoryGroup group, long start, int length) where T : struct { Guard.NotNull(group, nameof(group)); diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs new file mode 100644 index 0000000000..fbaf2dcf04 --- /dev/null +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs @@ -0,0 +1,50 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System.Buffers; +using SixLabors.ImageSharp.Memory.Internals; + +namespace SixLabors.ImageSharp.Memory +{ + internal unsafe struct MemoryGroupSpanCache + { + public SpanCacheMode Mode; + public byte[] SingleArray; + public void* SinglePointer; + public void*[] MultiPointer; + + public static MemoryGroupSpanCache Create(IMemoryOwner[] memoryOwners) + where T : struct + { + IMemoryOwner owner0 = memoryOwners[0]; + MemoryGroupSpanCache memoryGroupSpanCache = default; + if (memoryOwners.Length == 1) + { + if (owner0 is SharedArrayPoolBuffer sharedPoolBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.SingleArray; + memoryGroupSpanCache.SingleArray = sharedPoolBuffer.Array; + } + else if (owner0 is UnmanagedBuffer unmanagedBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.SinglePointer; + memoryGroupSpanCache.SinglePointer = unmanagedBuffer.Pointer; + } + } + else + { + if (owner0 is UnmanagedBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.MultiPointer; + memoryGroupSpanCache.MultiPointer = new void*[memoryOwners.Length]; + for (int i = 0; i < memoryOwners.Length; i++) + { + memoryGroupSpanCache.MultiPointer[i] = ((UnmanagedBuffer)memoryOwners[i]).Pointer; + } + } + } + + return memoryGroupSpanCache; + } + } +} diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs index b578b417f9..21faf8e562 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs @@ -26,6 +26,7 @@ namespace SixLabors.ImageSharp.Memory this.memoryOwners = memoryOwners; this.Swappable = swappable; this.View = new MemoryGroupView(this); + this.memoryGroupSpanCache = MemoryGroupSpanCache.Create(memoryOwners); } public Owned( @@ -173,32 +174,6 @@ namespace SixLabors.ImageSharp.Memory [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(MemoryGroup)); - internal static void SwapContents(Owned a, Owned b) - { - a.EnsureNotDisposed(); - b.EnsureNotDisposed(); - - IMemoryOwner[] tempOwners = a.memoryOwners; - long tempTotalLength = a.TotalLength; - int tempBufferLength = a.BufferLength; - RefCountedLifetimeGuard tempGroupOwner = a.groupLifetimeGuard; - - a.memoryOwners = b.memoryOwners; - a.TotalLength = b.TotalLength; - a.BufferLength = b.BufferLength; - a.groupLifetimeGuard = b.groupLifetimeGuard; - - b.memoryOwners = tempOwners; - b.TotalLength = tempTotalLength; - b.BufferLength = tempBufferLength; - b.groupLifetimeGuard = tempGroupOwner; - - a.View.Invalidate(); - b.View.Invalidate(); - a.View = new MemoryGroupView(a); - b.View = new MemoryGroupView(b); - } - // When the MemoryGroup points to multiple buffers via `groupLifetimeGuard`, // the lifetime of the individual buffers is managed by the guard. // Group buffer IMemoryOwner-s d not manage ownership. diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs index 1f18d91692..a324f55e4c 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs @@ -6,6 +6,8 @@ using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; using SixLabors.ImageSharp.Memory.Internals; namespace SixLabors.ImageSharp.Memory @@ -21,6 +23,8 @@ namespace SixLabors.ImageSharp.Memory { private static readonly int ElementSize = Unsafe.SizeOf(); + private MemoryGroupSpanCache memoryGroupSpanCache; + private MemoryGroup(int bufferLength, long totalLength) { this.BufferLength = bufferLength; @@ -31,10 +35,10 @@ namespace SixLabors.ImageSharp.Memory public abstract int Count { get; } /// - public int BufferLength { get; private set; } + public int BufferLength { get; } /// - public long TotalLength { get; private set; } + public long TotalLength { get; } /// public bool IsValid { get; private set; } = true; @@ -241,6 +245,40 @@ namespace SixLabors.ImageSharp.Memory return new Owned(source, bufferLength, totalLength, false); } + [MethodImpl(InliningOptions.ShortMethod)] + public unsafe Span GetRowSpanCoreUnsafe(int y, int width) + { + switch (this.memoryGroupSpanCache.Mode) + { + case SpanCacheMode.SingleArray: + { + ref byte b0 = ref MemoryMarshal.GetReference(this.memoryGroupSpanCache.SingleArray); + ref T e0 = ref Unsafe.As(ref b0); + e0 = ref Unsafe.Add(ref e0, y * width); + return MemoryMarshal.CreateSpan(ref e0, width); + } + + case SpanCacheMode.SinglePointer: + { + void* start = Unsafe.Add(this.memoryGroupSpanCache.SinglePointer, y * width); + return new Span(start, width); + } + + case SpanCacheMode.MultiPointer: + { + this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart); + void* start = Unsafe.Add(this.memoryGroupSpanCache.MultiPointer[bufferIdx], bufferStart); + return new Span(start, width); + } + + default: + { + this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart); + return this[bufferIdx].Span.Slice(bufferStart, width); + } + } + } + public static bool CanSwapContent(MemoryGroup target, MemoryGroup source) => source is Owned { Swappable: true } && target is Owned { Swappable: true }; @@ -255,5 +293,13 @@ namespace SixLabors.ImageSharp.Memory public virtual void DecreaseRefCounts() { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void GetMultiBufferPosition(int y, int width, out int bufferIdx, out int bufferStart) + { + long start = y * (long)width; + bufferIdx = (int)(start / this.BufferLength); + bufferStart = (int)(start % this.BufferLength); + } } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs new file mode 100644 index 0000000000..6ab8db0caa --- /dev/null +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs @@ -0,0 +1,13 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +namespace SixLabors.ImageSharp.Memory +{ + internal enum SpanCacheMode + { + Default = default, + SingleArray, + SinglePointer, + MultiPointer + } +} diff --git a/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs b/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs index 84f035583f..1a130147ad 100644 --- a/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs +++ b/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Apache License, Version 2.0. +using System.Runtime.CompilerServices; using BenchmarkDotNet.Attributes; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.PixelFormats; @@ -9,18 +10,21 @@ namespace SixLabors.ImageSharp.Benchmarks.General { public class Buffer2D_DangerousGetRowSpan { - [Params(true, false)] - public bool IsDiscontiguousBuffer { get; set; } + private const int Height = 1024; + + [Params(0.5, 2.0, 10.0)] + public double SizeMegaBytes { get; set; } private Buffer2D buffer; [GlobalSetup] - public void Setup() + public unsafe void Setup() { + int totalElements = (int)(1024 * 1024 * this.SizeMegaBytes) / sizeof(Rgba32); + + int width = totalElements / Height; MemoryAllocator allocator = Configuration.Default.MemoryAllocator; - this.buffer = this.IsDiscontiguousBuffer - ? allocator.Allocate2D(4000, 1000) - : allocator.Allocate2D(500, 1000); + this.buffer = allocator.Allocate2D(width, Height); } [GlobalCleanup] @@ -29,7 +33,7 @@ namespace SixLabors.ImageSharp.Benchmarks.General [Benchmark] public int DangerousGetRowSpan() => this.buffer.DangerousGetRowSpan(1).Length + - this.buffer.DangerousGetRowSpan(999).Length; + this.buffer.DangerousGetRowSpan(Height - 1).Length; // BenchmarkDotNet=v0.13.0, OS=Windows 10.0.19044 // Intel Core i9-10900X CPU 3.70GHz, 1 CPU, 20 logical and 10 physical cores diff --git a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs index 72b4ccadf0..0fee2c8782 100644 --- a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs +++ b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs @@ -120,7 +120,7 @@ namespace SixLabors.ImageSharp.Tests.Memory [InlineData(200, 100, 30, 1, 0)] [InlineData(200, 100, 30, 2, 1)] [InlineData(200, 100, 30, 4, 2)] - public unsafe void GetRowSpanY(int bufferCapacity, int width, int height, int y, int expectedBufferIndex) + public unsafe void DangerousGetRowSpan_TestAllocator(int bufferCapacity, int width, int height, int y, int expectedBufferIndex) { this.MemoryAllocator.BufferCapacityInBytes = sizeof(TestStructs.Foo) * bufferCapacity; @@ -135,6 +135,57 @@ namespace SixLabors.ImageSharp.Tests.Memory } } + [Theory] + [InlineData(100, 5)] // Within shared pool + [InlineData(77, 11)] // Within shared pool + [InlineData(100, 19)] // Single unmanaged pooled buffer + [InlineData(103, 17)] // Single unmanaged pooled buffer + [InlineData(100, 22)] // 2 unmanaged pooled buffers + [InlineData(100, 99)] // 9 unmanaged pooled buffers + [InlineData(100, 120)] // 2 unpooled buffers + public unsafe void DangerousGetRowSpan_UnmanagedAllocator(int width, int height) + { + const int sharedPoolThreshold = 1_000; + const int poolBufferSize = 2_000; + const int maxPoolSize = 10_000; + const int unpooledBufferSize = 8_000; + + int elementSize = sizeof(TestStructs.Foo); + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator( + sharedPoolThreshold * elementSize, + poolBufferSize * elementSize, + maxPoolSize * elementSize, + unpooledBufferSize * elementSize); + + using Buffer2D buffer = allocator.Allocate2D(width, height); + + var rnd = new Random(42); + + for (int y = 0; y < buffer.Height; y++) + { + Span span = buffer.DangerousGetRowSpan(y); + for (int x = 0; x < span.Length; x++) + { + ref TestStructs.Foo e = ref span[x]; + e.A = rnd.Next(); + e.B = rnd.NextDouble(); + } + } + + // Re-seed + rnd = new Random(42); + for (int y = 0; y < buffer.Height; y++) + { + Span span = buffer.GetSafeRowMemory(y).Span; + for (int x = 0; x < span.Length; x++) + { + ref TestStructs.Foo e = ref span[x]; + Assert.True(rnd.Next() == e.A, $"Mismatch @ y={y} x={x}"); + Assert.True(rnd.NextDouble() == e.B, $"Mismatch @ y={y} x={x}"); + } + } + } + [Theory] [InlineData(10, 0, 0, 0)] [InlineData(10, 0, 2, 0)] diff --git a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs index 13e47bdee2..cb6c34b77c 100644 --- a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs +++ b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs @@ -146,7 +146,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers { using MemoryGroup group = this.CreateTestGroup(totalLength, bufferLength, true); - Memory slice = group.GetBoundedSlice(start, length); + Memory slice = group.GetBoundedMemorySlice(start, length); Assert.Equal(length, slice.Length); @@ -172,7 +172,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers public void GetBoundedSlice_WhenOverlapsBuffers_Throws(long totalLength, int bufferLength, long start, int length) { using MemoryGroup group = this.CreateTestGroup(totalLength, bufferLength, true); - Assert.ThrowsAny(() => group.GetBoundedSlice(start, length)); + Assert.ThrowsAny(() => group.GetBoundedMemorySlice(start, length)); } [Fact]