diff --git a/src/ImageSharp/Common/Helpers/Numerics.cs b/src/ImageSharp/Common/Helpers/Numerics.cs index 8f82476e13..7de838bc94 100644 --- a/src/ImageSharp/Common/Helpers/Numerics.cs +++ b/src/ImageSharp/Common/Helpers/Numerics.cs @@ -970,6 +970,7 @@ namespace SixLabors.ImageSharp /// Value. /// Mininum value, inclusive. /// Maximum value, inclusive. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsOutOfRange(int value, int min, int max) => (uint)(value - min) > (uint)(max - min); } diff --git a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs index 87f11a085f..40411ef288 100644 --- a/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs +++ b/src/ImageSharp/Formats/Jpeg/Components/Decoder/SpectralConverter{TPixel}.cs @@ -179,7 +179,7 @@ namespace SixLabors.ImageSharp.Formats.Jpeg.Components.Decoder // PackFromRgbPlanes expects the destination to be padded, so try to get padded span containing extra elements from the next row. // If we can't get such a padded row because we are on a MemoryGroup boundary or at the last row, // pack pixels to a temporary, padded proxy buffer, then copy the relevant values to the destination row. - if (this.pixelBuffer.TryGetPaddedRowSpan(yy, 3, out Span destRow)) + if (this.pixelBuffer.DangerousTryGetPaddedRowSpan(yy, 3, out Span destRow)) { PixelOperations.Instance.PackFromRgbPlanes(this.configuration, r, g, b, destRow); } diff --git a/src/ImageSharp/ImageFrame{TPixel}.cs b/src/ImageSharp/ImageFrame{TPixel}.cs index 4753e90e0c..cc1b01ff7d 100644 --- a/src/ImageSharp/ImageFrame{TPixel}.cs +++ b/src/ImageSharp/ImageFrame{TPixel}.cs @@ -443,12 +443,12 @@ namespace SixLabors.ImageSharp [MethodImpl(InliningOptions.ShortMethod)] private void VerifyCoords(int x, int y) { - if (x < 0 || x >= this.Width) + if ((uint)x >= (uint)this.Width) { ThrowArgumentOutOfRangeException(nameof(x)); } - if (y < 0 || y >= this.Height) + if ((uint)y >= (uint)this.Height) { ThrowArgumentOutOfRangeException(nameof(y)); } diff --git a/src/ImageSharp/Image{TPixel}.cs b/src/ImageSharp/Image{TPixel}.cs index d01706b01d..403a0f7ea6 100644 --- a/src/ImageSharp/Image{TPixel}.cs +++ b/src/ImageSharp/Image{TPixel}.cs @@ -452,12 +452,12 @@ namespace SixLabors.ImageSharp [MethodImpl(InliningOptions.ShortMethod)] private void VerifyCoords(int x, int y) { - if (x < 0 || x >= this.Width) + if ((uint)x >= (uint)this.Width) { ThrowArgumentOutOfRangeException(nameof(x)); } - if (y < 0 || y >= this.Height) + if ((uint)y >= (uint)this.Height) { ThrowArgumentOutOfRangeException(nameof(y)); } diff --git a/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs index 21673215ae..9302c67e7d 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/SharedArrayPoolBuffer{T}.cs @@ -13,34 +13,35 @@ namespace SixLabors.ImageSharp.Memory.Internals where T : struct { private readonly int lengthInBytes; - private byte[] array; private LifetimeGuard lifetimeGuard; public SharedArrayPoolBuffer(int lengthInElements) { this.lengthInBytes = lengthInElements * Unsafe.SizeOf(); - this.array = ArrayPool.Shared.Rent(this.lengthInBytes); - this.lifetimeGuard = new LifetimeGuard(this.array); + this.Array = ArrayPool.Shared.Rent(this.lengthInBytes); + this.lifetimeGuard = new LifetimeGuard(this.Array); } + public byte[] Array { get; private set; } + protected override void Dispose(bool disposing) { - if (this.array == null) + if (this.Array == null) { return; } this.lifetimeGuard.Dispose(); - this.array = null; + this.Array = null; } public override Span GetSpan() { this.CheckDisposed(); - return MemoryMarshal.Cast(this.array.AsSpan(0, this.lengthInBytes)); + return MemoryMarshal.Cast(this.Array.AsSpan(0, this.lengthInBytes)); } - protected override object GetPinnableObject() => this.array; + protected override object GetPinnableObject() => this.Array; public void AddRef() { @@ -53,7 +54,7 @@ namespace SixLabors.ImageSharp.Memory.Internals [Conditional("DEBUG")] private void CheckDisposed() { - if (this.array == null) + if (this.Array == null) { throw new ObjectDisposedException("SharedArrayPoolBuffer"); } diff --git a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs index 6504787a84..584de44648 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/UniformUnmanagedMemoryPool.cs @@ -8,12 +8,10 @@ using System.Threading; namespace SixLabors.ImageSharp.Memory.Internals { - internal partial class UniformUnmanagedMemoryPool -#if !NETSTANDARD1_3 - // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers, - // but we should not rely on this. - : System.Runtime.ConstrainedExecution.CriticalFinalizerObject -#endif + // CriticalFinalizerObject: + // In case UniformUnmanagedMemoryPool is finalized, we prefer to run its finalizer after the guard finalizers, + // but we should not rely on this. + internal partial class UniformUnmanagedMemoryPool : System.Runtime.ConstrainedExecution.CriticalFinalizerObject { private static int minTrimPeriodMilliseconds = int.MaxValue; private static readonly List> AllPools = new(); diff --git a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs index 5d0c6dd613..a948bf8487 100644 --- a/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs +++ b/src/ImageSharp/Memory/Allocators/Internals/UnmanagedBuffer{T}.cs @@ -31,7 +31,7 @@ namespace SixLabors.ImageSharp.Memory.Internals this.lifetimeGuard = lifetimeGuard; } - private void* Pointer => this.lifetimeGuard.Handle.Pointer; + public void* Pointer => this.lifetimeGuard.Handle.Pointer; public override Span GetSpan() { diff --git a/src/ImageSharp/Memory/Buffer2D{T}.cs b/src/ImageSharp/Memory/Buffer2D{T}.cs index ba39a924ea..7ffaae312a 100644 --- a/src/ImageSharp/Memory/Buffer2D{T}.cs +++ b/src/ImageSharp/Memory/Buffer2D{T}.cs @@ -56,7 +56,7 @@ namespace SixLabors.ImageSharp.Memory /// It's public counterpart is , /// which only exposes the view of the MemoryGroup. /// - internal MemoryGroup FastMemoryGroup { get; } + internal MemoryGroup FastMemoryGroup { get; private set; } /// /// Gets a reference to the element at the specified position. @@ -97,35 +97,37 @@ namespace SixLabors.ImageSharp.Memory [MethodImpl(InliningOptions.ShortMethod)] public Span DangerousGetRowSpan(int y) { - DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); - DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); + if ((uint)y >= (uint)this.Height) + { + this.ThrowYOutOfRangeException(y); + } - return this.GetRowMemoryCore(y).Span; + return this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width); } - internal bool TryGetPaddedRowSpan(int y, int padding, out Span paddedSpan) + internal bool DangerousTryGetPaddedRowSpan(int y, int padding, out Span paddedSpan) { DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); int stride = this.Width + padding; - Memory memory = this.FastMemoryGroup.GetRemainingSliceOfBuffer(y * (long)this.Width); + Span slice = this.FastMemoryGroup.GetRemainingSliceOfBuffer(y * (long)this.Width); - if (memory.Length < stride) + if (slice.Length < stride) { paddedSpan = default; return false; } - paddedSpan = memory.Span.Slice(0, stride); + paddedSpan = slice.Slice(0, stride); return true; } [MethodImpl(InliningOptions.ShortMethod)] internal ref T GetElementUnsafe(int x, int y) { - Span span = this.GetRowMemoryCore(y).Span; + Span span = this.FastMemoryGroup.GetRowSpanCoreUnsafe(y, this.Width); return ref span[x]; } @@ -139,7 +141,7 @@ namespace SixLabors.ImageSharp.Memory { DebugGuard.MustBeGreaterThanOrEqualTo(y, 0, nameof(y)); DebugGuard.MustBeLessThan(y, this.Height, nameof(y)); - return this.FastMemoryGroup.View.GetBoundedSlice(y * (long)this.Width, this.Width); + return this.FastMemoryGroup.View.GetBoundedMemorySlice(y * (long)this.Width, this.Width); } /// @@ -168,25 +170,36 @@ namespace SixLabors.ImageSharp.Memory /// Swaps the contents of 'destination' with 'source' if the buffers are owned (1), /// copies the contents of 'source' to 'destination' otherwise (2). Buffers should be of same size in case 2! /// - internal static void SwapOrCopyContent(Buffer2D destination, Buffer2D source) - { - MemoryGroup.SwapOrCopyContent(destination.FastMemoryGroup, source.FastMemoryGroup); - SwapOwnData(destination, source); - } - - [MethodImpl(InliningOptions.ShortMethod)] - private Memory GetRowMemoryCore(int y) => this.FastMemoryGroup.GetBoundedSlice(y * (long)this.Width, this.Width); - - private static void SwapOwnData(Buffer2D a, Buffer2D b) + internal static bool SwapOrCopyContent(Buffer2D destination, Buffer2D source) { - Size aSize = a.Size(); - Size bSize = b.Size(); + bool swapped = false; + if (MemoryGroup.CanSwapContent(destination.FastMemoryGroup, source.FastMemoryGroup)) + { + (destination.FastMemoryGroup, source.FastMemoryGroup) = + (source.FastMemoryGroup, destination.FastMemoryGroup); + destination.FastMemoryGroup.RecreateViewAfterSwap(); + source.FastMemoryGroup.RecreateViewAfterSwap(); + swapped = true; + } + else + { + if (destination.FastMemoryGroup.TotalLength != source.FastMemoryGroup.TotalLength) + { + throw new InvalidMemoryOperationException( + "Trying to copy/swap incompatible buffers. This is most likely caused by applying an unsupported processor to wrapped-memory images."); + } - b.Width = aSize.Width; - b.Height = aSize.Height; + source.FastMemoryGroup.CopyTo(destination.MemoryGroup); + } - a.Width = bSize.Width; - a.Height = bSize.Height; + (destination.Width, source.Width) = (source.Width, destination.Width); + (destination.Height, source.Height) = (source.Height, destination.Height); + return swapped; } + + [MethodImpl(InliningOptions.ColdPath)] + private void ThrowYOutOfRangeException(int y) => + throw new ArgumentOutOfRangeException( + $"DangerousGetRowSpan({y}). Y was out of range. Height={this.Height}"); } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs index 8e6f38d145..d200b223a7 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupExtensions.cs @@ -29,7 +29,7 @@ namespace SixLabors.ImageSharp.Memory /// Returns a slice that is expected to be within the bounds of a single buffer. /// Otherwise is thrown. /// - internal static Memory GetBoundedSlice(this IMemoryGroup group, long start, int length) + internal static Memory GetBoundedMemorySlice(this IMemoryGroup group, long start, int length) where T : struct { Guard.NotNull(group, nameof(group)); @@ -37,7 +37,8 @@ namespace SixLabors.ImageSharp.Memory Guard.MustBeGreaterThanOrEqualTo(length, 0, nameof(length)); Guard.MustBeLessThan(start, group.TotalLength, nameof(start)); - int bufferIdx = (int)(start / group.BufferLength); + int bufferIdx = (int)Math.DivRem(start, group.BufferLength, out long bufferStartLong); + int bufferStart = (int)bufferStartLong; // if (bufferIdx < 0 || bufferIdx >= group.Count) if ((uint)bufferIdx >= group.Count) @@ -45,7 +46,6 @@ namespace SixLabors.ImageSharp.Memory throw new ArgumentOutOfRangeException(nameof(start)); } - int bufferStart = (int)(start % group.BufferLength); int bufferEnd = bufferStart + length; Memory memory = group[bufferIdx]; @@ -57,31 +57,6 @@ namespace SixLabors.ImageSharp.Memory return memory.Slice(bufferStart, length); } - /// - /// Returns the slice of the buffer starting at global index that goes until the end of the buffer. - /// - internal static Memory GetRemainingSliceOfBuffer(this IMemoryGroup group, long start) - where T : struct - { - Guard.NotNull(group, nameof(group)); - Guard.IsTrue(group.IsValid, nameof(group), "Group must be valid!"); - Guard.MustBeLessThan(start, group.TotalLength, nameof(start)); - - int bufferIdx = (int)(start / group.BufferLength); - - // if (bufferIdx < 0 || bufferIdx >= group.Count) - if ((uint)bufferIdx >= group.Count) - { - throw new ArgumentOutOfRangeException(nameof(start)); - } - - int bufferStart = (int)(start % group.BufferLength); - - Memory memory = group[bufferIdx]; - - return memory.Slice(bufferStart); - } - internal static void CopyTo(this IMemoryGroup source, Span target) where T : struct { diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs new file mode 100644 index 0000000000..f2e02bcfe3 --- /dev/null +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroupSpanCache.cs @@ -0,0 +1,52 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Buffers; +using SixLabors.ImageSharp.Memory.Internals; + +namespace SixLabors.ImageSharp.Memory +{ + /// + /// Cached pointer or array data enabling fast access from + /// known implementations. + /// + internal unsafe struct MemoryGroupSpanCache + { + public SpanCacheMode Mode; + public byte[] SingleArray; + public void* SinglePointer; + public void*[] MultiPointer; + + public static MemoryGroupSpanCache Create(IMemoryOwner[] memoryOwners) + where T : struct + { + IMemoryOwner owner0 = memoryOwners[0]; + MemoryGroupSpanCache memoryGroupSpanCache = default; + if (memoryOwners.Length == 1) + { + if (owner0 is SharedArrayPoolBuffer sharedPoolBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.SingleArray; + memoryGroupSpanCache.SingleArray = sharedPoolBuffer.Array; + } + else if (owner0 is UnmanagedBuffer unmanagedBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.SinglePointer; + memoryGroupSpanCache.SinglePointer = unmanagedBuffer.Pointer; + } + } + else if (owner0 is UnmanagedBuffer) + { + memoryGroupSpanCache.Mode = SpanCacheMode.MultiPointer; + memoryGroupSpanCache.MultiPointer = new void*[memoryOwners.Length]; + for (int i = 0; i < memoryOwners.Length; i++) + { + memoryGroupSpanCache.MultiPointer[i] = ((UnmanagedBuffer)memoryOwners[i]).Pointer; + } + } + + return memoryGroupSpanCache; + } + } +} diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs index 3b92413833..21faf8e562 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.Owned.cs @@ -26,6 +26,7 @@ namespace SixLabors.ImageSharp.Memory this.memoryOwners = memoryOwners; this.Swappable = swappable; this.View = new MemoryGroupView(this); + this.memoryGroupSpanCache = MemoryGroupSpanCache.Create(memoryOwners); } public Owned( @@ -122,6 +123,12 @@ namespace SixLabors.ImageSharp.Memory } } + public override void RecreateViewAfterSwap() + { + this.View.Invalidate(); + this.View = new MemoryGroupView(this); + } + /// IEnumerator> IEnumerable>.GetEnumerator() { @@ -167,32 +174,6 @@ namespace SixLabors.ImageSharp.Memory [MethodImpl(MethodImplOptions.NoInlining)] private static void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(MemoryGroup)); - internal static void SwapContents(Owned a, Owned b) - { - a.EnsureNotDisposed(); - b.EnsureNotDisposed(); - - IMemoryOwner[] tempOwners = a.memoryOwners; - long tempTotalLength = a.TotalLength; - int tempBufferLength = a.BufferLength; - RefCountedLifetimeGuard tempGroupOwner = a.groupLifetimeGuard; - - a.memoryOwners = b.memoryOwners; - a.TotalLength = b.TotalLength; - a.BufferLength = b.BufferLength; - a.groupLifetimeGuard = b.groupLifetimeGuard; - - b.memoryOwners = tempOwners; - b.TotalLength = tempTotalLength; - b.BufferLength = tempBufferLength; - b.groupLifetimeGuard = tempGroupOwner; - - a.View.Invalidate(); - b.View.Invalidate(); - a.View = new MemoryGroupView(a); - b.View = new MemoryGroupView(b); - } - // When the MemoryGroup points to multiple buffers via `groupLifetimeGuard`, // the lifetime of the individual buffers is managed by the guard. // Group buffer IMemoryOwner-s d not manage ownership. diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs index cdd8e6a758..9844f4a3bc 100644 --- a/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/MemoryGroup{T}.cs @@ -6,6 +6,8 @@ using System.Buffers; using System.Collections; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; using SixLabors.ImageSharp.Memory.Internals; namespace SixLabors.ImageSharp.Memory @@ -21,6 +23,8 @@ namespace SixLabors.ImageSharp.Memory { private static readonly int ElementSize = Unsafe.SizeOf(); + private MemoryGroupSpanCache memoryGroupSpanCache; + private MemoryGroup(int bufferLength, long totalLength) { this.BufferLength = bufferLength; @@ -31,10 +35,10 @@ namespace SixLabors.ImageSharp.Memory public abstract int Count { get; } /// - public int BufferLength { get; private set; } + public int BufferLength { get; } /// - public long TotalLength { get; private set; } + public long TotalLength { get; } /// public bool IsValid { get; private set; } = true; @@ -241,32 +245,62 @@ namespace SixLabors.ImageSharp.Memory return new Owned(source, bufferLength, totalLength, false); } - /// - /// Swaps the contents of 'target' with 'source' if the buffers are allocated (1), - /// copies the contents of 'source' to 'target' otherwise (2). - /// Groups should be of same TotalLength in case 2. - /// - public static bool SwapOrCopyContent(MemoryGroup target, MemoryGroup source) + [MethodImpl(InliningOptions.ShortMethod)] + public unsafe Span GetRowSpanCoreUnsafe(int y, int width) { - if (source is Owned ownedSrc && ownedSrc.Swappable && - target is Owned ownedTarget && ownedTarget.Swappable) + switch (this.memoryGroupSpanCache.Mode) { - Owned.SwapContents(ownedTarget, ownedSrc); - return true; - } - else - { - if (target.TotalLength != source.TotalLength) + case SpanCacheMode.SingleArray: + { +#if SUPPORTS_CREATESPAN + ref byte b0 = ref MemoryMarshal.GetReference(this.memoryGroupSpanCache.SingleArray); + ref T e0 = ref Unsafe.As(ref b0); + e0 = ref Unsafe.Add(ref e0, y * width); + return MemoryMarshal.CreateSpan(ref e0, width); +#else + return MemoryMarshal.Cast(this.memoryGroupSpanCache.SingleArray).Slice(y * width, width); +#endif + + } + + case SpanCacheMode.SinglePointer: { - throw new InvalidMemoryOperationException( - "Trying to copy/swap incompatible buffers. This is most likely caused by applying an unsupported processor to wrapped-memory images."); + void* start = Unsafe.Add(this.memoryGroupSpanCache.SinglePointer, y * width); + return new Span(start, width); } - source.CopyTo(target); - return false; + case SpanCacheMode.MultiPointer: + { + this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart); + void* start = Unsafe.Add(this.memoryGroupSpanCache.MultiPointer[bufferIdx], bufferStart); + return new Span(start, width); + } + + default: + { + this.GetMultiBufferPosition(y, width, out int bufferIdx, out int bufferStart); + return this[bufferIdx].Span.Slice(bufferStart, width); + } } } + /// + /// Returns the slice of the buffer starting at global index that goes until the end of the buffer. + /// + public Span GetRemainingSliceOfBuffer(long start) + { + long bufferIdx = Math.DivRem(start, this.BufferLength, out long bufferStart); + Memory memory = this[(int)bufferIdx]; + return memory.Span.Slice((int)bufferStart); + } + + public static bool CanSwapContent(MemoryGroup target, MemoryGroup source) => + source is Owned { Swappable: true } && target is Owned { Swappable: true }; + + public virtual void RecreateViewAfterSwap() + { + } + public virtual void IncreaseRefCounts() { } @@ -274,5 +308,14 @@ namespace SixLabors.ImageSharp.Memory public virtual void DecreaseRefCounts() { } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void GetMultiBufferPosition(int y, int width, out int bufferIdx, out int bufferStart) + { + long start = y * (long)width; + long bufferIdxLong = Math.DivRem(start, this.BufferLength, out long bufferStartLong); + bufferIdx = (int)bufferIdxLong; + bufferStart = (int)bufferStartLong; + } } } diff --git a/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs b/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs new file mode 100644 index 0000000000..8bd32efa9c --- /dev/null +++ b/src/ImageSharp/Memory/DiscontiguousBuffers/SpanCacheMode.cs @@ -0,0 +1,16 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +namespace SixLabors.ImageSharp.Memory +{ + /// + /// Selects active values in . + /// + internal enum SpanCacheMode + { + Default = default, + SingleArray, + SinglePointer, + MultiPointer + } +} diff --git a/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs b/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs new file mode 100644 index 0000000000..01d06bf753 --- /dev/null +++ b/tests/ImageSharp.Benchmarks/General/Buffer2D_DangerousGetRowSpan.cs @@ -0,0 +1,46 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Attributes; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; + +namespace SixLabors.ImageSharp.Benchmarks.General +{ + public class Buffer2D_DangerousGetRowSpan + { + private const int Height = 1024; + + [Params(0.5, 2.0, 10.0)] public double SizeMegaBytes { get; set; } + + private Buffer2D buffer; + + [GlobalSetup] + public unsafe void Setup() + { + int totalElements = (int)(1024 * 1024 * this.SizeMegaBytes) / sizeof(Rgba32); + + int width = totalElements / Height; + MemoryAllocator allocator = Configuration.Default.MemoryAllocator; + this.buffer = allocator.Allocate2D(width, Height); + } + + [GlobalCleanup] + public void Cleanup() => this.buffer.Dispose(); + + [Benchmark] + public int DangerousGetRowSpan() => + this.buffer.DangerousGetRowSpan(1).Length + + this.buffer.DangerousGetRowSpan(Height - 1).Length; + + // BenchmarkDotNet=v0.13.0, OS=Windows 10.0.19044 + // Intel Core i9-10900X CPU 3.70GHz, 1 CPU, 20 logical and 10 physical cores + // + // | Method | SizeMegaBytes | Mean | Error | StdDev | + // |-------------------- |-------------- |----------:|----------:|----------:| + // | DangerousGetRowSpan | 0.5 | 7.498 ns | 0.1784 ns | 0.3394 ns | + // | DangerousGetRowSpan | 2 | 6.542 ns | 0.1565 ns | 0.3659 ns | + // | DangerousGetRowSpan | 10 | 38.556 ns | 0.6604 ns | 0.8587 ns | + } +} diff --git a/tests/ImageSharp.Tests/Memory/Buffer2DTests.SwapOrCopyContent.cs b/tests/ImageSharp.Tests/Memory/Buffer2DTests.SwapOrCopyContent.cs new file mode 100644 index 0000000000..f58136f738 --- /dev/null +++ b/tests/ImageSharp.Tests/Memory/Buffer2DTests.SwapOrCopyContent.cs @@ -0,0 +1,160 @@ +// Copyright (c) Six Labors. +// Licensed under the Apache License, Version 2.0. + +using System; +using System.Linq; +using SixLabors.ImageSharp.Memory; +using SixLabors.ImageSharp.PixelFormats; +using Xunit; + +namespace SixLabors.ImageSharp.Tests.Memory +{ + public partial class Buffer2DTests + { + public class SwapOrCopyContent + { + private readonly TestMemoryAllocator MemoryAllocator = new TestMemoryAllocator(); + + [Fact] + public void SwapOrCopyContent_WhenBothAllocated() + { + using (Buffer2D a = this.MemoryAllocator.Allocate2D(10, 5, AllocationOptions.Clean)) + using (Buffer2D b = this.MemoryAllocator.Allocate2D(3, 7, AllocationOptions.Clean)) + { + a[1, 3] = 666; + b[1, 3] = 444; + + Memory aa = a.FastMemoryGroup.Single(); + Memory bb = b.FastMemoryGroup.Single(); + + Buffer2D.SwapOrCopyContent(a, b); + + Assert.Equal(bb, a.FastMemoryGroup.Single()); + Assert.Equal(aa, b.FastMemoryGroup.Single()); + + Assert.Equal(new Size(3, 7), a.Size()); + Assert.Equal(new Size(10, 5), b.Size()); + + Assert.Equal(666, b[1, 3]); + Assert.Equal(444, a[1, 3]); + } + } + + [Fact] + public void SwapOrCopyContent_WhenDestinationIsOwned_ShouldNotSwapInDisposedSourceBuffer() + { + using var destData = MemoryGroup.Wrap(new int[100]); + using var dest = new Buffer2D(destData, 10, 10); + + using (Buffer2D source = this.MemoryAllocator.Allocate2D(10, 10, AllocationOptions.Clean)) + { + source[0, 0] = 1; + dest[0, 0] = 2; + + Buffer2D.SwapOrCopyContent(dest, source); + } + + int actual1 = dest.DangerousGetRowSpan(0)[0]; + int actual2 = dest.DangerousGetRowSpan(0)[0]; + int actual3 = dest.GetSafeRowMemory(0).Span[0]; + int actual5 = dest[0, 0]; + + Assert.Equal(1, actual1); + Assert.Equal(1, actual2); + Assert.Equal(1, actual3); + Assert.Equal(1, actual5); + } + + [Fact] + public void WhenBothAreMemoryOwners_ShouldSwap() + { + this.MemoryAllocator.BufferCapacityInBytes = sizeof(int) * 50; + using Buffer2D a = this.MemoryAllocator.Allocate2D(48, 2); + using Buffer2D b = this.MemoryAllocator.Allocate2D(50, 2); + + Memory a0 = a.FastMemoryGroup[0]; + Memory a1 = a.FastMemoryGroup[1]; + Memory b0 = b.FastMemoryGroup[0]; + Memory b1 = b.FastMemoryGroup[1]; + + bool swap = Buffer2D.SwapOrCopyContent(a, b); + Assert.True(swap); + + Assert.Equal(b0, a.FastMemoryGroup[0]); + Assert.Equal(b1, a.FastMemoryGroup[1]); + Assert.Equal(a0, b.FastMemoryGroup[0]); + Assert.Equal(a1, b.FastMemoryGroup[1]); + Assert.NotEqual(a.FastMemoryGroup[0], b.FastMemoryGroup[0]); + } + + [Fact] + public void WhenBothAreMemoryOwners_ShouldReplaceViews() + { + using Buffer2D a = this.MemoryAllocator.Allocate2D(100, 1); + using Buffer2D b = this.MemoryAllocator.Allocate2D(100, 2); + + a.FastMemoryGroup[0].Span[42] = 1; + b.FastMemoryGroup[0].Span[33] = 2; + MemoryGroupView aView0 = (MemoryGroupView)a.MemoryGroup; + MemoryGroupView bView0 = (MemoryGroupView)b.MemoryGroup; + + Buffer2D.SwapOrCopyContent(a, b); + Assert.False(aView0.IsValid); + Assert.False(bView0.IsValid); + Assert.ThrowsAny(() => _ = aView0[0].Span); + Assert.ThrowsAny(() => _ = bView0[0].Span); + + Assert.True(a.MemoryGroup.IsValid); + Assert.True(b.MemoryGroup.IsValid); + Assert.Equal(2, a.MemoryGroup[0].Span[33]); + Assert.Equal(1, b.MemoryGroup[0].Span[42]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void WhenDestIsNotAllocated_SameSize_ShouldCopy(bool sourceIsAllocated) + { + var data = new Rgba32[21]; + var color = new Rgba32(1, 2, 3, 4); + + using var destOwner = new TestMemoryManager(data); + using var dest = new Buffer2D(MemoryGroup.Wrap(destOwner.Memory), 21, 1); + + using Buffer2D source = this.MemoryAllocator.Allocate2D(21, 1); + + source.FastMemoryGroup[0].Span[10] = color; + + // Act: + bool swap = Buffer2D.SwapOrCopyContent(dest, source); + + // Assert: + Assert.False(swap); + Assert.Equal(color, dest.MemoryGroup[0].Span[10]); + Assert.NotEqual(source.FastMemoryGroup[0], dest.FastMemoryGroup[0]); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void WhenDestIsNotMemoryOwner_DifferentSize_Throws(bool sourceIsOwner) + { + var data = new Rgba32[21]; + var color = new Rgba32(1, 2, 3, 4); + + using var destOwner = new TestMemoryManager(data); + using var dest = new Buffer2D(MemoryGroup.Wrap(destOwner.Memory), 21, 1); + + using Buffer2D source = this.MemoryAllocator.Allocate2D(22, 1); + + source.FastMemoryGroup[0].Span[10] = color; + + // Act: + Assert.ThrowsAny(() => Buffer2D.SwapOrCopyContent(dest, source)); + + Assert.Equal(color, source.MemoryGroup[0].Span[10]); + Assert.NotEqual(color, dest.MemoryGroup[0].Span[10]); + } + } + } +} diff --git a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs index 73a0f4d60e..486fbe4640 100644 --- a/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs +++ b/tests/ImageSharp.Tests/Memory/Buffer2DTests.cs @@ -4,7 +4,6 @@ using System; using System.Buffers; using System.Diagnostics; -using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using SixLabors.ImageSharp.Memory; @@ -14,7 +13,7 @@ using Xunit; // ReSharper disable InconsistentNaming namespace SixLabors.ImageSharp.Tests.Memory { - public class Buffer2DTests + public partial class Buffer2DTests { // ReSharper disable once ClassNeverInstantiated.Local private class Assert : Xunit.Assert @@ -121,7 +120,7 @@ namespace SixLabors.ImageSharp.Tests.Memory [InlineData(200, 100, 30, 1, 0)] [InlineData(200, 100, 30, 2, 1)] [InlineData(200, 100, 30, 4, 2)] - public unsafe void GetRowSpanY(int bufferCapacity, int width, int height, int y, int expectedBufferIndex) + public unsafe void DangerousGetRowSpan_TestAllocator(int bufferCapacity, int width, int height, int y, int expectedBufferIndex) { this.MemoryAllocator.BufferCapacityInBytes = sizeof(TestStructs.Foo) * bufferCapacity; @@ -136,6 +135,57 @@ namespace SixLabors.ImageSharp.Tests.Memory } } + [Theory] + [InlineData(100, 5)] // Within shared pool + [InlineData(77, 11)] // Within shared pool + [InlineData(100, 19)] // Single unmanaged pooled buffer + [InlineData(103, 17)] // Single unmanaged pooled buffer + [InlineData(100, 22)] // 2 unmanaged pooled buffers + [InlineData(100, 99)] // 9 unmanaged pooled buffers + [InlineData(100, 120)] // 2 unpooled buffers + public unsafe void DangerousGetRowSpan_UnmanagedAllocator(int width, int height) + { + const int sharedPoolThreshold = 1_000; + const int poolBufferSize = 2_000; + const int maxPoolSize = 10_000; + const int unpooledBufferSize = 8_000; + + int elementSize = sizeof(TestStructs.Foo); + var allocator = new UniformUnmanagedMemoryPoolMemoryAllocator( + sharedPoolThreshold * elementSize, + poolBufferSize * elementSize, + maxPoolSize * elementSize, + unpooledBufferSize * elementSize); + + using Buffer2D buffer = allocator.Allocate2D(width, height); + + var rnd = new Random(42); + + for (int y = 0; y < buffer.Height; y++) + { + Span span = buffer.DangerousGetRowSpan(y); + for (int x = 0; x < span.Length; x++) + { + ref TestStructs.Foo e = ref span[x]; + e.A = rnd.Next(); + e.B = rnd.NextDouble(); + } + } + + // Re-seed + rnd = new Random(42); + for (int y = 0; y < buffer.Height; y++) + { + Span span = buffer.GetSafeRowMemory(y).Span; + for (int x = 0; x < span.Length; x++) + { + ref TestStructs.Foo e = ref span[x]; + Assert.True(rnd.Next() == e.A, $"Mismatch @ y={y} x={x}"); + Assert.True(rnd.NextDouble() == e.B, $"Mismatch @ y={y} x={x}"); + } + } + } + [Theory] [InlineData(10, 0, 0, 0)] [InlineData(10, 0, 2, 0)] @@ -153,7 +203,7 @@ namespace SixLabors.ImageSharp.Tests.Memory using Buffer2D buffer = this.MemoryAllocator.Allocate2D(3, 5); bool expectSuccess = expectedBufferIndex >= 0; - bool success = buffer.TryGetPaddedRowSpan(y, padding, out Span paddedSpan); + bool success = buffer.DangerousTryGetPaddedRowSpan(y, padding, out Span paddedSpan); Xunit.Assert.Equal(expectSuccess, success); if (success) { @@ -229,56 +279,6 @@ namespace SixLabors.ImageSharp.Tests.Memory } } - [Fact] - public void SwapOrCopyContent_WhenBothAllocated() - { - using (Buffer2D a = this.MemoryAllocator.Allocate2D(10, 5, AllocationOptions.Clean)) - using (Buffer2D b = this.MemoryAllocator.Allocate2D(3, 7, AllocationOptions.Clean)) - { - a[1, 3] = 666; - b[1, 3] = 444; - - Memory aa = a.FastMemoryGroup.Single(); - Memory bb = b.FastMemoryGroup.Single(); - - Buffer2D.SwapOrCopyContent(a, b); - - Assert.Equal(bb, a.FastMemoryGroup.Single()); - Assert.Equal(aa, b.FastMemoryGroup.Single()); - - Assert.Equal(new Size(3, 7), a.Size()); - Assert.Equal(new Size(10, 5), b.Size()); - - Assert.Equal(666, b[1, 3]); - Assert.Equal(444, a[1, 3]); - } - } - - [Fact] - public void SwapOrCopyContent_WhenDestinationIsOwned_ShouldNotSwapInDisposedSourceBuffer() - { - using var destData = MemoryGroup.Wrap(new int[100]); - using var dest = new Buffer2D(destData, 10, 10); - - using (Buffer2D source = this.MemoryAllocator.Allocate2D(10, 10, AllocationOptions.Clean)) - { - source[0, 0] = 1; - dest[0, 0] = 2; - - Buffer2D.SwapOrCopyContent(dest, source); - } - - int actual1 = dest.DangerousGetRowSpan(0)[0]; - int actual2 = dest.DangerousGetRowSpan(0)[0]; - int actual3 = dest.GetSafeRowMemory(0).Span[0]; - int actual5 = dest[0, 0]; - - Assert.Equal(1, actual1); - Assert.Equal(1, actual2); - Assert.Equal(1, actual3); - Assert.Equal(1, actual5); - } - [Theory] [InlineData(100, 20, 0, 90, 10)] [InlineData(100, 3, 0, 50, 50)] diff --git a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.SwapOrCopyContent.cs b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.SwapOrCopyContent.cs deleted file mode 100644 index 61b9f7a895..0000000000 --- a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.SwapOrCopyContent.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) Six Labors. -// Licensed under the Apache License, Version 2.0. - -using System; -using SixLabors.ImageSharp.Memory; -using SixLabors.ImageSharp.PixelFormats; -using Xunit; - -namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers -{ - public partial class MemoryGroupTests - { - public class SwapOrCopyContent : MemoryGroupTestsBase - { - [Fact] - public void WhenBothAreMemoryOwners_ShouldSwap() - { - this.MemoryAllocator.BufferCapacityInBytes = sizeof(int) * 50; - using MemoryGroup a = this.MemoryAllocator.AllocateGroup(100, 50); - using MemoryGroup b = this.MemoryAllocator.AllocateGroup(120, 50); - - Memory a0 = a[0]; - Memory a1 = a[1]; - Memory b0 = b[0]; - Memory b1 = b[1]; - - bool swap = MemoryGroup.SwapOrCopyContent(a, b); - - Assert.True(swap); - Assert.Equal(b0, a[0]); - Assert.Equal(b1, a[1]); - Assert.Equal(a0, b[0]); - Assert.Equal(a1, b[1]); - Assert.NotEqual(a[0], b[0]); - } - - [Fact] - public void WhenBothAreMemoryOwners_ShouldReplaceViews() - { - using MemoryGroup a = this.MemoryAllocator.AllocateGroup(100, 100); - using MemoryGroup b = this.MemoryAllocator.AllocateGroup(120, 100); - - a[0].Span[42] = 1; - b[0].Span[33] = 2; - MemoryGroupView aView0 = a.View; - MemoryGroupView bView0 = b.View; - - MemoryGroup.SwapOrCopyContent(a, b); - Assert.False(aView0.IsValid); - Assert.False(bView0.IsValid); - Assert.ThrowsAny(() => _ = aView0[0].Span); - Assert.ThrowsAny(() => _ = bView0[0].Span); - - Assert.True(a.View.IsValid); - Assert.True(b.View.IsValid); - Assert.Equal(2, a.View[0].Span[33]); - Assert.Equal(1, b.View[0].Span[42]); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public void WhenDestIsNotAllocated_SameSize_ShouldCopy(bool sourceIsAllocated) - { - var data = new Rgba32[21]; - var color = new Rgba32(1, 2, 3, 4); - - using var destOwner = new TestMemoryManager(data); - using var dest = MemoryGroup.Wrap(destOwner.Memory); - - using MemoryGroup source = this.MemoryAllocator.AllocateGroup(21, 30); - - source[0].Span[10] = color; - - // Act: - bool swap = MemoryGroup.SwapOrCopyContent(dest, source); - - // Assert: - Assert.False(swap); - Assert.Equal(color, dest[0].Span[10]); - Assert.NotEqual(source[0], dest[0]); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public void WhenDestIsNotMemoryOwner_DifferentSize_Throws(bool sourceIsOwner) - { - var data = new Rgba32[21]; - var color = new Rgba32(1, 2, 3, 4); - - using var destOwner = new TestMemoryManager(data); - var dest = MemoryGroup.Wrap(destOwner.Memory); - - using MemoryGroup source = this.MemoryAllocator.AllocateGroup(22, 30); - - source[0].Span[10] = color; - - // Act: - Assert.ThrowsAny(() => MemoryGroup.SwapOrCopyContent(dest, source)); - - Assert.Equal(color, source[0].Span[10]); - Assert.NotEqual(color, dest[0].Span[10]); - } - } - } -} diff --git a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs index 13e47bdee2..cb6c34b77c 100644 --- a/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs +++ b/tests/ImageSharp.Tests/Memory/DiscontiguousBuffers/MemoryGroupTests.cs @@ -146,7 +146,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers { using MemoryGroup group = this.CreateTestGroup(totalLength, bufferLength, true); - Memory slice = group.GetBoundedSlice(start, length); + Memory slice = group.GetBoundedMemorySlice(start, length); Assert.Equal(length, slice.Length); @@ -172,7 +172,7 @@ namespace SixLabors.ImageSharp.Tests.Memory.DiscontiguousBuffers public void GetBoundedSlice_WhenOverlapsBuffers_Throws(long totalLength, int bufferLength, long start, int length) { using MemoryGroup group = this.CreateTestGroup(totalLength, bufferLength, true); - Assert.ThrowsAny(() => group.GetBoundedSlice(start, length)); + Assert.ThrowsAny(() => group.GetBoundedMemorySlice(start, length)); } [Fact]