tsai
/
ImageSharp
mirror of https://github.com/SixLabors/ImageSharp
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Scale allocated chunk sizes to match allocations

pull/2006/head
James Jackson-South 4 years ago
parent
95cdd1cef5
commit
fd4e3a5108
2 changed files with 77 additions and 61 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 68
      src/ImageSharp/IO/ChunkedMemoryStream.cs
  2. 70
      tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs

68
src/ImageSharp/IO/ChunkedMemoryStream.cs
View File

@ -17,9 +17,19 @@ namespace SixLabors.ImageSharp.IO
internal sealed class ChunkedMemoryStream : Stream internal sealed class ChunkedMemoryStream : Stream
{ {
/// <summary> /// <summary>
/// The default length in bytes of each buffer chunk. /// The default length in bytes of each buffer chunk when allocating large buffers.
/// </summary> /// </summary>
public const int DefaultBufferLength = 128 * 1024; public const int DefaultLargeChunkSize = 1024 * 1024 * 4; // 4 Mb
/// <summary>
/// The threshold at which to switch to using the large buffer.
/// </summary>
public const int DefaultLargeChunkThreshold = DefaultLargeChunkSize / 4; // 1 Mb
/// <summary>
/// The default length in bytes of each buffer chunk when allocating small buffers.
/// </summary>
public const int DefaultSmallChunkSize = DefaultLargeChunkSize / 32; // 128 Kb
// The memory allocator. // The memory allocator.
private readonly MemoryAllocator allocator; private readonly MemoryAllocator allocator;
@ -27,8 +37,18 @@ namespace SixLabors.ImageSharp.IO
// Data // Data
private MemoryChunk memoryChunk; private MemoryChunk memoryChunk;
// The length of each buffer chunk // The length, in bytes, of each large buffer chunk
private readonly int chunkLength; private readonly int largeChunkSize;
// The length, in bytes of the total allocation threshold that triggers switching from
// small to large buffer chunks.
private readonly int largeChunkThreshold;
// The current length, in bytes, of each buffer chunk
private int chunkSize;
// The total allocation length, in bytes
private int totalAllocation;
// Has the stream been disposed. // Has the stream been disposed.
private bool isDisposed; private bool isDisposed;
@ -49,21 +69,15 @@ namespace SixLabors.ImageSharp.IO
/// Initializes a new instance of the <see cref="ChunkedMemoryStream"/> class. /// Initializes a new instance of the <see cref="ChunkedMemoryStream"/> class.
/// </summary> /// </summary>
public ChunkedMemoryStream(MemoryAllocator allocator) public ChunkedMemoryStream(MemoryAllocator allocator)
: this(DefaultBufferLength, allocator)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="ChunkedMemoryStream"/> class.
/// </summary>
/// <param name="bufferLength">The length, in bytes of each buffer chunk.</param>
/// <param name="allocator">The memory allocator.</param>
public ChunkedMemoryStream(int bufferLength, MemoryAllocator allocator)
{ {
Guard.MustBeGreaterThan(bufferLength, 0, nameof(bufferLength));
Guard.NotNull(allocator, nameof(allocator)); Guard.NotNull(allocator, nameof(allocator));
this.chunkLength = bufferLength; // Tweak our buffer sizes to take the minimum of the provided buffer sizes
// or values scaled from the allocator buffer capacity which provides us with the largest
// available contiguous buffer size.
this.largeChunkSize = Math.Min(DefaultLargeChunkSize, allocator.GetBufferCapacityInBytes());
this.largeChunkThreshold = Math.Min(DefaultLargeChunkThreshold, this.largeChunkSize / 4);
this.chunkSize = Math.Min(DefaultSmallChunkSize, this.largeChunkSize / 32);
this.allocator = allocator; this.allocator = allocator;
} }
@ -191,6 +205,9 @@ namespace SixLabors.ImageSharp.IO
case SeekOrigin.End: case SeekOrigin.End:
this.Position = this.Length + offset; this.Position = this.Length + offset;
break; break;
default:
ThrowInvalidSeek();
break;
} }
return this.Position; return this.Position;
@ -219,6 +236,7 @@ namespace SixLabors.ImageSharp.IO
this.memoryChunk = null; this.memoryChunk = null;
this.writeChunk = null; this.writeChunk = null;
this.readChunk = null; this.readChunk = null;
this.totalAllocation = 0;
} }
finally finally
{ {
@ -519,17 +537,25 @@ namespace SixLabors.ImageSharp.IO
} }
[MethodImpl(MethodImplOptions.NoInlining)] [MethodImpl(MethodImplOptions.NoInlining)]
private static void ThrowDisposed() private static void ThrowDisposed() => throw new ObjectDisposedException(null, "The stream is closed.");
=> throw new ObjectDisposedException(null, "The stream is closed.");
[MethodImpl(MethodImplOptions.NoInlining)] [MethodImpl(MethodImplOptions.NoInlining)]
private static void ThrowArgumentOutOfRange(string value) private static void ThrowArgumentOutOfRange(string value) => throw new ArgumentOutOfRangeException(value);
=> throw new ArgumentOutOfRangeException(value);
[MethodImpl(MethodImplOptions.NoInlining)]
private static void ThrowInvalidSeek() => throw new ArgumentException("Invalid seek origin.");
[MethodImpl(MethodImplOptions.AggressiveInlining)] [MethodImpl(MethodImplOptions.AggressiveInlining)]
private MemoryChunk AllocateMemoryChunk() private MemoryChunk AllocateMemoryChunk()
{ {
IMemoryOwner<byte> buffer = this.allocator.Allocate<byte>(this.chunkLength); if (this.totalAllocation >= this.largeChunkThreshold)
{
this.chunkSize = this.largeChunkSize;
}
IMemoryOwner<byte> buffer = this.allocator.Allocate<byte>(this.chunkSize);
this.totalAllocation += this.chunkSize;
return new MemoryChunk return new MemoryChunk
{ {
Buffer = buffer, Buffer = buffer,

70
tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs
View File

@ -20,17 +20,7 @@ namespace SixLabors.ImageSharp.Tests.IO
{ {
private readonly MemoryAllocator allocator; private readonly MemoryAllocator allocator;
public ChunkedMemoryStreamTests() public ChunkedMemoryStreamTests() => this.allocator = Configuration.Default.MemoryAllocator;
{
this.allocator = Configuration.Default.MemoryAllocator;
}
[Fact]
public void MemoryStream_Ctor_InvalidCapacities()
{
Assert.Throws<ArgumentOutOfRangeException>(() => new ChunkedMemoryStream(int.MinValue, this.allocator));
Assert.Throws<ArgumentOutOfRangeException>(() => new ChunkedMemoryStream(0, this.allocator));
}
[Fact] [Fact]
public void MemoryStream_GetPositionTest_Negative() public void MemoryStream_GetPositionTest_Negative()
@ -61,11 +51,11 @@ namespace SixLabors.ImageSharp.Tests.IO
} }
[Theory] [Theory]
[InlineData(ChunkedMemoryStream.DefaultBufferLength)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 1.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 1.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 4)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 4)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 5.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 5.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 8)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 16)]
public void MemoryStream_ReadByteTest(int length) public void MemoryStream_ReadByteTest(int length)
{ {
using MemoryStream ms = this.CreateTestStream(length); using MemoryStream ms = this.CreateTestStream(length);
@ -73,7 +63,7 @@ namespace SixLabors.ImageSharp.Tests.IO
ms.CopyTo(cms); ms.CopyTo(cms);
cms.Position = 0; cms.Position = 0;
var expected = ms.ToArray(); byte[] expected = ms.ToArray();
for (int i = 0; i < expected.Length; i++) for (int i = 0; i < expected.Length; i++)
{ {
@ -82,11 +72,11 @@ namespace SixLabors.ImageSharp.Tests.IO
} }
[Theory] [Theory]
[InlineData(ChunkedMemoryStream.DefaultBufferLength)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 1.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 1.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 4)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 4)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 5.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 5.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 8)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 16)]
public void MemoryStream_ReadByteBufferTest(int length) public void MemoryStream_ReadByteBufferTest(int length)
{ {
using MemoryStream ms = this.CreateTestStream(length); using MemoryStream ms = this.CreateTestStream(length);
@ -94,8 +84,8 @@ namespace SixLabors.ImageSharp.Tests.IO
ms.CopyTo(cms); ms.CopyTo(cms);
cms.Position = 0; cms.Position = 0;
var expected = ms.ToArray(); byte[] expected = ms.ToArray();
var buffer = new byte[2]; byte[] buffer = new byte[2];
for (int i = 0; i < expected.Length; i += 2) for (int i = 0; i < expected.Length; i += 2)
{ {
cms.Read(buffer); cms.Read(buffer);
@ -105,11 +95,11 @@ namespace SixLabors.ImageSharp.Tests.IO
} }
[Theory] [Theory]
[InlineData(ChunkedMemoryStream.DefaultBufferLength)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 1.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 1.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 4)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 4)]
[InlineData((int)(ChunkedMemoryStream.DefaultBufferLength * 5.5))] [InlineData((int)(ChunkedMemoryStream.DefaultSmallChunkSize * 5.5))]
[InlineData(ChunkedMemoryStream.DefaultBufferLength * 8)] [InlineData(ChunkedMemoryStream.DefaultSmallChunkSize * 16)]
public void MemoryStream_ReadByteBufferSpanTest(int length) public void MemoryStream_ReadByteBufferSpanTest(int length)
{ {
using MemoryStream ms = this.CreateTestStream(length); using MemoryStream ms = this.CreateTestStream(length);
@ -117,7 +107,7 @@ namespace SixLabors.ImageSharp.Tests.IO
ms.CopyTo(cms); ms.CopyTo(cms);
cms.Position = 0; cms.Position = 0;
var expected = ms.ToArray(); byte[] expected = ms.ToArray();
Span<byte> buffer = new byte[2]; Span<byte> buffer = new byte[2];
for (int i = 0; i < expected.Length; i += 2) for (int i = 0; i < expected.Length; i += 2)
{ {
@ -257,24 +247,24 @@ namespace SixLabors.ImageSharp.Tests.IO
public void MemoryStream_CopyTo_Invalid() public void MemoryStream_CopyTo_Invalid()
{ {
ChunkedMemoryStream memoryStream; ChunkedMemoryStream memoryStream;
const string BufferSize = "bufferSize"; const string bufferSize = nameof(bufferSize);
using (memoryStream = new ChunkedMemoryStream(this.allocator)) using (memoryStream = new ChunkedMemoryStream(this.allocator))
{ {
const string Destination = "destination"; const string destination = nameof(destination);
Assert.Throws<ArgumentNullException>(Destination, () => memoryStream.CopyTo(destination: null)); Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null));
// Validate the destination parameter first. // Validate the destination parameter first.
Assert.Throws<ArgumentNullException>(Destination, () => memoryStream.CopyTo(destination: null, bufferSize: 0)); Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null, bufferSize: 0));
Assert.Throws<ArgumentNullException>(Destination, () => memoryStream.CopyTo(destination: null, bufferSize: -1)); Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null, bufferSize: -1));
// Then bufferSize. // Then bufferSize.
Assert.Throws<ArgumentOutOfRangeException>(BufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // 0-length buffer doesn't make sense. Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // 0-length buffer doesn't make sense.
Assert.Throws<ArgumentOutOfRangeException>(BufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1));
} }
// After the Stream is disposed, we should fail on all CopyTos. // After the Stream is disposed, we should fail on all CopyTos.
Assert.Throws<ArgumentOutOfRangeException>(BufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // Not before bufferSize is validated. Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // Not before bufferSize is validated.
Assert.Throws<ArgumentOutOfRangeException>(BufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1)); Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1));
ChunkedMemoryStream disposedStream = memoryStream; ChunkedMemoryStream disposedStream = memoryStream;
@ -369,7 +359,7 @@ namespace SixLabors.ImageSharp.Tests.IO
private MemoryStream CreateTestStream(int length) private MemoryStream CreateTestStream(int length)
{ {
var buffer = new byte[length]; byte[] buffer = new byte[length];
var random = new Random(); var random = new Random();
random.NextBytes(buffer); random.NextBytes(buffer);

Write Preview
Loading…
Cancel
Save