Revert "Remove ChunkedMemoryStream"

This reverts commit 1e58db2205.

src/ImageSharp/Formats/Gif/GifDecoderCore.cs

@@ -317,7 +317,7 @@ internal sealed class GifDecoderCore : ImageDecoderCore
             bool isXmp = this.buffer.Span.StartsWith(GifConstants.XmpApplicationIdentificationBytes);
             if (isXmp && !this.skipMetadata)
             {
-                GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream);
+                GifXmpApplicationExtension extension = GifXmpApplicationExtension.Read(stream, this.memoryAllocator);
                 if (extension.Data.Length > 0)
                 {
                     this.metadata!.XmpProfile = new XmpProfile(extension.Data);

src/ImageSharp/Formats/Gif/Sections/GifXmpApplicationExtension.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors.
 // Licensed under the Six Labors Split License.
 
+using SixLabors.ImageSharp.IO;
 using SixLabors.ImageSharp.Memory;
 
 namespace SixLabors.ImageSharp.Formats.Gif;
@@ -25,10 +26,11 @@ internal readonly struct GifXmpApplicationExtension : IGifExtension
     /// Reads the XMP metadata from the specified stream.
     /// </summary>
     /// <param name="stream">The stream to read from.</param>
+    /// <param name="allocator">The memory allocator.</param>
     /// <returns>The XMP metadata</returns>
-    public static GifXmpApplicationExtension Read(Stream stream)
+    public static GifXmpApplicationExtension Read(Stream stream, MemoryAllocator allocator)
     {
-        byte[] xmpBytes = ReadXmpData(stream);
+        byte[] xmpBytes = ReadXmpData(stream, allocator);
 
         // Exclude the "magic trailer", see XMP Specification Part 3, 1.1.2 GIF
         int xmpLength = xmpBytes.Length - 256; // 257 - unread 0x0
@@ -69,9 +71,9 @@ internal readonly struct GifXmpApplicationExtension : IGifExtension
         return this.ContentLength;
     }
 
-    private static byte[] ReadXmpData(Stream stream)
+    private static byte[] ReadXmpData(Stream stream, MemoryAllocator allocator)
     {
-        using MemoryStream bytes = new();
+        using ChunkedMemoryStream bytes = new(allocator);
 
         // XMP data doesn't have a fixed length nor is there an indicator of the length.
         // So we simply read one byte at a time until we hit the 0x0 value at the end

src/ImageSharp/Formats/ImageDecoder.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors.
 // Licensed under the Six Labors Split License.
 
+using SixLabors.ImageSharp.IO;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
 
@@ -209,7 +210,7 @@ public abstract class ImageDecoder : IImageDecoder
         }
 
         Configuration configuration = options.Configuration;
-        using MemoryStream memoryStream = new();
+        using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator);
         stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize);
         memoryStream.Position = 0;
 
@@ -265,6 +266,11 @@ public abstract class ImageDecoder : IImageDecoder
             return PerformActionAndResetPosition(ms, ms.Position, cancellationToken);
         }
 
+        if (stream is ChunkedMemoryStream cms)
+        {
+            return PerformActionAndResetPosition(cms, cms.Position, cancellationToken);
+        }
+
         return CopyToMemoryStreamAndActionAsync(options, stream, PerformActionAndResetPosition, cancellationToken);
     }
 
@@ -276,11 +282,9 @@ public abstract class ImageDecoder : IImageDecoder
     {
         long position = stream.CanSeek ? stream.Position : 0;
         Configuration configuration = options.Configuration;
-
+        await using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator);
-        await using MemoryStream memoryStream = new();
         await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false);
         memoryStream.Position = 0;
-
         return await action(memoryStream, position, cancellationToken).ConfigureAwait(false);
     }
 

src/ImageSharp/Formats/ImageEncoder.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Six Labors.
 // Licensed under the Six Labors Split License.
 
+using SixLabors.ImageSharp.IO;
 using SixLabors.ImageSharp.PixelFormats;
 
 namespace SixLabors.ImageSharp.Formats;
@@ -47,8 +48,8 @@ public abstract class ImageEncoder : IImageEncoder
         }
         else
         {
-            using MemoryStream ms = new();
+            using ChunkedMemoryStream ms = new(configuration.MemoryAllocator);
-            this.Encode(image, ms, cancellationToken);
+            this.Encode(image, stream, cancellationToken);
             ms.Position = 0;
             ms.CopyTo(stream, configuration.StreamProcessingBufferSize);
         }
@@ -64,7 +65,7 @@ public abstract class ImageEncoder : IImageEncoder
         }
         else
         {
-            await using MemoryStream ms = new();
+            using ChunkedMemoryStream ms = new(configuration.MemoryAllocator);
             await DoEncodeAsync(ms);
             ms.Position = 0;
             await ms.CopyToAsync(stream, configuration.StreamProcessingBufferSize, cancellationToken)

src/ImageSharp/IO/ChunkedMemoryStream.cs

@@ -0,0 +1,585 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using System.Buffers;
+using System.Runtime.CompilerServices;
+using SixLabors.ImageSharp.Memory;
+
+namespace SixLabors.ImageSharp.IO;
+
+/// <summary>
+/// Provides an in-memory stream composed of non-contiguous chunks that doesn't need to be resized.
+/// Chunks are allocated by the <see cref="MemoryAllocator"/> assigned via the constructor
+/// and is designed to take advantage of buffer pooling when available.
+/// </summary>
+internal sealed class ChunkedMemoryStream : Stream
+{
+    // The memory allocator.
+    private readonly MemoryAllocator allocator;
+
+    // Data
+    private MemoryChunk? memoryChunk;
+
+    // The total number of allocated chunks
+    private int chunkCount;
+
+    // The length of the largest contiguous buffer that can be handled by the allocator.
+    private readonly int allocatorCapacity;
+
+    // Has the stream been disposed.
+    private bool isDisposed;
+
+    // Current chunk to write to
+    private MemoryChunk? writeChunk;
+
+    // Offset into chunk to write to
+    private int writeOffset;
+
+    // Current chunk to read from
+    private MemoryChunk? readChunk;
+
+    // Offset into chunk to read from
+    private int readOffset;
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="ChunkedMemoryStream"/> class.
+    /// </summary>
+    /// <param name="allocator">The memory allocator.</param>
+    public ChunkedMemoryStream(MemoryAllocator allocator)
+    {
+        this.allocatorCapacity = allocator.GetBufferCapacityInBytes();
+        this.allocator = allocator;
+    }
+
+    /// <inheritdoc/>
+    public override bool CanRead => !this.isDisposed;
+
+    /// <inheritdoc/>
+    public override bool CanSeek => !this.isDisposed;
+
+    /// <inheritdoc/>
+    public override bool CanWrite => !this.isDisposed;
+
+    /// <inheritdoc/>
+    public override long Length
+    {
+        get
+        {
+            this.EnsureNotDisposed();
+
+            int length = 0;
+            MemoryChunk? chunk = this.memoryChunk;
+            while (chunk != null)
+            {
+                MemoryChunk? next = chunk.Next;
+                if (next != null)
+                {
+                    length += chunk.Length;
+                }
+                else
+                {
+                    length += this.writeOffset;
+                }
+
+                chunk = next;
+            }
+
+            return length;
+        }
+    }
+
+    /// <inheritdoc/>
+    public override long Position
+    {
+        get
+        {
+            this.EnsureNotDisposed();
+
+            if (this.readChunk is null)
+            {
+                return 0;
+            }
+
+            int pos = 0;
+            MemoryChunk? chunk = this.memoryChunk;
+            while (chunk != this.readChunk && chunk is not null)
+            {
+                pos += chunk.Length;
+                chunk = chunk.Next;
+            }
+
+            pos += this.readOffset;
+
+            return pos;
+        }
+
+        set
+        {
+            this.EnsureNotDisposed();
+
+            if (value < 0)
+            {
+                ThrowArgumentOutOfRange(nameof(value));
+            }
+
+            // Back up current position in case new position is out of range
+            MemoryChunk? backupReadChunk = this.readChunk;
+            int backupReadOffset = this.readOffset;
+
+            this.readChunk = null;
+            this.readOffset = 0;
+
+            int leftUntilAtPos = (int)value;
+            MemoryChunk? chunk = this.memoryChunk;
+            while (chunk != null)
+            {
+                if ((leftUntilAtPos < chunk.Length)
+                        || ((leftUntilAtPos == chunk.Length)
+                        && (chunk.Next is null)))
+                {
+                    // The desired position is in this chunk
+                    this.readChunk = chunk;
+                    this.readOffset = leftUntilAtPos;
+                    break;
+                }
+
+                leftUntilAtPos -= chunk.Length;
+                chunk = chunk.Next;
+            }
+
+            if (this.readChunk is null)
+            {
+                // Position is out of range
+                this.readChunk = backupReadChunk;
+                this.readOffset = backupReadOffset;
+            }
+        }
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override long Seek(long offset, SeekOrigin origin)
+    {
+        this.EnsureNotDisposed();
+
+        switch (origin)
+        {
+            case SeekOrigin.Begin:
+                this.Position = offset;
+                break;
+
+            case SeekOrigin.Current:
+                this.Position += offset;
+                break;
+
+            case SeekOrigin.End:
+                this.Position = this.Length + offset;
+                break;
+            default:
+                ThrowInvalidSeek();
+                break;
+        }
+
+        return this.Position;
+    }
+
+    /// <inheritdoc/>
+    public override void SetLength(long value)
+        => throw new NotSupportedException();
+
+    /// <inheritdoc/>
+    protected override void Dispose(bool disposing)
+    {
+        if (this.isDisposed)
+        {
+            return;
+        }
+
+        try
+        {
+            this.isDisposed = true;
+            if (disposing)
+            {
+                ReleaseMemoryChunks(this.memoryChunk);
+            }
+
+            this.memoryChunk = null;
+            this.writeChunk = null;
+            this.readChunk = null;
+            this.chunkCount = 0;
+        }
+        finally
+        {
+            base.Dispose(disposing);
+        }
+    }
+
+    /// <inheritdoc/>
+    public override void Flush()
+    {
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override int Read(byte[] buffer, int offset, int count)
+    {
+        Guard.NotNull(buffer, nameof(buffer));
+        Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset));
+        Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count));
+
+        const string bufferMessage = "Offset subtracted from the buffer length is less than count.";
+        Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage);
+
+        return this.ReadImpl(buffer.AsSpan(offset, count));
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override int Read(Span<byte> buffer) => this.ReadImpl(buffer);
+
+    private int ReadImpl(Span<byte> buffer)
+    {
+        this.EnsureNotDisposed();
+
+        if (this.readChunk is null)
+        {
+            if (this.memoryChunk is null)
+            {
+                return 0;
+            }
+
+            this.readChunk = this.memoryChunk;
+            this.readOffset = 0;
+        }
+
+        IMemoryOwner<byte> chunkBuffer = this.readChunk.Buffer;
+        int chunkSize = this.readChunk.Length;
+        if (this.readChunk.Next is null)
+        {
+            chunkSize = this.writeOffset;
+        }
+
+        int bytesRead = 0;
+        int offset = 0;
+        int count = buffer.Length;
+        while (count > 0)
+        {
+            if (this.readOffset == chunkSize)
+            {
+                // Exit if no more chunks are currently available
+                if (this.readChunk.Next is null)
+                {
+                    break;
+                }
+
+                this.readChunk = this.readChunk.Next;
+                this.readOffset = 0;
+                chunkBuffer = this.readChunk.Buffer;
+                chunkSize = this.readChunk.Length;
+                if (this.readChunk.Next is null)
+                {
+                    chunkSize = this.writeOffset;
+                }
+            }
+
+            int readCount = Math.Min(count, chunkSize - this.readOffset);
+            chunkBuffer.Slice(this.readOffset, readCount).CopyTo(buffer[offset..]);
+            offset += readCount;
+            count -= readCount;
+            this.readOffset += readCount;
+            bytesRead += readCount;
+        }
+
+        return bytesRead;
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override int ReadByte()
+    {
+        this.EnsureNotDisposed();
+
+        if (this.readChunk is null)
+        {
+            if (this.memoryChunk is null)
+            {
+                return 0;
+            }
+
+            this.readChunk = this.memoryChunk;
+            this.readOffset = 0;
+        }
+
+        IMemoryOwner<byte> chunkBuffer = this.readChunk.Buffer;
+        int chunkSize = this.readChunk.Length;
+        if (this.readChunk.Next is null)
+        {
+            chunkSize = this.writeOffset;
+        }
+
+        if (this.readOffset == chunkSize)
+        {
+            // Exit if no more chunks are currently available
+            if (this.readChunk.Next is null)
+            {
+                return -1;
+            }
+
+            this.readChunk = this.readChunk.Next;
+            this.readOffset = 0;
+            chunkBuffer = this.readChunk.Buffer;
+        }
+
+        return chunkBuffer.GetSpan()[this.readOffset++];
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override void Write(byte[] buffer, int offset, int count)
+    {
+        Guard.NotNull(buffer, nameof(buffer));
+        Guard.MustBeGreaterThanOrEqualTo(offset, 0, nameof(offset));
+        Guard.MustBeGreaterThanOrEqualTo(count, 0, nameof(count));
+
+        const string bufferMessage = "Offset subtracted from the buffer length is less than count.";
+        Guard.IsFalse(buffer.Length - offset < count, nameof(buffer), bufferMessage);
+
+        this.WriteImpl(buffer.AsSpan(offset, count));
+    }
+
+    /// <inheritdoc/>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public override void Write(ReadOnlySpan<byte> buffer) => this.WriteImpl(buffer);
+
+    private void WriteImpl(ReadOnlySpan<byte> buffer)
+    {
+        this.EnsureNotDisposed();
+
+        if (this.memoryChunk is null)
+        {
+            this.memoryChunk = this.AllocateMemoryChunk();
+            this.writeChunk = this.memoryChunk;
+            this.writeOffset = 0;
+        }
+
+        Guard.NotNull(this.writeChunk);
+
+        Span<byte> chunkBuffer = this.writeChunk.Buffer.GetSpan();
+        int chunkSize = this.writeChunk.Length;
+        int count = buffer.Length;
+        int offset = 0;
+        while (count > 0)
+        {
+            if (this.writeOffset == chunkSize)
+            {
+                // Allocate a new chunk if the current one is full
+                this.writeChunk.Next = this.AllocateMemoryChunk();
+                this.writeChunk = this.writeChunk.Next;
+                this.writeOffset = 0;
+                chunkBuffer = this.writeChunk.Buffer.GetSpan();
+                chunkSize = this.writeChunk.Length;
+            }
+
+            int copyCount = Math.Min(count, chunkSize - this.writeOffset);
+            buffer.Slice(offset, copyCount).CopyTo(chunkBuffer[this.writeOffset..]);
+
+            offset += copyCount;
+            count -= copyCount;
+            this.writeOffset += copyCount;
+        }
+    }
+
+    /// <inheritdoc/>
+    public override void WriteByte(byte value)
+    {
+        this.EnsureNotDisposed();
+
+        if (this.memoryChunk is null)
+        {
+            this.memoryChunk = this.AllocateMemoryChunk();
+            this.writeChunk = this.memoryChunk;
+            this.writeOffset = 0;
+        }
+
+        Guard.NotNull(this.writeChunk);
+
+        IMemoryOwner<byte> chunkBuffer = this.writeChunk.Buffer;
+        int chunkSize = this.writeChunk.Length;
+
+        if (this.writeOffset == chunkSize)
+        {
+            // Allocate a new chunk if the current one is full
+            this.writeChunk.Next = this.AllocateMemoryChunk();
+            this.writeChunk = this.writeChunk.Next;
+            this.writeOffset = 0;
+            chunkBuffer = this.writeChunk.Buffer;
+        }
+
+        chunkBuffer.GetSpan()[this.writeOffset++] = value;
+    }
+
+    /// <summary>
+    /// Copy entire buffer into an array.
+    /// </summary>
+    /// <returns>The <see cref="T:byte[]"/>.</returns>
+    public byte[] ToArray()
+    {
+        int length = (int)this.Length; // This will throw if stream is closed
+        byte[] copy = new byte[this.Length];
+
+        MemoryChunk? backupReadChunk = this.readChunk;
+        int backupReadOffset = this.readOffset;
+
+        this.readChunk = this.memoryChunk;
+        this.readOffset = 0;
+        this.Read(copy, 0, length);
+
+        this.readChunk = backupReadChunk;
+        this.readOffset = backupReadOffset;
+
+        return copy;
+    }
+
+    /// <summary>
+    /// Write remainder of this stream to another stream.
+    /// </summary>
+    /// <param name="stream">The stream to write to.</param>
+    public void WriteTo(Stream stream)
+    {
+        this.EnsureNotDisposed();
+
+        Guard.NotNull(stream, nameof(stream));
+
+        if (this.readChunk is null)
+        {
+            if (this.memoryChunk is null)
+            {
+                return;
+            }
+
+            this.readChunk = this.memoryChunk;
+            this.readOffset = 0;
+        }
+
+        IMemoryOwner<byte> chunkBuffer = this.readChunk.Buffer;
+        int chunkSize = this.readChunk.Length;
+        if (this.readChunk.Next is null)
+        {
+            chunkSize = this.writeOffset;
+        }
+
+        // Following code mirrors Read() logic (readChunk/readOffset should
+        // point just past last byte of last chunk when done)
+        // loop until end of chunks is found
+        while (true)
+        {
+            if (this.readOffset == chunkSize)
+            {
+                // Exit if no more chunks are currently available
+                if (this.readChunk.Next is null)
+                {
+                    break;
+                }
+
+                this.readChunk = this.readChunk.Next;
+                this.readOffset = 0;
+                chunkBuffer = this.readChunk.Buffer;
+                chunkSize = this.readChunk.Length;
+                if (this.readChunk.Next is null)
+                {
+                    chunkSize = this.writeOffset;
+                }
+            }
+
+            int writeCount = chunkSize - this.readOffset;
+            stream.Write(chunkBuffer.GetSpan(), this.readOffset, writeCount);
+            this.readOffset = chunkSize;
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private void EnsureNotDisposed()
+    {
+        if (this.isDisposed)
+        {
+            ThrowDisposed();
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ThrowDisposed() => throw new ObjectDisposedException(null, "The stream is closed.");
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ThrowArgumentOutOfRange(string value) => throw new ArgumentOutOfRangeException(value);
+
+    [MethodImpl(MethodImplOptions.NoInlining)]
+    private static void ThrowInvalidSeek() => throw new ArgumentException("Invalid seek origin.");
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private MemoryChunk AllocateMemoryChunk()
+    {
+        // Tweak our buffer sizes to take the minimum of the provided buffer sizes
+        // or the allocator buffer capacity which provides us with the largest
+        // available contiguous buffer size.
+        IMemoryOwner<byte> buffer = this.allocator.Allocate<byte>(Math.Min(this.allocatorCapacity, GetChunkSize(this.chunkCount++)));
+
+        return new MemoryChunk(buffer)
+        {
+            Next = null,
+            Length = buffer.Length()
+        };
+    }
+
+    private static void ReleaseMemoryChunks(MemoryChunk? chunk)
+    {
+        while (chunk != null)
+        {
+            chunk.Dispose();
+            chunk = chunk.Next;
+        }
+    }
+
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    private static int GetChunkSize(int i)
+    {
+        // Increment chunks sizes with moderate speed, but without using too many buffers from the same ArrayPool bucket of the default MemoryAllocator.
+        // https://github.com/SixLabors/ImageSharp/pull/2006#issuecomment-1066244720
+#pragma warning disable IDE1006 // Naming Styles
+        const int _128K = 1 << 17;
+        const int _4M = 1 << 22;
+        return i < 16 ? _128K * (1 << (int)((uint)i / 4)) : _4M;
+#pragma warning restore IDE1006 // Naming Styles
+    }
+
+    private sealed class MemoryChunk : IDisposable
+    {
+        private bool isDisposed;
+
+        public MemoryChunk(IMemoryOwner<byte> buffer) => this.Buffer = buffer;
+
+        public IMemoryOwner<byte> Buffer { get; }
+
+        public MemoryChunk? Next { get; set; }
+
+        public int Length { get; init; }
+
+        private void Dispose(bool disposing)
+        {
+            if (!this.isDisposed)
+            {
+                if (disposing)
+                {
+                    this.Buffer.Dispose();
+                }
+
+                this.isDisposed = true;
+            }
+        }
+
+        public void Dispose()
+        {
+            this.Dispose(disposing: true);
+            GC.SuppressFinalize(this);
+        }
+    }
+}

src/ImageSharp/Image.FromStream.cs

@@ -2,6 +2,7 @@
 // Licensed under the Six Labors Split License.
 
 using SixLabors.ImageSharp.Formats;
+using SixLabors.ImageSharp.IO;
 using SixLabors.ImageSharp.PixelFormats;
 
 namespace SixLabors.ImageSharp;
@@ -300,7 +301,7 @@ public abstract partial class Image
             return action(stream);
         }
 
-        using MemoryStream memoryStream = new();
+        using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator);
         stream.CopyTo(memoryStream, configuration.StreamProcessingBufferSize);
         memoryStream.Position = 0;
 
@@ -342,7 +343,7 @@ public abstract partial class Image
             return await action(stream, cancellationToken).ConfigureAwait(false);
         }
 
-        await using MemoryStream memoryStream = new();
+        using ChunkedMemoryStream memoryStream = new(configuration.MemoryAllocator);
         await stream.CopyToAsync(memoryStream, configuration.StreamProcessingBufferSize, cancellationToken).ConfigureAwait(false);
         memoryStream.Position = 0;
 

tests/ImageSharp.Tests/Formats/WebP/WebpEncoderTests.cs

@@ -544,25 +544,6 @@ public class WebpEncoderTests
     [Fact]
     public void RunEncodeLossy_WithPeakImage_WithoutHardwareIntrinsics_Works() => FeatureTestRunner.RunWithHwIntrinsicsFeature(RunEncodeLossy_WithPeakImage, HwIntrinsics.DisableHWIntrinsic);
 
-    [Theory]
-    [WithFile(TestPatternOpaque, PixelTypes.Rgba32)]
-    public void CanSave_NonSeekableStream<TPixel>(TestImageProvider<TPixel> provider)
-        where TPixel : unmanaged, IPixel<TPixel>
-    {
-        using Image<TPixel> image = provider.GetImage();
-        WebpEncoder encoder = new();
-
-        using MemoryStream seekable = new();
-        image.Save(seekable, encoder);
-
-        using MemoryStream memoryStream = new();
-        using NonSeekableStream nonSeekable = new(memoryStream);
-
-        image.Save(nonSeekable, encoder);
-
-        Assert.True(seekable.ToArray().SequenceEqual(memoryStream.ToArray()));
-    }
-
     private static ImageComparer GetComparer(int quality)
     {
         float tolerance = 0.01f; // ~1.0%

tests/ImageSharp.Tests/IO/ChunkedMemoryStreamTests.cs

@@ -0,0 +1,373 @@
+// Copyright (c) Six Labors.
+// Licensed under the Six Labors Split License.
+
+using SixLabors.ImageSharp.IO;
+using SixLabors.ImageSharp.Memory;
+using SixLabors.ImageSharp.PixelFormats;
+using SixLabors.ImageSharp.Tests.TestUtilities.ImageComparison;
+
+namespace SixLabors.ImageSharp.Tests.IO;
+
+/// <summary>
+/// Tests for the <see cref="ChunkedMemoryStream"/> class.
+/// </summary>
+public class ChunkedMemoryStreamTests
+{
+    /// <summary>
+    /// The default length in bytes of each buffer chunk when allocating large buffers.
+    /// </summary>
+    private const int DefaultLargeChunkSize = 1024 * 1024 * 4; // 4 Mb
+
+    /// <summary>
+    /// The default length in bytes of each buffer chunk when allocating small buffers.
+    /// </summary>
+    private const int DefaultSmallChunkSize = DefaultLargeChunkSize / 32; // 128 Kb
+
+    private readonly MemoryAllocator allocator;
+
+    public ChunkedMemoryStreamTests() => this.allocator = Configuration.Default.MemoryAllocator;
+
+    [Fact]
+    public void MemoryStream_GetPositionTest_Negative()
+    {
+        using var ms = new ChunkedMemoryStream(this.allocator);
+        long iCurrentPos = ms.Position;
+        for (int i = -1; i > -6; i--)
+        {
+            Assert.Throws<ArgumentOutOfRangeException>(() => ms.Position = i);
+            Assert.Equal(ms.Position, iCurrentPos);
+        }
+    }
+
+    [Fact]
+    public void MemoryStream_ReadTest_Negative()
+    {
+        var ms2 = new ChunkedMemoryStream(this.allocator);
+
+        Assert.Throws<ArgumentNullException>(() => ms2.Read(null, 0, 0));
+        Assert.Throws<ArgumentOutOfRangeException>(() => ms2.Read(new byte[] { 1 }, -1, 0));
+        Assert.Throws<ArgumentOutOfRangeException>(() => ms2.Read(new byte[] { 1 }, 0, -1));
+        Assert.Throws<ArgumentException>(() => ms2.Read(new byte[] { 1 }, 2, 0));
+        Assert.Throws<ArgumentException>(() => ms2.Read(new byte[] { 1 }, 0, 2));
+
+        ms2.Dispose();
+
+        Assert.Throws<ObjectDisposedException>(() => ms2.Read(new byte[] { 1 }, 0, 1));
+    }
+
+    [Theory]
+    [InlineData(DefaultSmallChunkSize)]
+    [InlineData((int)(DefaultSmallChunkSize * 1.5))]
+    [InlineData(DefaultSmallChunkSize * 4)]
+    [InlineData((int)(DefaultSmallChunkSize * 5.5))]
+    [InlineData(DefaultSmallChunkSize * 16)]
+    public void MemoryStream_ReadByteTest(int length)
+    {
+        using MemoryStream ms = this.CreateTestStream(length);
+        using var cms = new ChunkedMemoryStream(this.allocator);
+
+        ms.CopyTo(cms);
+        cms.Position = 0;
+        byte[] expected = ms.ToArray();
+
+        for (int i = 0; i < expected.Length; i++)
+        {
+            Assert.Equal(expected[i], cms.ReadByte());
+        }
+    }
+
+    [Theory]
+    [InlineData(DefaultSmallChunkSize)]
+    [InlineData((int)(DefaultSmallChunkSize * 1.5))]
+    [InlineData(DefaultSmallChunkSize * 4)]
+    [InlineData((int)(DefaultSmallChunkSize * 5.5))]
+    [InlineData(DefaultSmallChunkSize * 16)]
+    public void MemoryStream_ReadByteBufferTest(int length)
+    {
+        using MemoryStream ms = this.CreateTestStream(length);
+        using var cms = new ChunkedMemoryStream(this.allocator);
+
+        ms.CopyTo(cms);
+        cms.Position = 0;
+        byte[] expected = ms.ToArray();
+        byte[] buffer = new byte[2];
+        for (int i = 0; i < expected.Length; i += 2)
+        {
+            cms.Read(buffer);
+            Assert.Equal(expected[i], buffer[0]);
+            Assert.Equal(expected[i + 1], buffer[1]);
+        }
+    }
+
+    [Theory]
+    [InlineData(DefaultSmallChunkSize)]
+    [InlineData((int)(DefaultSmallChunkSize * 1.5))]
+    [InlineData(DefaultSmallChunkSize * 4)]
+    [InlineData((int)(DefaultSmallChunkSize * 5.5))]
+    [InlineData(DefaultSmallChunkSize * 16)]
+    public void MemoryStream_ReadByteBufferSpanTest(int length)
+    {
+        using MemoryStream ms = this.CreateTestStream(length);
+        using var cms = new ChunkedMemoryStream(this.allocator);
+
+        ms.CopyTo(cms);
+        cms.Position = 0;
+        byte[] expected = ms.ToArray();
+        Span<byte> buffer = new byte[2];
+        for (int i = 0; i < expected.Length; i += 2)
+        {
+            cms.Read(buffer);
+            Assert.Equal(expected[i], buffer[0]);
+            Assert.Equal(expected[i + 1], buffer[1]);
+        }
+    }
+
+    [Fact]
+    public void MemoryStream_WriteToTests()
+    {
+        using (var ms2 = new ChunkedMemoryStream(this.allocator))
+        {
+            byte[] bytArrRet;
+            byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 };
+
+            // [] Write to memoryStream, check the memoryStream
+            ms2.Write(bytArr, 0, bytArr.Length);
+
+            using var readonlyStream = new ChunkedMemoryStream(this.allocator);
+            ms2.WriteTo(readonlyStream);
+            readonlyStream.Flush();
+            readonlyStream.Position = 0;
+            bytArrRet = new byte[(int)readonlyStream.Length];
+            readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length);
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                Assert.Equal(bytArr[i], bytArrRet[i]);
+            }
+        }
+
+        // [] Write to memoryStream, check the memoryStream
+        using (var ms2 = new ChunkedMemoryStream(this.allocator))
+        using (var ms3 = new ChunkedMemoryStream(this.allocator))
+        {
+            byte[] bytArrRet;
+            byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 };
+
+            ms2.Write(bytArr, 0, bytArr.Length);
+            ms2.WriteTo(ms3);
+            ms3.Position = 0;
+            bytArrRet = new byte[(int)ms3.Length];
+            ms3.Read(bytArrRet, 0, (int)ms3.Length);
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                Assert.Equal(bytArr[i], bytArrRet[i]);
+            }
+        }
+    }
+
+    [Fact]
+    public void MemoryStream_WriteToSpanTests()
+    {
+        using (var ms2 = new ChunkedMemoryStream(this.allocator))
+        {
+            Span<byte> bytArrRet;
+            Span<byte> bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 };
+
+            // [] Write to memoryStream, check the memoryStream
+            ms2.Write(bytArr, 0, bytArr.Length);
+
+            using var readonlyStream = new ChunkedMemoryStream(this.allocator);
+            ms2.WriteTo(readonlyStream);
+            readonlyStream.Flush();
+            readonlyStream.Position = 0;
+            bytArrRet = new byte[(int)readonlyStream.Length];
+            readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length);
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                Assert.Equal(bytArr[i], bytArrRet[i]);
+            }
+        }
+
+        // [] Write to memoryStream, check the memoryStream
+        using (var ms2 = new ChunkedMemoryStream(this.allocator))
+        using (var ms3 = new ChunkedMemoryStream(this.allocator))
+        {
+            Span<byte> bytArrRet;
+            Span<byte> bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 };
+
+            ms2.Write(bytArr, 0, bytArr.Length);
+            ms2.WriteTo(ms3);
+            ms3.Position = 0;
+            bytArrRet = new byte[(int)ms3.Length];
+            ms3.Read(bytArrRet, 0, (int)ms3.Length);
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                Assert.Equal(bytArr[i], bytArrRet[i]);
+            }
+        }
+    }
+
+    [Fact]
+    public void MemoryStream_WriteByteTests()
+    {
+        using (var ms2 = new ChunkedMemoryStream(this.allocator))
+        {
+            byte[] bytArrRet;
+            byte[] bytArr = new byte[] { byte.MinValue, byte.MaxValue, 1, 2, 3, 4, 5, 6, 128, 250 };
+
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                ms2.WriteByte(bytArr[i]);
+            }
+
+            using var readonlyStream = new ChunkedMemoryStream(this.allocator);
+            ms2.WriteTo(readonlyStream);
+            readonlyStream.Flush();
+            readonlyStream.Position = 0;
+            bytArrRet = new byte[(int)readonlyStream.Length];
+            readonlyStream.Read(bytArrRet, 0, (int)readonlyStream.Length);
+            for (int i = 0; i < bytArr.Length; i++)
+            {
+                Assert.Equal(bytArr[i], bytArrRet[i]);
+            }
+        }
+    }
+
+    [Fact]
+    public void MemoryStream_WriteToTests_Negative()
+    {
+        using var ms2 = new ChunkedMemoryStream(this.allocator);
+        Assert.Throws<ArgumentNullException>(() => ms2.WriteTo(null));
+
+        ms2.Write(new byte[] { 1 }, 0, 1);
+        var readonlyStream = new MemoryStream(new byte[1028], false);
+        Assert.Throws<NotSupportedException>(() => ms2.WriteTo(readonlyStream));
+
+        readonlyStream.Dispose();
+
+        // [] Pass in a closed stream
+        Assert.Throws<ObjectDisposedException>(() => ms2.WriteTo(readonlyStream));
+    }
+
+    [Fact]
+    public void MemoryStream_CopyTo_Invalid()
+    {
+        ChunkedMemoryStream memoryStream;
+        const string bufferSize = nameof(bufferSize);
+        using (memoryStream = new ChunkedMemoryStream(this.allocator))
+        {
+            const string destination = nameof(destination);
+            Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null));
+
+            // Validate the destination parameter first.
+            Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null, bufferSize: 0));
+            Assert.Throws<ArgumentNullException>(destination, () => memoryStream.CopyTo(destination: null, bufferSize: -1));
+
+            // Then bufferSize.
+            Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // 0-length buffer doesn't make sense.
+            Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1));
+        }
+
+        // After the Stream is disposed, we should fail on all CopyTos.
+        Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: 0)); // Not before bufferSize is validated.
+        Assert.Throws<ArgumentOutOfRangeException>(bufferSize, () => memoryStream.CopyTo(Stream.Null, bufferSize: -1));
+
+        ChunkedMemoryStream disposedStream = memoryStream;
+
+        // We should throw first for the source being disposed...
+        Assert.Throws<ObjectDisposedException>(() => memoryStream.CopyTo(disposedStream, 1));
+
+        // Then for the destination being disposed.
+        memoryStream = new ChunkedMemoryStream(this.allocator);
+        Assert.Throws<ObjectDisposedException>(() => memoryStream.CopyTo(disposedStream, 1));
+        memoryStream.Dispose();
+    }
+
+    [Theory]
+    [MemberData(nameof(CopyToData))]
+    public void CopyTo(Stream source, byte[] expected)
+    {
+        using var destination = new ChunkedMemoryStream(this.allocator);
+        source.CopyTo(destination);
+        Assert.InRange(source.Position, source.Length, int.MaxValue); // Copying the data should have read to the end of the stream or stayed past the end.
+        Assert.Equal(expected, destination.ToArray());
+    }
+
+    public static IEnumerable<string> GetAllTestImages()
+    {
+        IEnumerable<string> allImageFiles = Directory.EnumerateFiles(TestEnvironment.InputImagesDirectoryFullPath, "*.*", SearchOption.AllDirectories)
+            .Where(s => !s.EndsWith("txt", StringComparison.OrdinalIgnoreCase));
+
+        var result = new List<string>();
+        foreach (string path in allImageFiles)
+        {
+            result.Add(path.Substring(TestEnvironment.InputImagesDirectoryFullPath.Length));
+        }
+
+        return result;
+    }
+
+    public static IEnumerable<string> AllTestImages = GetAllTestImages();
+
+    [Theory]
+    [WithFileCollection(nameof(AllTestImages), PixelTypes.Rgba32)]
+    public void DecoderIntegrationTest<TPixel>(TestImageProvider<TPixel> provider)
+        where TPixel : unmanaged, IPixel<TPixel>
+    {
+        if (!TestEnvironment.Is64BitProcess)
+        {
+            return;
+        }
+
+        Image<TPixel> expected;
+        try
+        {
+            expected = provider.GetImage();
+        }
+        catch
+        {
+            // The image is invalid
+            return;
+        }
+
+        string fullPath = Path.Combine(
+            TestEnvironment.InputImagesDirectoryFullPath,
+            ((TestImageProvider<TPixel>.FileProvider)provider).FilePath);
+
+        using FileStream fs = File.OpenRead(fullPath);
+        using var nonSeekableStream = new NonSeekableStream(fs);
+
+        var actual = Image.Load<TPixel>(nonSeekableStream);
+
+        ImageComparer.Exact.VerifySimilarity(expected, actual);
+    }
+
+    public static IEnumerable<object[]> CopyToData()
+    {
+        // Stream is positioned @ beginning of data
+        byte[] data1 = new byte[] { 1, 2, 3 };
+        var stream1 = new MemoryStream(data1);
+
+        yield return new object[] { stream1, data1 };
+
+        // Stream is positioned in the middle of data
+        byte[] data2 = new byte[] { 0xff, 0xf3, 0xf0 };
+        var stream2 = new MemoryStream(data2) { Position = 1 };
+
+        yield return new object[] { stream2, new byte[] { 0xf3, 0xf0 } };
+
+        // Stream is positioned after end of data
+        byte[] data3 = data2;
+        var stream3 = new MemoryStream(data3) { Position = data3.Length + 1 };
+
+        yield return new object[] { stream3, Array.Empty<byte>() };
+    }
+
+    private MemoryStream CreateTestStream(int length)
+    {
+        byte[] buffer = new byte[length];
+        var random = new Random();
+        random.NextBytes(buffer);
+
+        return new MemoryStream(buffer);
+    }
+}

tests/ImageSharp.Tests/Image/NonSeekableStream.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Six Labors.
+// Copyright (c) Six Labors.
 // Licensed under the Six Labors Split License.
 
 namespace SixLabors.ImageSharp.Tests;
@@ -14,7 +14,7 @@ internal class NonSeekableStream : Stream
 
     public override bool CanSeek => false;
 
-    public override bool CanWrite => this.dataStream.CanWrite;
+    public override bool CanWrite => false;
 
     public override bool CanTimeout => this.dataStream.CanTimeout;
 
@@ -91,5 +91,5 @@ internal class NonSeekableStream : Stream
         => throw new NotSupportedException();
 
     public override void Write(byte[] buffer, int offset, int count)
-        => this.dataStream.Write(buffer, offset, count);
+        => throw new NotImplementedException();
 }