From 9c5a393650bc40928caa89765501fdaf7bea1484 Mon Sep 17 00:00:00 2001 From: Dan Kroymann Date: Mon, 22 Apr 2024 17:29:20 -0700 Subject: [PATCH 1/4] Fix another async-over-sync issue where DecodeAsync() would internally block on a synchronous stream read when detecting the image format --- src/ImageSharp/Image.Decode.cs | 89 +++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/src/ImageSharp/Image.Decode.cs b/src/ImageSharp/Image.Decode.cs index 887cb23ca4..8eec3c55f0 100644 --- a/src/ImageSharp/Image.Decode.cs +++ b/src/ImageSharp/Image.Decode.cs @@ -1,6 +1,7 @@ // Copyright (c) Six Labors. // Licensed under the Six Labors Split License. +using System.Buffers; using SixLabors.ImageSharp.Formats; using SixLabors.ImageSharp.Memory; using SixLabors.ImageSharp.Metadata; @@ -67,20 +68,70 @@ public abstract partial class Image int i; do { - i = stream.Read(headersBuffer, n, headerSize - n); + i = stream.Read(headersBuffer[n..headerSize]); n += i; } while (n < headerSize && i > 0); stream.Position = startPosition; + return InternalDetectFormat(configuration, headersBuffer[..n]); + } + + /// + /// By reading the header on the provided stream this calculates the images format. + /// + /// The general configuration. + /// The image stream to read the header from. + /// The token to monitor for cancellation requests. + /// The mime type or null if none found. + /// The input format is not recognized. + private static async ValueTask InternalDetectFormatAsync( + Configuration configuration, + Stream stream, + CancellationToken cancellationToken) + { + // We take a minimum of the stream length vs the max header size and always check below + // to ensure that only formats that headers fit within the given buffer length are tested. + int headerSize = (int)Math.Min(configuration.MaxHeaderSize, stream.Length); + if (headerSize <= 0) + { + ImageFormatManager.ThrowInvalidDecoder(configuration.ImageFormatsManager); + } + + using (IMemoryOwner memoryOwner = MemoryPool.Shared.Rent(headerSize)) + { + Memory headersBuffer = memoryOwner.Memory; + long startPosition = stream.Position; + + // Read doesn't always guarantee the full returned length so read a byte + // at a time until we get either our count or hit the end of the stream. + int n = 0; + int i; + do + { + i = await stream.ReadAsync(headersBuffer[n..headerSize], cancellationToken); + n += i; + } + while (n < headerSize && i > 0); + + stream.Position = startPosition; + + return InternalDetectFormat(configuration, headersBuffer.Span[..n]); + } + } + + private static IImageFormat InternalDetectFormat( + Configuration configuration, + ReadOnlySpan headersBuffer) + { // Does the given stream contain enough data to fit in the header for the format // and does that data match the format specification? // Individual formats should still check since they are public. IImageFormat? format = null; foreach (IImageFormatDetector formatDetector in configuration.ImageFormatsManager.FormatDetectors) { - if (formatDetector.HeaderSize <= headerSize && formatDetector.TryDetectFormat(headersBuffer, out IImageFormat? attemptFormat)) + if (formatDetector.HeaderSize <= headersBuffer.Length && formatDetector.TryDetectFormat(headersBuffer, out IImageFormat? attemptFormat)) { format = attemptFormat; } @@ -106,6 +157,22 @@ public abstract partial class Image return options.Configuration.ImageFormatsManager.GetDecoder(format); } + /// + /// By reading the header on the provided stream this calculates the images format. + /// + /// The general decoder options. + /// The image stream to read the header from. + /// The token to monitor for cancellation requests. + /// The . + private static async ValueTask DiscoverDecoderAsync( + DecoderOptions options, + Stream stream, + CancellationToken cancellationToken) + { + IImageFormat format = await InternalDetectFormatAsync(options.Configuration, stream, cancellationToken); + return options.Configuration.ImageFormatsManager.GetDecoder(format); + } + /// /// Decodes the image stream to the current image. /// @@ -122,14 +189,14 @@ public abstract partial class Image return decoder.Decode(options, stream); } - private static Task> DecodeAsync( + private static async Task> DecodeAsync( DecoderOptions options, Stream stream, CancellationToken cancellationToken) where TPixel : unmanaged, IPixel { - IImageDecoder decoder = DiscoverDecoder(options, stream); - return decoder.DecodeAsync(options, stream, cancellationToken); + IImageDecoder decoder = await DiscoverDecoderAsync(options, stream, cancellationToken); + return await decoder.DecodeAsync(options, stream, cancellationToken); } private static Image Decode(DecoderOptions options, Stream stream) @@ -138,13 +205,13 @@ public abstract partial class Image return decoder.Decode(options, stream); } - private static Task DecodeAsync( + private static async Task DecodeAsync( DecoderOptions options, Stream stream, CancellationToken cancellationToken) { - IImageDecoder decoder = DiscoverDecoder(options, stream); - return decoder.DecodeAsync(options, stream, cancellationToken); + IImageDecoder decoder = await DiscoverDecoderAsync(options, stream, cancellationToken); + return await decoder.DecodeAsync(options, stream, cancellationToken); } /// @@ -166,12 +233,12 @@ public abstract partial class Image /// The stream. /// The token to monitor for cancellation requests. /// The . - private static Task InternalIdentifyAsync( + private static async Task InternalIdentifyAsync( DecoderOptions options, Stream stream, CancellationToken cancellationToken) { - IImageDecoder decoder = DiscoverDecoder(options, stream); - return decoder.IdentifyAsync(options, stream, cancellationToken); + IImageDecoder decoder = await DiscoverDecoderAsync(options, stream, cancellationToken); + return await decoder.IdentifyAsync(options, stream, cancellationToken); } } From 22e1c064c2825d7d5bb81c5040f0effb2dc6f0ff Mon Sep 17 00:00:00 2001 From: Dan Kroymann Date: Wed, 24 Apr 2024 18:20:34 -0700 Subject: [PATCH 2/4] Use MemoryAllocator from configuration --- src/ImageSharp/Image.Decode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImageSharp/Image.Decode.cs b/src/ImageSharp/Image.Decode.cs index 8eec3c55f0..7f58c6ecd8 100644 --- a/src/ImageSharp/Image.Decode.cs +++ b/src/ImageSharp/Image.Decode.cs @@ -99,7 +99,7 @@ public abstract partial class Image ImageFormatManager.ThrowInvalidDecoder(configuration.ImageFormatsManager); } - using (IMemoryOwner memoryOwner = MemoryPool.Shared.Rent(headerSize)) + using (IMemoryOwner memoryOwner = configuration.MemoryAllocator.Allocate(headerSize)) { Memory headersBuffer = memoryOwner.Memory; long startPosition = stream.Position; From e0d1493c3b12788cb960c3c375d0e6ee1b0d39c0 Mon Sep 17 00:00:00 2001 From: Dan Kroymann Date: Wed, 24 Apr 2024 18:45:11 -0700 Subject: [PATCH 3/4] Call the new InternalDetectFormatAsync() from DetectFormatAsync() --- src/ImageSharp/Image.FromStream.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ImageSharp/Image.FromStream.cs b/src/ImageSharp/Image.FromStream.cs index 63f9e64f6c..21345fdeb0 100644 --- a/src/ImageSharp/Image.FromStream.cs +++ b/src/ImageSharp/Image.FromStream.cs @@ -72,7 +72,7 @@ public abstract partial class Image => WithSeekableStreamAsync( options, stream, - (s, _) => Task.FromResult(InternalDetectFormat(options.Configuration, s)), + async (s, ct) => await InternalDetectFormatAsync(options.Configuration, s, ct).ConfigureAwait(false), cancellationToken); /// From 467850f7570fd79f3c2e144fceb57c5542cd2307 Mon Sep 17 00:00:00 2001 From: Anton Firszov Date: Wed, 8 May 2024 18:01:38 +0200 Subject: [PATCH 4/4] Fix overflow in MemoryAllocator.Create(options) (#2730) Fix an overlook from #2706. See https://github.com/SixLabors/ImageSharp/commit/92b82779ac8e7a032989533bb9a01ef092186b2e#r141770676. --- .../Memory/Allocators/MemoryAllocator.cs | 2 +- .../UniformUnmanagedPoolMemoryAllocatorTests.cs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs b/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs index b56bedd045..8eaf0b6d69 100644 --- a/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs +++ b/src/ImageSharp/Memory/Allocators/MemoryAllocator.cs @@ -49,7 +49,7 @@ public abstract class MemoryAllocator UniformUnmanagedMemoryPoolMemoryAllocator allocator = new(options.MaximumPoolSizeMegabytes); if (options.AllocationLimitMegabytes.HasValue) { - allocator.MemoryGroupAllocationLimitBytes = options.AllocationLimitMegabytes.Value * 1024 * 1024; + allocator.MemoryGroupAllocationLimitBytes = options.AllocationLimitMegabytes.Value * 1024L * 1024L; allocator.SingleBufferAllocationLimitBytes = (int)Math.Min(allocator.SingleBufferAllocationLimitBytes, allocator.MemoryGroupAllocationLimitBytes); } diff --git a/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs index 077d0afed8..aa34a5c108 100644 --- a/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs +++ b/tests/ImageSharp.Tests/Memory/Allocators/UniformUnmanagedPoolMemoryAllocatorTests.cs @@ -442,4 +442,20 @@ public class UniformUnmanagedPoolMemoryAllocatorTests allocator.AllocateGroup(4 * oneMb, 1024).Dispose(); // Should work Assert.Throws(() => allocator.AllocateGroup(5 * oneMb, 1024)); } + + [ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))] + public void MemoryAllocator_Create_SetHighLimit() + { + RemoteExecutor.Invoke(RunTest).Dispose(); + static void RunTest() + { + const long threeGB = 3L * (1 << 30); + MemoryAllocator allocator = MemoryAllocator.Create(new MemoryAllocatorOptions() + { + AllocationLimitMegabytes = (int)(threeGB / 1024) + }); + using MemoryGroup memoryGroup = allocator.AllocateGroup(threeGB, 1024); + Assert.Equal(threeGB, memoryGroup.TotalLength); + } + } }