Merge c40f4538d9 into 074a2c0a92

docs/en/framework/infrastructure/blob-storing/aws.md

@@ -7,7 +7,7 @@
 
 # BLOB Storing Aws Provider
 
-BLOB Storing Aws Provider can store BLOBs in [Amazon Simple Storage Service](https://aws.amazon.com/s3/).
+BLOB Storing Aws Provider can store BLOBs in [Amazon Simple Storage Service](https://aws.amazon.com/s3/) and **S3-compatible storage services** like MinIO, DigitalOcean Spaces, Cloudflare R2, and others.
 
 > Read the [BLOB Storing document](../blob-storing) to understand how to use the BLOB storing system. This document only covers how to configure containers to use a Aws BLOB as the storage provider.
 
@@ -42,6 +42,7 @@ Configure<AbpBlobStoringOptions>(options =>
             Aws.ProfileName = "the name of the profile to get credentials from";
             Aws.ProfilesLocation = "the path to the aws credentials file to look at";
             Aws.Region = "the system name of the service";
+            Aws.ServiceURL = "custom service URL for S3-compatible APIs (optional)";
             Aws.Name = "the name of the federated user";
             Aws.Policy = "policy";
             Aws.DurationSeconds = "expiration date";
@@ -65,6 +66,7 @@ Configure<AbpBlobStoringOptions>(options =>
 * **ProfileName** (string): The [name of the profile](https://docs.aws.amazon.com/sdk-for-net/v3/developer-guide/net-dg-config-creds.html) to get credentials from.
 * **ProfilesLocation** (string): The path to the aws credentials file to look at.
 * **Region** (string): The system name of the service.
+* **ServiceURL** (string): Custom service URL for S3-compatible APIs (e.g., MinIO, DigitalOcean Spaces). If not specified, the default AWS S3 service URL will be used based on the region. When using S3-compatible services, this should point to your service endpoint (e.g., `https://minio.example.com:9000`).
 * **Policy** (string): An IAM policy in JSON format that you want to use as an inline session policy.
 * **DurationSeconds** (int): Validity period(s) of a temporary access certificate,minimum is 900 and the maximum is 3600. **note**: Using sub-accounts operated OSS,if the value is 0.
 * **ContainerName** (string): You can specify the container name in Aws. If this is not specified, it uses the name of the BLOB container defined with the `BlobContainerName` attribute (see the [BLOB storing document](../blob-storing)). Please note that Aws has some **rules for naming containers**. A container name must be a valid DNS name, conforming to the [following naming rules](https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html):
@@ -77,6 +79,72 @@ Configure<AbpBlobStoringOptions>(options =>
     * Buckets used with Amazon S3 Transfer Acceleration can't have dots (.) in their names. For more information about transfer acceleration, see Amazon S3 Transfer Acceleration.
 * **CreateContainerIfNotExists** (bool): Default value is `false`, If a container does not exist in Aws, `AwsBlobProvider` will try to create it.
 
+## S3-Compatible Services
+
+The AWS provider supports S3-compatible storage services by configuring the `ServiceURL` property. Here are some examples:
+
+### MinIO Configuration
+
+````csharp
+Configure<AbpBlobStoringOptions>(options =>
+{
+    options.Containers.ConfigureDefault(container =>
+    {
+        container.UseAws(aws =>
+        {
+            aws.AccessKeyId = "your-minio-access-key";
+            aws.SecretAccessKey = "your-minio-secret-key";
+            aws.ServiceURL = "https://minio.example.com:9000";
+            aws.Region = "us-east-1"; // MinIO region (can be any valid region)
+            aws.ContainerName = "my-bucket";
+            aws.CreateContainerIfNotExists = true;
+        });
+    });
+});
+````
+
+### DigitalOcean Spaces Configuration
+
+````csharp
+Configure<AbpBlobStoringOptions>(options =>
+{
+    options.Containers.ConfigureDefault(container =>
+    {
+        container.UseAws(aws =>
+        {
+            aws.AccessKeyId = "your-spaces-access-key";
+            aws.SecretAccessKey = "your-spaces-secret-key";
+            aws.ServiceURL = "https://nyc3.digitaloceanspaces.com";
+            aws.Region = "us-east-1"; // DigitalOcean Spaces region
+            aws.ContainerName = "my-space";
+            aws.CreateContainerIfNotExists = true;
+        });
+    });
+});
+````
+
+### Cloudflare R2 Configuration
+
+````csharp
+Configure<AbpBlobStoringOptions>(options =>
+{
+    options.Containers.ConfigureDefault(container =>
+    {
+        container.UseAws(aws =>
+        {
+            aws.AccessKeyId = "your-r2-access-key";
+            aws.SecretAccessKey = "your-r2-secret-key";
+            aws.ServiceURL = "https://your-account-id.r2.cloudflarestorage.com";
+            aws.Region = "auto"; // Cloudflare R2 uses 'auto' as region
+            aws.ContainerName = "my-bucket";
+            aws.CreateContainerIfNotExists = true;
+        });
+    });
+});
+````
+
+> **Note**: When using S3-compatible services, the provider automatically enables path-style requests which are required by most S3-compatible implementations.
+
 ## Aws Blob Name Calculator
 
 Aws Blob Provider organizes BLOB name and implements some conventions. The full name of a BLOB is determined by the following rules by default:

docs/en/framework/infrastructure/blob-storing/index.md

@@ -36,6 +36,20 @@ More providers will be implemented by the time. You can [request](https://github
 
 Multiple providers **can be used together** by the help of the **container system**, where each container can uses a different provider.
 
+### S3 Compatibility
+
+The [AWS provider](./aws.md) supports not only Amazon S3 but also **S3-compatible APIs** from various cloud providers and self-hosted solutions. This means you can use the same AWS provider to connect to:
+
+* **Amazon S3** - The original AWS S3 service
+* **MinIO** - Self-hosted S3-compatible object storage
+* **Cloudflare R2** - Cloudflare's S3-compatible object storage
+* **DigitalOcean Spaces** - DigitalOcean's S3-compatible object storage
+* **Wasabi** - S3-compatible cloud storage
+* **Backblaze B2** - S3-compatible cloud storage
+* **Any other S3-compatible storage** - Including private cloud solutions
+
+To use S3-compatible services, simply configure the `ServiceURL` property in the AWS provider configuration to point to your S3-compatible endpoint. The provider will automatically handle the necessary protocol adjustments for compatibility.
+
 > BLOB storing system can not work unless you **configure a storage provider**. Refer to the linked documents for the storage provider configurations.
 
 ## Installation

framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs

@@ -57,9 +57,18 @@ public class AwsBlobProviderConfiguration
         set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Policy, value);
     }
 
-    public string Region {
-        get => _containerConfiguration.GetConfiguration<string>(AwsBlobProviderConfigurationNames.Region);
-        set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, Check.NotNull(value, nameof(value)));
+    public string? Region {
+        get => _containerConfiguration.GetConfigurationOrDefault<string>(AwsBlobProviderConfigurationNames.Region);
+        set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, value);
+    }
+
+    /// <summary>
+    /// Custom service URL for S3-compatible APIs (e.g., MinIO, DigitalOcean Spaces).
+    /// If not specified, the default AWS S3 service URL will be used based on the region.
+    /// </summary>
+    public string? ServiceURL {
+        get => _containerConfiguration.GetConfigurationOrDefault<string>(AwsBlobProviderConfigurationNames.ServiceURL);
+        set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ServiceURL, value);
     }
 
     /// <summary>

framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs

@@ -14,6 +14,7 @@ public static class AwsBlobProviderConfigurationNames
     public const string Name = "Aws.Name";
     public const string Policy = "Aws.Policy";
     public const string Region = "Aws.Region";
+    public const string ServiceURL = "Aws.ServiceURL";
     public const string ContainerName = "Aws.ContainerName";
     public const string CreateContainerIfNotExists = "Aws.CreateContainerIfNotExists";
 }

framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs

@@ -30,31 +30,58 @@ public class DefaultAmazonS3ClientFactory : IAmazonS3ClientFactory, ITransientDe
     public virtual async Task<AmazonS3Client> GetAmazonS3Client(
         AwsBlobProviderConfiguration configuration)
     {
-        var region = RegionEndpoint.GetBySystemName(configuration.Region);
+        var region = !configuration.Region.IsNullOrWhiteSpace() 
+            ? RegionEndpoint.GetBySystemName(configuration.Region)
+            : null;
+        var clientConfig = CreateS3ClientConfig(configuration, region);
 
         if (configuration.UseCredentials)
         {
             var awsCredentials = GetAwsCredentials(configuration);
             return awsCredentials == null
-                ? new AmazonS3Client(region)
-                : new AmazonS3Client(awsCredentials, region);
+                ? new AmazonS3Client(clientConfig)
+                : new AmazonS3Client(awsCredentials, clientConfig);
         }
 
         if (configuration.UseTemporaryCredentials)
         {
-            return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), region);
+            return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), clientConfig);
         }
 
         if (configuration.UseTemporaryFederatedCredentials)
         {
             return new AmazonS3Client(await GetTemporaryFederatedCredentialsAsync(configuration),
-                region);
+                clientConfig);
         }
 
         Check.NotNullOrWhiteSpace(configuration.AccessKeyId, nameof(configuration.AccessKeyId));
         Check.NotNullOrWhiteSpace(configuration.SecretAccessKey, nameof(configuration.SecretAccessKey));
 
-        return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey, region);
+        return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey, clientConfig);
+    }
+
+    protected virtual AmazonS3Config CreateS3ClientConfig(AwsBlobProviderConfiguration configuration, RegionEndpoint? region)
+    {
+        var clientConfig = new AmazonS3Config();
+
+        // Set region only if it's provided (for AWS S3)
+        if (region != null)
+        {
+            clientConfig.RegionEndpoint = region;
+        }
+
+        if (!configuration.ServiceURL.IsNullOrWhiteSpace())
+        {
+            clientConfig.ServiceURL = configuration.ServiceURL;
+            clientConfig.ForcePathStyle = true; // Required for most S3-compatible services
+            
+            // Set checksum properties for S3-compatible services (e.g., Cloudflare R2)
+            // These settings help with compatibility issues in non-AWS S3 services
+            clientConfig.RequestChecksumCalculation = RequestChecksumCalculation.WHEN_REQUIRED;
+            clientConfig.ResponseChecksumValidation = ResponseChecksumValidation.WHEN_REQUIRED;
+        }
+
+        return clientConfig;
     }
 
     protected virtual AWSCredentials? GetAwsCredentials(

framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs

@@ -69,23 +69,39 @@ public class AbpBlobStoringAwsTestModule : AbpModule
 
     private async Task DeleteBucketAsync(ApplicationShutdownContext context)
     {
-        var amazonS3Client = await context.ServiceProvider.GetRequiredService<IAmazonS3ClientFactory>()
-            .GetAmazonS3Client(_configuration);
+        // Skip bucket deletion if configuration is not properly set (e.g., in unit tests)
+        if (_configuration == null || 
+            string.IsNullOrWhiteSpace(_configuration.AccessKeyId) || 
+            string.IsNullOrWhiteSpace(_configuration.SecretAccessKey) ||
+            (string.IsNullOrWhiteSpace(_configuration.Region) && string.IsNullOrWhiteSpace(_configuration.ServiceURL)))
+        {
+            return;
+        }
 
-        if (await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, _randomContainerName))
+        try
         {
-            var blobs = await amazonS3Client.ListObjectsAsync(_randomContainerName);
+            var amazonS3Client = await context.ServiceProvider.GetRequiredService<IAmazonS3ClientFactory>()
+                .GetAmazonS3Client(_configuration);
 
-            if (blobs.S3Objects.Any())
+            if (await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, _randomContainerName))
             {
-                await amazonS3Client.DeleteObjectsAsync(new DeleteObjectsRequest
+                var blobs = await amazonS3Client.ListObjectsAsync(_randomContainerName);
+
+                if (blobs.S3Objects.Any())
                 {
-                    BucketName = _randomContainerName,
-                    Objects = blobs.S3Objects.Select(o => new KeyVersion { Key = o.Key }).ToList()
-                });
-            }
+                    await amazonS3Client.DeleteObjectsAsync(new DeleteObjectsRequest
+                    {
+                        BucketName = _randomContainerName,
+                        Objects = blobs.S3Objects.Select(o => new KeyVersion { Key = o.Key }).ToList()
+                    });
+                }
 
-            await amazonS3Client.DeleteBucketAsync(_randomContainerName);
+                await amazonS3Client.DeleteBucketAsync(_randomContainerName);
+            }
+        }
+        catch
+        {
+            // Ignore errors during test cleanup
         }
     }
 }

framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration_Tests.cs

@@ -0,0 +1,52 @@
+using Shouldly;
+using Xunit;
+
+namespace Volo.Abp.BlobStoring.Aws;
+
+public class AwsBlobProviderConfiguration_Tests : AbpBlobStoringAwsTestCommonBase
+{
+    [Fact]
+    public void Should_Set_And_Get_ServiceURL()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration);
+        const string serviceUrl = "https://minio.example.com:9000";
+
+        // Act
+        awsConfiguration.ServiceURL = serviceUrl;
+
+        // Assert
+        awsConfiguration.ServiceURL.ShouldBe(serviceUrl);
+    }
+
+    [Fact]
+    public void Should_Return_Null_When_ServiceURL_Not_Set()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration);
+
+        // Act & Assert
+        awsConfiguration.ServiceURL.ShouldBeNull();
+    }
+
+    [Fact]
+    public void Should_Configure_ServiceURL_Using_UseAws_Extension()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        const string serviceUrl = "https://spaces.digitalocean.com";
+
+        // Act
+        containerConfiguration.UseAws(config =>
+        {
+            config.ServiceURL = serviceUrl;
+            config.Region = "us-east-1";
+        });
+
+        // Assert
+        var awsConfig = containerConfiguration.GetAwsConfiguration();
+        awsConfig.ServiceURL.ShouldBe(serviceUrl);
+    }
+} 

framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory_Tests.cs

@@ -0,0 +1,116 @@
+using System.Threading.Tasks;
+using Amazon.S3;
+using Microsoft.Extensions.DependencyInjection;
+using Shouldly;
+using Xunit;
+
+namespace Volo.Abp.BlobStoring.Aws;
+
+public class DefaultAmazonS3ClientFactory_Tests : AbpBlobStoringAwsTestBase
+{
+    private readonly IAmazonS3ClientFactory _amazonS3ClientFactory;
+
+    public DefaultAmazonS3ClientFactory_Tests()
+    {
+        _amazonS3ClientFactory = GetRequiredService<IAmazonS3ClientFactory>();
+    }
+
+    [Fact]
+    public async Task Should_Create_S3Client_With_Custom_ServiceURL()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        const string serviceUrl = "https://minio.example.com:9000";
+        
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration)
+        {
+            AccessKeyId = "test-access-key",
+            SecretAccessKey = "test-secret-key",
+            Region = "us-east-1",
+            ServiceURL = serviceUrl
+        };
+
+        // Act
+        using var s3Client = await _amazonS3ClientFactory.GetAmazonS3Client(awsConfiguration);
+
+        // Assert
+        s3Client.ShouldNotBeNull();
+        s3Client.Config.ServiceURL.ShouldBe(serviceUrl + "/"); // AWS SDK automatically appends trailing slash
+        ((AmazonS3Config)s3Client.Config).ForcePathStyle.ShouldBeTrue(); // Should be enabled for S3-compatible services
+    }
+
+    [Fact]
+    public async Task Should_Create_S3Client_Without_Custom_ServiceURL()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration)
+        {
+            AccessKeyId = "test-access-key",
+            SecretAccessKey = "test-secret-key",
+            Region = "us-east-1"
+            // ServiceURL not set
+        };
+
+        // Act
+        using var s3Client = await _amazonS3ClientFactory.GetAmazonS3Client(awsConfiguration);
+
+        // Assert
+        s3Client.ShouldNotBeNull();
+        s3Client.Config.ServiceURL.ShouldBeNull(); // Should use default AWS S3 service
+        ((AmazonS3Config)s3Client.Config).ForcePathStyle.ShouldBeFalse(); // Should be false for AWS S3
+    }
+
+    [Fact]
+    public async Task Should_Create_S3Client_Without_Region_For_S3Compatible_Services()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration)
+        {
+            AccessKeyId = "test-access-key",
+            SecretAccessKey = "test-secret-key",
+            ServiceURL = "https://minio.example.com:9000"
+            // Region not set - should work for S3-compatible services
+        };
+
+        // Act
+        using var s3Client = await _amazonS3ClientFactory.GetAmazonS3Client(awsConfiguration);
+
+        // Assert
+        s3Client.ShouldNotBeNull();
+        s3Client.Config.ServiceURL.ShouldBe("https://minio.example.com:9000/"); // AWS SDK automatically appends trailing slash
+        ((AmazonS3Config)s3Client.Config).ForcePathStyle.ShouldBeTrue(); // Should be enabled for S3-compatible services
+    }
+
+    [Fact]
+    public async Task Should_Set_Checksum_Properties_For_S3Compatible_Services()
+    {
+        // Arrange
+        var containerConfiguration = new BlobContainerConfiguration();
+        
+        var awsConfiguration = new AwsBlobProviderConfiguration(containerConfiguration)
+        {
+            AccessKeyId = "test-access-key",
+            SecretAccessKey = "test-secret-key",
+            ServiceURL = "https://r2.cloudflarestorage.com",
+            Region = "auto"
+        };
+
+        // Act
+        using var s3Client = await _amazonS3ClientFactory.GetAmazonS3Client(awsConfiguration);
+
+        // Assert
+        s3Client.ShouldNotBeNull();
+        var config = (AmazonS3Config)s3Client.Config;
+        config.ServiceURL.ShouldBe("https://r2.cloudflarestorage.com/");
+        config.ForcePathStyle.ShouldBeTrue();
+        
+        // Verify checksum properties are set for S3-compatible services (required for Cloudflare R2)
+        // We just verify they are not null/default, indicating they have been set
+        config.RequestChecksumCalculation.ShouldNotBe(default);
+        config.ResponseChecksumValidation.ShouldNotBe(default);
+    }
+}