diff --git a/framework/Volo.Abp.sln b/framework/Volo.Abp.sln index 5a83ae1460..5778b054ed 100644 --- a/framework/Volo.Abp.sln +++ b/framework/Volo.Abp.sln @@ -319,6 +319,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun.Tests", "test\Volo.Abp.BlobStoring.Aliyun.Tests\Volo.Abp.BlobStoring.Aliyun.Tests.csproj", "{8E49687A-E69F-49F2-8DB0-428D0883A937}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws", "src\Volo.Abp.BlobStoring.Aws\Volo.Abp.BlobStoring.Aws.csproj", "{50968CDE-1029-4051-B2E5-B69D0ECF2A18}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws.Tests", "test\Volo.Abp.BlobStoring.Aws.Tests\Volo.Abp.BlobStoring.Aws.Tests.csproj", "{2CD3B26A-CA81-4279-8D5D-6A594517BB3F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -949,6 +953,14 @@ Global {8E49687A-E69F-49F2-8DB0-428D0883A937}.Debug|Any CPU.Build.0 = Debug|Any CPU {8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.ActiveCfg = Release|Any CPU {8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.Build.0 = Release|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.Build.0 = Release|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1110,6 +1122,8 @@ Global {60D0E384-965E-4F81-9D71-B28F419254FC} = {447C8A77-E5F0-4538-8687-7383196D04EA} {845E6A13-D1B5-4DDC-A16C-68D807E3B4C7} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} {8E49687A-E69F-49F2-8DB0-428D0883A937} = {447C8A77-E5F0-4538-8687-7383196D04EA} + {50968CDE-1029-4051-B2E5-B69D0ECF2A18} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F} = {447C8A77-E5F0-4538-8687-7383196D04EA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BB97ECF4-9A84-433F-A80B-2A3285BDD1D5} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml new file mode 100644 index 0000000000..be0de3a908 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd new file mode 100644 index 0000000000..3f3946e282 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. + + + + + A comma-separated list of error codes that can be safely ignored in assembly verification. + + + + + 'false' to turn off automatic generation of the XML Schema file. + + + + + \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj new file mode 100644 index 0000000000..d30601d1e8 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj @@ -0,0 +1,24 @@ + + + + + + + netstandard2.0 + false + false + false + + + + + + + + + + + + + + diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs new file mode 100644 index 0000000000..28426321c8 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs @@ -0,0 +1,14 @@ +using Volo.Abp.Caching; +using Volo.Abp.Modularity; + +namespace Volo.Abp.BlobStoring.Aws +{ + [DependsOn(typeof(AbpBlobStoringModule), + typeof(AbpCachingModule))] + public class AbpBlobStoringAwsModule : AbpModule + { + public override void ConfigureServices(ServiceConfigurationContext context) + { + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs new file mode 100644 index 0000000000..d63e408aab --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs @@ -0,0 +1,25 @@ +using System; + +namespace Volo.Abp.BlobStoring.Aws +{ + public static class AwsBlobContainerConfigurationExtensions + { + public static AwsBlobProviderConfiguration GetAwsConfiguration( + this BlobContainerConfiguration containerConfiguration) + { + return new AwsBlobProviderConfiguration(containerConfiguration); + } + + public static BlobContainerConfiguration UseAws( + this BlobContainerConfiguration containerConfiguration, + Action awsConfigureAction) + { + containerConfiguration.ProviderType = typeof(AwsBlobProvider); + containerConfiguration.NamingNormalizers.TryAdd(); + + awsConfigureAction(new AwsBlobProviderConfiguration(containerConfiguration)); + + return containerConfiguration; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs new file mode 100644 index 0000000000..cd8eca021e --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs @@ -0,0 +1,52 @@ +using System.Text.RegularExpressions; +using Volo.Abp.DependencyInjection; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobNamingNormalizer : IBlobNamingNormalizer, ITransientDependency + { + /// + ///https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html + /// + public virtual string NormalizeContainerName(string containerName) + { + // All letters in a container name must be lowercase. + containerName = containerName.ToLower(); + + // Container names can contain only letters, numbers, and the dash (-) character. + containerName = Regex.Replace(containerName, "[^a-z0-9-]", string.Empty); + + // Every dash (-) character must be immediately preceded and followed by a letter or number; + // consecutive dashes are not permitted in container names. + // Container names must start or end with a letter or number + containerName = Regex.Replace(containerName, "-{2,}", "-"); + containerName = Regex.Replace(containerName, "^-", string.Empty); + containerName = Regex.Replace(containerName, "-$", string.Empty); + + // Container names must be from 3 through 63 characters long. + if (containerName.Length < 3) + { + var length = containerName.Length; + for (var i = 0; i < 3 - length; i++) + { + containerName += "0"; + } + } + + if (containerName.Length > 63) + { + containerName = containerName.Substring(0, 63); + } + + return containerName; + } + + /// + /// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + /// + public virtual string NormalizeBlobName(string blobName) + { + return blobName; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs new file mode 100644 index 0000000000..6acb91aea9 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs @@ -0,0 +1,158 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Util; +using Volo.Abp.DependencyInjection; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobProvider : BlobProviderBase, ITransientDependency + { + protected IAwsBlobNameCalculator AwsBlobNameCalculator { get; } + protected IAmazonS3ClientFactory AmazonS3ClientFactory { get; } + + public AwsBlobProvider(IAwsBlobNameCalculator awsBlobNameCalculator, + IAmazonS3ClientFactory amazonS3ClientFactory) + { + AwsBlobNameCalculator = awsBlobNameCalculator; + AmazonS3ClientFactory = amazonS3ClientFactory; + } + + public override async Task SaveAsync(BlobProviderSaveArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var configuration = args.Configuration.GetAwsConfiguration(); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!args.OverrideExisting && await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + throw new BlobAlreadyExistsException( + $"Saving BLOB '{args.BlobName}' does already exists in the container '{containerName}'! Set {nameof(args.OverrideExisting)} if it should be overwritten."); + } + + if (configuration.CreateContainerIfNotExists) + { + await CreateContainerIfNotExists(amazonS3Client, containerName); + } + + await amazonS3Client.PutObjectAsync(new PutObjectRequest + { + BucketName = containerName, + Key = blobName, + InputStream = args.BlobStream + }); + } + } + + public override async Task DeleteAsync(BlobProviderDeleteArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + return false; + } + + await amazonS3Client.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = containerName, + Key = blobName + }); + + return true; + } + } + + public override async Task ExistsAsync(BlobProviderExistsArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + return await BlobExistsAsync(amazonS3Client, containerName, blobName); + } + } + + public override async Task GetOrNullAsync(BlobProviderGetArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + return null; + } + + var response = await amazonS3Client.GetObjectAsync(new GetObjectRequest + { + BucketName = containerName, + Key = blobName + }); + + var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + return memoryStream; + } + } + + protected virtual async Task GetAmazonS3Client(BlobProviderArgs args) + { + var configuration = args.Configuration.GetAwsConfiguration(); + return await AmazonS3ClientFactory.GetAmazonS3Client(configuration); + } + + private async Task BlobExistsAsync(AmazonS3Client amazonS3Client, string containerName, string blobName) + { + // Make sure Blob Container exists. + if (!await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, containerName)) + { + return false; + } + + try + { + await amazonS3Client.GetObjectMetadataAsync(containerName, blobName); + } + catch (Exception ex) + { + if (ex is AmazonS3Exception) + { + return false; + } + + throw; + } + + return true; + } + + protected virtual async Task CreateContainerIfNotExists(AmazonS3Client amazonS3Client, string containerName) + { + if (!await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, containerName)) + { + await amazonS3Client.PutBucketAsync(new PutBucketRequest + { + BucketName = containerName + }); + } + } + + private static string GetContainerName(BlobProviderArgs args) + { + var configuration = args.Configuration.GetAwsConfiguration(); + return configuration.ContainerName.IsNullOrWhiteSpace() + ? args.ContainerName + : configuration.ContainerName; + } + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs new file mode 100644 index 0000000000..c2d3297f4e --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs @@ -0,0 +1,112 @@ +using System; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobProviderConfiguration + { + public string AccessKeyId + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.AccessKeyId); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId, value); + } + + public string SecretAccessKey + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.SecretAccessKey); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey, value); + } + + public bool UseCredentials + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseCredentials,false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseCredentials, value); + } + + public bool UseTemporaryCredentials + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseTemporaryCredentials,false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials, value); + } + + public bool UseTemporaryFederatedCredentials + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials,false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials, value); + } + + public string ProfileName + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.ProfileName); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfileName, value); + } + + public string ProfilesLocation + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.ProfilesLocation); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation, value); + } + + /// + /// Set the validity period of the temporary access credential, the unit is s, the minimum is 900, and the maximum is 129600. + /// + public int DurationSeconds + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.DurationSeconds, 0); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.DurationSeconds, value); + } + + public string Name + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Name); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Name, value); + } + + public string Policy + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.Policy); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Policy, value); + } + + public string Region + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Region); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, Check.NotNull(value, nameof(value))); + } + + /// + /// This name may only contain lowercase letters, numbers, and hyphens, and must begin with a letter or a number. + /// Each hyphen must be preceded and followed by a non-hyphen character. + /// The name must also be between 3 and 63 characters long. + /// If this parameter is not specified, the ContainerName of the will be used. + /// + public string ContainerName + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ContainerName); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ContainerName, Check.NotNullOrWhiteSpace(value, nameof(value))); + } + + /// + /// Default value: false. + /// + public bool CreateContainerIfNotExists + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, value); + } + + private readonly string _temporaryCredentialsCacheKey; + public string TemporaryCredentialsCacheKey + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.TemporaryCredentialsCacheKey, _temporaryCredentialsCacheKey); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.TemporaryCredentialsCacheKey, value); + } + + private readonly BlobContainerConfiguration _containerConfiguration; + + public AwsBlobProviderConfiguration(BlobContainerConfiguration containerConfiguration) + { + _containerConfiguration = containerConfiguration; + _temporaryCredentialsCacheKey = Guid.NewGuid().ToString("N"); + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs new file mode 100644 index 0000000000..24dfca53d2 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs @@ -0,0 +1,20 @@ +namespace Volo.Abp.BlobStoring.Aws +{ + public static class AwsBlobProviderConfigurationNames + { + public const string AccessKeyId = "Aws.AccessKeyId"; + public const string SecretAccessKey = "Aws.SecretAccessKey"; + public const string UseCredentials = "Aws.UseCredentials"; + public const string UseTemporaryCredentials = "Aws.UseTemporaryCredentials"; + public const string UseTemporaryFederatedCredentials = "Aws.UseTemporaryFederatedCredentials"; + public const string ProfileName = "Aws.ProfileName"; + public const string ProfilesLocation = "Aws.ProfilesLocation"; + public const string DurationSeconds = "Aws.DurationSeconds"; + public const string TemporaryCredentialsCacheKey = "Aws.TemporaryCredentialsCacheKey"; + public const string Name = "Aws.Name"; + public const string Policy = "Aws.Policy"; + public const string Region = "Aws.Region"; + public const string ContainerName = "Aws.ContainerName"; + public const string CreateContainerIfNotExists = "Aws.CreateContainerIfNotExists"; + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs new file mode 100644 index 0000000000..887acddc5d --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs @@ -0,0 +1,26 @@ +using System; + +namespace Volo.Abp.BlobStoring.Aws +{ + [Serializable] + public class AwsTemporaryCredentialsCacheItem + { + public string AccessKeyId { get; set; } + + public string SecretAccessKey { get; set; } + + public string SessionToken { get; set; } + + public AwsTemporaryCredentialsCacheItem() + { + + } + + public AwsTemporaryCredentialsCacheItem(string accessKeyId,string secretAccessKey,string sessionToken) + { + AccessKeyId = accessKeyId; + SecretAccessKey = secretAccessKey; + SessionToken = sessionToken; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs new file mode 100644 index 0000000000..78ef55aebb --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs @@ -0,0 +1,193 @@ +using System; +using System.Threading.Tasks; +using Amazon; +using Amazon.Runtime; +using Amazon.Runtime.CredentialManagement; +using Amazon.S3; +using Amazon.SecurityToken; +using Amazon.SecurityToken.Model; +using Microsoft.Extensions.Caching.Distributed; +using Volo.Abp.Caching; +using Volo.Abp.DependencyInjection; +using Volo.Abp.Security.Encryption; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAmazonS3ClientFactory : IAmazonS3ClientFactory, ITransientDependency + { + protected IDistributedCache Cache { get; } + + protected IStringEncryptionService StringEncryptionService { get; } + + public DefaultAmazonS3ClientFactory( + IDistributedCache cache, + IStringEncryptionService stringEncryptionService) + { + Cache = cache; + StringEncryptionService = stringEncryptionService; + } + + public virtual async Task GetAmazonS3Client( + AwsBlobProviderConfiguration configuration) + { + var region = RegionEndpoint.GetBySystemName(configuration.Region); + + if (configuration.UseCredentials) + { + var awsCredentials = GetAwsCredentials(configuration); + return awsCredentials == null + ? new AmazonS3Client(region) + : new AmazonS3Client(GetAwsCredentials(configuration), region); + } + + if (configuration.UseTemporaryCredentials) + { + return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), region); + } + + if (configuration.UseTemporaryFederatedCredentials) + { + return new AmazonS3Client(await GetTemporaryFederatedCredentialsAsync(configuration), + region); + } + + Check.NotNullOrWhiteSpace(configuration.AccessKeyId, nameof(configuration.AccessKeyId)); + Check.NotNullOrWhiteSpace(configuration.SecretAccessKey, nameof(configuration.SecretAccessKey)); + + return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey, configuration.Region); + } + + protected virtual AWSCredentials GetAwsCredentials( + AwsBlobProviderConfiguration configuration) + { + if (configuration.ProfileName.IsNullOrWhiteSpace()) + { + return null; + } + + var chain = new CredentialProfileStoreChain(configuration.ProfilesLocation); + + if (chain.TryGetAWSCredentials(configuration.ProfileName, out var awsCredentials)) + { + return awsCredentials; + } + + throw new AmazonS3Exception("Not found aws credentials"); + } + + protected virtual async Task GetTemporaryCredentialsAsync( + AwsBlobProviderConfiguration configuration) + { + var temporaryCredentialsCache = await Cache.GetAsync(configuration.TemporaryCredentialsCacheKey); + + if (temporaryCredentialsCache == null) + { + AmazonSecurityTokenServiceClient stsClient; + + if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty()) + { + stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId, + configuration.SecretAccessKey); + } + else + { + var awsCredentials = GetAwsCredentials(configuration); + stsClient = awsCredentials == null + ? new AmazonSecurityTokenServiceClient() + : new AmazonSecurityTokenServiceClient(awsCredentials); + } + + using (stsClient) + { + var getSessionTokenRequest = new GetSessionTokenRequest + { + DurationSeconds = configuration.DurationSeconds + }; + + var sessionTokenResponse = + await stsClient.GetSessionTokenAsync(getSessionTokenRequest); + + var credentials = sessionTokenResponse.Credentials; + + temporaryCredentialsCache = + await SetTemporaryCredentialsCache(configuration, credentials); + } + } + + var sessionCredentials = new SessionAWSCredentials( + StringEncryptionService.Decrypt(temporaryCredentialsCache.AccessKeyId), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SecretAccessKey), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SessionToken)); + return sessionCredentials; + } + + protected virtual async Task GetTemporaryFederatedCredentialsAsync( + AwsBlobProviderConfiguration configuration) + { + Check.NotNullOrWhiteSpace(configuration.Name, nameof(configuration.Name)); + Check.NotNullOrWhiteSpace(configuration.Policy, nameof(configuration.Policy)); + + var temporaryCredentialsCache = await Cache.GetAsync(configuration.TemporaryCredentialsCacheKey); + + if (temporaryCredentialsCache == null) + { + AmazonSecurityTokenServiceClient stsClient; + + if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty()) + { + stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId, + configuration.SecretAccessKey); + } + else + { + var awsCredentials = GetAwsCredentials(configuration); + stsClient = awsCredentials == null + ? new AmazonSecurityTokenServiceClient() + : new AmazonSecurityTokenServiceClient(awsCredentials); + } + + using (stsClient) + { + var federationTokenRequest = + new GetFederationTokenRequest + { + DurationSeconds = configuration.DurationSeconds, + Name = configuration.Name, + Policy = configuration.Policy + }; + + var federationTokenResponse = + await stsClient.GetFederationTokenAsync(federationTokenRequest); + var credentials = federationTokenResponse.Credentials; + + temporaryCredentialsCache = + await SetTemporaryCredentialsCache(configuration, credentials); + } + } + + var sessionCredentials = new SessionAWSCredentials( + StringEncryptionService.Decrypt(temporaryCredentialsCache.AccessKeyId), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SecretAccessKey), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SessionToken)); + return sessionCredentials; + } + + private async Task SetTemporaryCredentialsCache( + AwsBlobProviderConfiguration configuration, + Credentials credentials) + { + var temporaryCredentialsCache = new AwsTemporaryCredentialsCacheItem( + StringEncryptionService.Encrypt(credentials.AccessKeyId), + StringEncryptionService.Encrypt(credentials.SecretAccessKey), + StringEncryptionService.Encrypt(credentials.SessionToken)); + + await Cache.SetAsync(configuration.TemporaryCredentialsCacheKey, temporaryCredentialsCache, + new DistributedCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(configuration.DurationSeconds - 10) + }); + + return temporaryCredentialsCache; + } + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs new file mode 100644 index 0000000000..9ff3511e6f --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs @@ -0,0 +1,22 @@ +using Volo.Abp.DependencyInjection; +using Volo.Abp.MultiTenancy; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAwsBlobNameCalculator : IAwsBlobNameCalculator, ITransientDependency + { + protected ICurrentTenant CurrentTenant { get; } + + public DefaultAwsBlobNameCalculator(ICurrentTenant currentTenant) + { + CurrentTenant = currentTenant; + } + + public virtual string Calculate(BlobProviderArgs args) + { + return CurrentTenant.Id == null + ? $"host/{args.BlobName}" + : $"tenants/{CurrentTenant.Id.Value.ToString("D")}/{args.BlobName}"; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs new file mode 100644 index 0000000000..92d7ddc906 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Amazon.S3; + +namespace Volo.Abp.BlobStoring.Aws +{ + public interface IAmazonS3ClientFactory + { + Task GetAmazonS3Client(AwsBlobProviderConfiguration configuration); + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs new file mode 100644 index 0000000000..e4c470c505 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs @@ -0,0 +1,7 @@ +namespace Volo.Abp.BlobStoring.Aws +{ + public interface IAwsBlobNameCalculator + { + string Calculate(BlobProviderArgs args); + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj new file mode 100644 index 0000000000..2a4910e32c --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj @@ -0,0 +1,19 @@ + + + + + + netcoreapp3.1 + + 9f0d2c00-80c1-435b-bfab-2c39c8249091 + + + + + + + + + + + diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs new file mode 100644 index 0000000000..eef12cb0a1 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs @@ -0,0 +1,20 @@ +using Volo.Abp.Testing; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AbpBlobStoringAwsTestCommonBase : AbpIntegratedTest + { + protected override void SetAbpApplicationCreationOptions(AbpApplicationCreationOptions options) + { + options.UseAutofac(); + } + } + + public class AbpBlobStoringAwsTestBase : AbpIntegratedTest + { + protected override void SetAbpApplicationCreationOptions(AbpApplicationCreationOptions options) + { + options.UseAutofac(); + } + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs new file mode 100644 index 0000000000..d176a864e7 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs @@ -0,0 +1,92 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Amazon.S3.Model; +using Amazon.S3.Util; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Volo.Abp.Modularity; +using Volo.Abp.Threading; + +namespace Volo.Abp.BlobStoring.Aws +{ + /// + /// This module will not try to connect to aws. + /// + [DependsOn( + typeof(AbpBlobStoringAwsModule), + typeof(AbpBlobStoringTestModule) + )] + public class AbpBlobStoringAwsTestCommonModule : AbpModule + { + } + + [DependsOn( + typeof(AbpBlobStoringAwsTestCommonModule) + )] + public class AbpBlobStoringAwsTestModule : AbpModule + { + private const string UserSecretsId = "9f0d2c00-80c1-435b-bfab-2c39c8249091"; + + private readonly string _randomContainerName = "abp-aws-test-container-" + Guid.NewGuid().ToString("N"); + + private AwsBlobProviderConfiguration _configuration; + + public override void ConfigureServices(ServiceConfigurationContext context) + { + context.Services.ReplaceConfiguration(ConfigurationHelper.BuildConfiguration(builderAction: builder => + { + builder.AddUserSecrets(UserSecretsId); + })); + + var configuration = context.Services.GetConfiguration(); + var accessKeyId = configuration["Aws:AccessKeyId"]; + var secretAccessKey = configuration["Aws:SecretAccessKey"]; + var region = configuration["Aws:Region"]; + + Configure(options => + { + options.Containers.ConfigureAll((containerName, containerConfiguration) => + { + containerConfiguration.UseAws(aws => + { + aws.AccessKeyId = accessKeyId; + aws.SecretAccessKey = secretAccessKey; + aws.Region = region; + aws.CreateContainerIfNotExists = true; + aws.ContainerName = _randomContainerName; + + _configuration = aws; + }); + }); + }); + } + + public override void OnApplicationShutdown(ApplicationShutdownContext context) + { + AsyncHelper.RunSync(() => DeleteBucketAsync(context)); + } + + private async Task DeleteBucketAsync(ApplicationShutdownContext context) + { + var amazonS3Client = await context.ServiceProvider.GetService() + .GetAmazonS3Client(_configuration); + + if (await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, _randomContainerName)) + { + var blobs = await amazonS3Client.ListObjectsAsync(_randomContainerName); + + if (blobs.S3Objects.Any()) + { + await amazonS3Client.DeleteObjectsAsync(new DeleteObjectsRequest + { + BucketName = _randomContainerName, + Objects = blobs.S3Objects.Select(o => new KeyVersion {Key = o.Key}).ToList() + }); + } + + await amazonS3Client.DeleteBucketAsync(_randomContainerName); + } + } + } +} \ No newline at end of file diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs new file mode 100644 index 0000000000..662949e7e3 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs @@ -0,0 +1,16 @@ +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + /* + //Please set the correct connection string in secrets.json and continue the test. + + public class AwsBlobContainer_Tests : BlobContainer_Tests + { + public AwsBlobContainer_Tests() + { + + } + } + */ +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs new file mode 100644 index 0000000000..e4fbe714f8 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs @@ -0,0 +1,57 @@ +using System; +using Shouldly; +using Volo.Abp.MultiTenancy; +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobNameCalculatorTests : AbpBlobStoringAwsTestCommonBase + { + private readonly IAwsBlobNameCalculator _calculator; + private readonly ICurrentTenant _currentTenant; + + private const string AwsContainerName = "/"; + private const string AwsSeparator = "/"; + + public AwsBlobNameCalculatorTests() + { + _calculator = GetRequiredService(); + _currentTenant = GetRequiredService(); + } + + [Fact] + public void Default_Settings() + { + _calculator.Calculate( + GetArgs("my-container", "my-blob") + ).ShouldBe($"host{AwsSeparator}my-blob"); + } + + [Fact] + public void Default_Settings_With_TenantId() + { + var tenantId = Guid.NewGuid(); + + using (_currentTenant.Change(tenantId)) + { + _calculator.Calculate( + GetArgs("my-container", "my-blob") + ).ShouldBe($"tenants{AwsSeparator}{tenantId:D}{AwsSeparator}my-blob"); + } + } + + private static BlobProviderArgs GetArgs( + string containerName, + string blobName) + { + return new BlobProviderGetArgs( + containerName, + new BlobContainerConfiguration().UseAws(x => + { + x.ContainerName = containerName; + }), + blobName + ); + } + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs new file mode 100644 index 0000000000..7798c52419 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs @@ -0,0 +1,57 @@ +using Shouldly; +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAwsBlobNamingNormalizerProviderTests : AbpBlobStoringAwsTestCommonBase + { + private readonly IBlobNamingNormalizer _blobNamingNormalizer; + + public DefaultAwsBlobNamingNormalizerProviderTests() + { + _blobNamingNormalizer = GetRequiredService(); + } + + [Fact] + public void NormalizeContainerName_Lowercase() + { + var filename = "ThisIsMyContainerName"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("thisismycontainername"); + } + + [Fact] + public void NormalizeContainerName_Only_Letters_Numbers_Dash() + { + var filename = ",./this-i,./s-my-c,./ont,./ai+*/.=!@#$n^&*er-name.+/"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("this-is-my-container-name"); + } + + [Fact] + public void NormalizeContainerName_Dash() + { + var filename = "-this--is----my-container----name-"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("this-is-my-container-name"); + } + + + [Fact] + public void NormalizeContainerName_Min_Length() + { + var filename = "a"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.Length.ShouldBeGreaterThanOrEqualTo(3); + } + + + [Fact] + public void NormalizeContainerName_Max_Length() + { + var filename = "abpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabp"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.Length.ShouldBeLessThanOrEqualTo(63); + } + } +} diff --git a/nupkg/common.ps1 b/nupkg/common.ps1 index 6d79ad6b37..99244d5d6c 100644 --- a/nupkg/common.ps1 +++ b/nupkg/common.ps1 @@ -61,6 +61,7 @@ $projects = ( "framework/src/Volo.Abp.BlobStoring.Aliyun", "framework/src/Volo.Abp.BlobStoring.Azure", "framework/src/Volo.Abp.BlobStoring.Minio", + "framework/src/Volo.Abp.BlobStoring.Aws", "framework/src/Volo.Abp.Caching", "framework/src/Volo.Abp.Caching.StackExchangeRedis", "framework/src/Volo.Abp.Castle.Core",