From 643ab8d52637c6b972140a8fbb1e378eaba5ebca Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Mon, 20 Jul 2020 23:25:36 +0800 Subject: [PATCH 1/5] Add aws s3 blob provider --- framework/Volo.Abp.sln | 7 + .../Volo.Abp.BlobStoring.Aws/FodyWeavers.xml | 3 + .../Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd | 30 ++++ .../Volo.Abp.BlobStoring.Aws.csproj | 24 +++ .../Aws/AbpBlobStoringAzureModule.cs | 10 ++ .../Aws/AssumeRoleCredentialsCacheItem.cs | 30 ++++ ...AwsBlobContainerConfigurationExtensions.cs | 25 +++ .../Aws/AwsBlobNamingNormalizer.cs | 52 ++++++ .../Abp/BlobStoring/Aws/AwsBlobProvider.cs | 148 +++++++++++++++ .../Aws/AwsBlobProviderConfiguration.cs | 107 +++++++++++ .../Aws/AwsBlobProviderConfigurationNames.cs | 19 ++ .../Aws/DefaultAmazonS3ClientFactory.cs | 168 ++++++++++++++++++ .../BlobStoring/Aws/IAmazonS3ClientFactory.cs | 10 ++ .../BlobStoring/Aws/IAwsBlobNameCalculator.cs | 7 + 14 files changed, 640 insertions(+) create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs diff --git a/framework/Volo.Abp.sln b/framework/Volo.Abp.sln index 5a83ae1460..1c8e0baeca 100644 --- a/framework/Volo.Abp.sln +++ b/framework/Volo.Abp.sln @@ -319,6 +319,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun.Tests", "test\Volo.Abp.BlobStoring.Aliyun.Tests\Volo.Abp.BlobStoring.Aliyun.Tests.csproj", "{8E49687A-E69F-49F2-8DB0-428D0883A937}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws", "src\Volo.Abp.BlobStoring.Aws\Volo.Abp.BlobStoring.Aws.csproj", "{50968CDE-1029-4051-B2E5-B69D0ECF2A18}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -949,6 +951,10 @@ Global {8E49687A-E69F-49F2-8DB0-428D0883A937}.Debug|Any CPU.Build.0 = Debug|Any CPU {8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.ActiveCfg = Release|Any CPU {8E49687A-E69F-49F2-8DB0-428D0883A937}.Release|Any CPU.Build.0 = Release|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.Build.0 = Debug|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.ActiveCfg = Release|Any CPU + {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1110,6 +1116,7 @@ Global {60D0E384-965E-4F81-9D71-B28F419254FC} = {447C8A77-E5F0-4538-8687-7383196D04EA} {845E6A13-D1B5-4DDC-A16C-68D807E3B4C7} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} {8E49687A-E69F-49F2-8DB0-428D0883A937} = {447C8A77-E5F0-4538-8687-7383196D04EA} + {50968CDE-1029-4051-B2E5-B69D0ECF2A18} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BB97ECF4-9A84-433F-A80B-2A3285BDD1D5} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml new file mode 100644 index 0000000000..be0de3a908 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd new file mode 100644 index 0000000000..3f3946e282 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/FodyWeavers.xsd @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + 'true' to run assembly verification (PEVerify) on the target assembly after all weavers have been executed. + + + + + A comma-separated list of error codes that can be safely ignored in assembly verification. + + + + + 'false' to turn off automatic generation of the XML Schema file. + + + + + \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj new file mode 100644 index 0000000000..d30601d1e8 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj @@ -0,0 +1,24 @@ + + + + + + + netstandard2.0 + false + false + false + + + + + + + + + + + + + + diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs new file mode 100644 index 0000000000..0b386fbec1 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs @@ -0,0 +1,10 @@ +using Volo.Abp.Modularity; + +namespace Volo.Abp.BlobStoring.Azure +{ + [DependsOn(typeof(AbpBlobStoringModule))] + public class AbpBlobStoringAzureModule : AbpModule + { + + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs new file mode 100644 index 0000000000..dbc4783978 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs @@ -0,0 +1,30 @@ +using System; +using Volo.Abp.Caching; + +namespace Volo.Abp.BlobStoring.Aws +{ + [Serializable] + [CacheName("TemporaryCredentials")] + public class TemporaryCredentialsCacheItem + { + public const string Key = "AwsTemporaryCredentialsCache"; + + public string AccessKeyId { get; set; } + + public string SecretAccessKey { get; set; } + + public string SessionToken { get; set; } + + public TemporaryCredentialsCacheItem() + { + + } + + public TemporaryCredentialsCacheItem(string accessKeyId,string secretAccessKey,string sessionToken) + { + AccessKeyId = accessKeyId; + SecretAccessKey = secretAccessKey; + SessionToken = sessionToken; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs new file mode 100644 index 0000000000..d63e408aab --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobContainerConfigurationExtensions.cs @@ -0,0 +1,25 @@ +using System; + +namespace Volo.Abp.BlobStoring.Aws +{ + public static class AwsBlobContainerConfigurationExtensions + { + public static AwsBlobProviderConfiguration GetAwsConfiguration( + this BlobContainerConfiguration containerConfiguration) + { + return new AwsBlobProviderConfiguration(containerConfiguration); + } + + public static BlobContainerConfiguration UseAws( + this BlobContainerConfiguration containerConfiguration, + Action awsConfigureAction) + { + containerConfiguration.ProviderType = typeof(AwsBlobProvider); + containerConfiguration.NamingNormalizers.TryAdd(); + + awsConfigureAction(new AwsBlobProviderConfiguration(containerConfiguration)); + + return containerConfiguration; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs new file mode 100644 index 0000000000..cd8eca021e --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobNamingNormalizer.cs @@ -0,0 +1,52 @@ +using System.Text.RegularExpressions; +using Volo.Abp.DependencyInjection; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobNamingNormalizer : IBlobNamingNormalizer, ITransientDependency + { + /// + ///https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html + /// + public virtual string NormalizeContainerName(string containerName) + { + // All letters in a container name must be lowercase. + containerName = containerName.ToLower(); + + // Container names can contain only letters, numbers, and the dash (-) character. + containerName = Regex.Replace(containerName, "[^a-z0-9-]", string.Empty); + + // Every dash (-) character must be immediately preceded and followed by a letter or number; + // consecutive dashes are not permitted in container names. + // Container names must start or end with a letter or number + containerName = Regex.Replace(containerName, "-{2,}", "-"); + containerName = Regex.Replace(containerName, "^-", string.Empty); + containerName = Regex.Replace(containerName, "-$", string.Empty); + + // Container names must be from 3 through 63 characters long. + if (containerName.Length < 3) + { + var length = containerName.Length; + for (var i = 0; i < 3 - length; i++) + { + containerName += "0"; + } + } + + if (containerName.Length > 63) + { + containerName = containerName.Substring(0, 63); + } + + return containerName; + } + + /// + /// https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + /// + public virtual string NormalizeBlobName(string blobName) + { + return blobName; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs new file mode 100644 index 0000000000..17831b6f4b --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs @@ -0,0 +1,148 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Util; +using Volo.Abp.DependencyInjection; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobProvider : BlobProviderBase, ITransientDependency + { + protected IAwsBlobNameCalculator AwsBlobNameCalculator { get; } + protected IAmazonS3ClientFactory AmazonS3ClientFactory { get; } + + public AwsBlobProvider(IAwsBlobNameCalculator awsBlobNameCalculator, + IAmazonS3ClientFactory amazonS3ClientFactory) + { + AwsBlobNameCalculator = awsBlobNameCalculator; + AmazonS3ClientFactory = amazonS3ClientFactory; + } + + public override async Task SaveAsync(BlobProviderSaveArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var configuration = args.Configuration.GetAwsConfiguration(); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!args.OverrideExisting && await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + throw new BlobAlreadyExistsException( + $"Saving BLOB '{args.BlobName}' does already exists in the container '{containerName}'! Set {nameof(args.OverrideExisting)} if it should be overwritten."); + } + + if (configuration.CreateContainerIfNotExists) + { + await amazonS3Client.PutBucketAsync(containerName); + } + + await amazonS3Client.PutObjectAsync(new PutObjectRequest + { + BucketName = containerName, + Key = blobName, + InputStream = args.BlobStream + }); + } + } + + public override async Task DeleteAsync(BlobProviderDeleteArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + return false; + } + + await amazonS3Client.DeleteObjectAsync(new DeleteObjectRequest + { + BucketName = containerName, + Key = blobName + }); + + return true; + } + } + + public override async Task ExistsAsync(BlobProviderExistsArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + return await BlobExistsAsync(amazonS3Client, containerName, blobName); + } + } + + public override async Task GetOrNullAsync(BlobProviderGetArgs args) + { + var blobName = AwsBlobNameCalculator.Calculate(args); + var containerName = GetContainerName(args); + + using (var amazonS3Client = await GetAmazonS3Client(args)) + { + if (!await BlobExistsAsync(amazonS3Client, containerName, blobName)) + { + return null; + } + + var response = await amazonS3Client.GetObjectAsync(new GetObjectRequest + { + BucketName = containerName, + Key = blobName + }); + + var memoryStream = new MemoryStream(); + await response.ResponseStream.CopyToAsync(memoryStream); + return memoryStream; + } + } + + protected virtual async Task GetAmazonS3Client(BlobProviderArgs args) + { + var configuration = args.Configuration.GetAwsConfiguration(); + + return await AmazonS3ClientFactory.GetAmazonS3Client(configuration); + } + + private async Task BlobExistsAsync(AmazonS3Client amazonS3Client, string containerName, string blobName) + { + // Make sure Blob Container exists. + if (!await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, containerName)) + { + return false; + } + + try + { + await amazonS3Client.GetObjectMetadataAsync(containerName, blobName); + } + catch (Exception ex) + { + if (ex is AmazonS3Exception) + { + return false; + } + + throw; + } + + return true; + } + + private static string GetContainerName(BlobProviderArgs args) + { + var configuration = args.Configuration.GetAwsConfiguration(); + return configuration.ContainerName.IsNullOrWhiteSpace() + ? args.ContainerName + : configuration.ContainerName; + } + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs new file mode 100644 index 0000000000..fde0f10fc9 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs @@ -0,0 +1,107 @@ +using Amazon; +using Amazon.Runtime; +using Amazon.Runtime.CredentialManagement; +using Amazon.S3; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobProviderConfiguration + { + public string AccessKeyId + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId, value); + } + + public string SecretAccessKey + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey, value); + } + + public bool UseAwsCredentials + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseAwsCredentials); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseAwsCredentials, value); + } + + public bool UseTemporaryCredentials + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials, value); + } + + public bool UseTemporaryFederatedCredentials + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials, value); + } + + public string ProfileName + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ProfileName); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfileName, value); + } + + public string ProfilesLocation + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation, value); + } + + /// + /// Set the validity period of the temporary access credential, the unit is s, the minimum is 900, and the maximum is 129600. + /// + public int DurationSeconds + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.DurationSeconds, 0); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.DurationSeconds, value); + } + + public string Name + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Name); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Name, value); + } + + public string Policy + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Policy); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Policy, value); + } + + public RegionEndpoint Region + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Region); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, Check.NotNull(value, nameof(value))); + } + + /// + /// This name may only contain lowercase letters, numbers, and hyphens, and must begin with a letter or a number. + /// Each hyphen must be preceded and followed by a non-hyphen character. + /// The name must also be between 3 and 63 characters long. + /// If this parameter is not specified, the ContainerName of the will be used. + /// + public string ContainerName + { + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ContainerName); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ContainerName, Check.NotNullOrWhiteSpace(value, nameof(value))); + } + + /// + /// Default value: false. + /// + public bool CreateContainerIfNotExists + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, value); + } + + private readonly BlobContainerConfiguration _containerConfiguration; + + public AwsBlobProviderConfiguration(BlobContainerConfiguration containerConfiguration) + { + _containerConfiguration = containerConfiguration; + } + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs new file mode 100644 index 0000000000..f27efc1552 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs @@ -0,0 +1,19 @@ +namespace Volo.Abp.BlobStoring.Aws +{ + public static class AwsBlobProviderConfigurationNames + { + public const string AccessKeyId = "Aws.AccessKeyId"; + public const string SecretAccessKey = "Aws.SecretAccessKey"; + public const string UseAwsCredentials = "Aws.UseAWSCredentials"; + public const string UseTemporaryCredentials = "Aws.UseTemporaryCredentials"; + public const string UseTemporaryFederatedCredentials = "Aws.UseTemporaryFederatedCredentials"; + public const string ProfileName = "Aws.ProfileName"; + public const string ProfilesLocation = "Aws.ProfilesLocation"; + public const string DurationSeconds = "Aws.DurationSeconds"; + public const string Name = "Aws.Name"; + public const string Policy = "Aws.Policy"; + public const string Region = "Aws.Region"; + public const string ContainerName = "Aws.ContainerName"; + public const string CreateContainerIfNotExists = "Aws.CreateContainerIfNotExists"; + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs new file mode 100644 index 0000000000..febda577fc --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs @@ -0,0 +1,168 @@ +using System; +using System.Threading.Tasks; +using Amazon.Runtime; +using Amazon.Runtime.CredentialManagement; +using Amazon.S3; +using Amazon.SecurityToken; +using Amazon.SecurityToken.Model; +using Microsoft.Extensions.Caching.Distributed; +using Volo.Abp.Caching; +using Volo.Abp.DependencyInjection; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAmazonS3ClientFactory : IAmazonS3ClientFactory, ITransientDependency + { + protected IDistributedCache Cache { get; } + + public DefaultAmazonS3ClientFactory(IDistributedCache cache) + { + Cache = cache; + } + + public virtual async Task GetAmazonS3Client( + AwsBlobProviderConfiguration configuration) + { + if (configuration.UseAwsCredentials) + { + return new AmazonS3Client(GetAwsCredentials(configuration), configuration.Region); + } + + if (configuration.UseTemporaryCredentials) + { + return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), configuration.Region); + } + + if (configuration.UseTemporaryFederatedCredentials) + { + return new AmazonS3Client(await GetTemporaryFederatedCredentialsAsync(configuration), + configuration.Region); + } + + Check.NotNullOrWhiteSpace(configuration.AccessKeyId, nameof(configuration.AccessKeyId)); + Check.NotNullOrWhiteSpace(configuration.SecretAccessKey, nameof(configuration.SecretAccessKey)); + + return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey); + } + + protected virtual AWSCredentials GetAwsCredentials( + AwsBlobProviderConfiguration configuration) + { + var chain = new CredentialProfileStoreChain(configuration.ProfilesLocation); + + if (chain.TryGetAWSCredentials(configuration.ProfileName, out var awsCredentials)) + { + return awsCredentials; + } + + throw new AmazonS3Exception("Not found aws credentials"); + } + + protected virtual async Task GetTemporaryCredentialsAsync( + AwsBlobProviderConfiguration configuration) + { + var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key); + + if (temporaryCredentialsCache == null) + { + AmazonSecurityTokenServiceClient stsClient; + + if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty()) + { + stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId, + configuration.SecretAccessKey); + } + else + { + stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration)); + } + + using (stsClient) + { + var getSessionTokenRequest = new GetSessionTokenRequest + { + DurationSeconds = configuration.DurationSeconds + }; + + var sessionTokenResponse = + await stsClient.GetSessionTokenAsync(getSessionTokenRequest); + + var credentials = sessionTokenResponse.Credentials; + + temporaryCredentialsCache = + await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds); + } + } + + var sessionCredentials = new SessionAWSCredentials( + temporaryCredentialsCache.AccessKeyId, + temporaryCredentialsCache.SecretAccessKey, + temporaryCredentialsCache.SessionToken); + return sessionCredentials; + } + + protected virtual async Task GetTemporaryFederatedCredentialsAsync( + AwsBlobProviderConfiguration configuration) + { + Check.NotNullOrWhiteSpace(configuration.Name, nameof(configuration.Name)); + + var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key); + + if (temporaryCredentialsCache == null) + { + AmazonSecurityTokenServiceClient stsClient; + + if (!configuration.AccessKeyId.IsNullOrEmpty() && !configuration.SecretAccessKey.IsNullOrEmpty()) + { + stsClient = new AmazonSecurityTokenServiceClient(configuration.AccessKeyId, + configuration.SecretAccessKey); + } + else + { + stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration)); + } + + using (stsClient) + { + var federationTokenRequest = + new GetFederationTokenRequest + { + DurationSeconds = configuration.DurationSeconds, + Name = configuration.Name, + Policy = configuration.Policy + }; + + var federationTokenResponse = + await stsClient.GetFederationTokenAsync(federationTokenRequest); + var credentials = federationTokenResponse.Credentials; + + temporaryCredentialsCache = + await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds); + } + } + + var sessionCredentials = new SessionAWSCredentials( + temporaryCredentialsCache.AccessKeyId, + temporaryCredentialsCache.SecretAccessKey, + temporaryCredentialsCache.SessionToken); + return sessionCredentials; + } + + private async Task SetTemporaryCredentialsCache( + Credentials credentials, + int durationSeconds) + { + var temporaryCredentialsCache = new TemporaryCredentialsCacheItem(credentials.AccessKeyId, + credentials.SecretAccessKey, + credentials.SessionToken); + + await Cache.SetAsync(TemporaryCredentialsCacheItem.Key, temporaryCredentialsCache, + new DistributedCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(durationSeconds - 10) + }); + + return temporaryCredentialsCache; + } + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs new file mode 100644 index 0000000000..92d7ddc906 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAmazonS3ClientFactory.cs @@ -0,0 +1,10 @@ +using System.Threading.Tasks; +using Amazon.S3; + +namespace Volo.Abp.BlobStoring.Aws +{ + public interface IAmazonS3ClientFactory + { + Task GetAmazonS3Client(AwsBlobProviderConfiguration configuration); + } +} \ No newline at end of file diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs new file mode 100644 index 0000000000..e4c470c505 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/IAwsBlobNameCalculator.cs @@ -0,0 +1,7 @@ +namespace Volo.Abp.BlobStoring.Aws +{ + public interface IAwsBlobNameCalculator + { + string Calculate(BlobProviderArgs args); + } +} From ecd234de981eb44f44366a6c015d8f3f350b375a Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Tue, 21 Jul 2020 22:52:18 +0800 Subject: [PATCH 2/5] Add unit tests --- framework/Volo.Abp.sln | 7 ++ .../Aws/AbpBlobStoringAwsModule.cs | 12 +++ .../Aws/AbpBlobStoringAzureModule.cs | 10 -- .../Abp/BlobStoring/Aws/AwsBlobProvider.cs | 14 ++- .../Aws/AwsBlobProviderConfiguration.cs | 24 ++--- .../Aws/AwsBlobProviderConfigurationNames.cs | 2 +- ...cs => AwsTemporaryCredentialsCacheItem.cs} | 10 +- .../Aws/DefaultAmazonS3ClientFactory.cs | 46 +++++++--- .../Aws/DefaultAwsBlobNameCalculator.cs | 22 +++++ .../Volo.Abp.BlobStoring.Aws.Tests.csproj | 19 ++++ .../Aws/AbpBlobStoringAwsTestBase.cs | 20 ++++ .../Aws/AbpBlobStoringAwsTestModule.cs | 91 +++++++++++++++++++ .../BlobStoring/Aws/AwsBlobContainer_Tests.cs | 16 ++++ .../Aws/AwsBlobNameCalculator_Tests.cs | 57 ++++++++++++ ...ltAwsBlobNamingNormalizerProvider_Tests.cs | 57 ++++++++++++ 15 files changed, 362 insertions(+), 45 deletions(-) create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs delete mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs rename framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/{AssumeRoleCredentialsCacheItem.cs => AwsTemporaryCredentialsCacheItem.cs} (55%) create mode 100644 framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs create mode 100644 framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs diff --git a/framework/Volo.Abp.sln b/framework/Volo.Abp.sln index 1c8e0baeca..5778b054ed 100644 --- a/framework/Volo.Abp.sln +++ b/framework/Volo.Abp.sln @@ -321,6 +321,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Volo.Abp.BlobStoring.Aliyun EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws", "src\Volo.Abp.BlobStoring.Aws\Volo.Abp.BlobStoring.Aws.csproj", "{50968CDE-1029-4051-B2E5-B69D0ECF2A18}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Volo.Abp.BlobStoring.Aws.Tests", "test\Volo.Abp.BlobStoring.Aws.Tests\Volo.Abp.BlobStoring.Aws.Tests.csproj", "{2CD3B26A-CA81-4279-8D5D-6A594517BB3F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -955,6 +957,10 @@ Global {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Debug|Any CPU.Build.0 = Debug|Any CPU {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.ActiveCfg = Release|Any CPU {50968CDE-1029-4051-B2E5-B69D0ECF2A18}.Release|Any CPU.Build.0 = Release|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1117,6 +1123,7 @@ Global {845E6A13-D1B5-4DDC-A16C-68D807E3B4C7} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} {8E49687A-E69F-49F2-8DB0-428D0883A937} = {447C8A77-E5F0-4538-8687-7383196D04EA} {50968CDE-1029-4051-B2E5-B69D0ECF2A18} = {5DF0E140-0513-4D0D-BE2E-3D4D85CD70E6} + {2CD3B26A-CA81-4279-8D5D-6A594517BB3F} = {447C8A77-E5F0-4538-8687-7383196D04EA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {BB97ECF4-9A84-433F-A80B-2A3285BDD1D5} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs new file mode 100644 index 0000000000..84a18a5ca9 --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs @@ -0,0 +1,12 @@ +using Volo.Abp.Caching; +using Volo.Abp.Modularity; + +namespace Volo.Abp.BlobStoring.Aws +{ + [DependsOn(typeof(AbpBlobStoringModule), + typeof(AbpCachingModule))] + public class AbpBlobStoringAwsModule : AbpModule + { + + } +} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs deleted file mode 100644 index 0b386fbec1..0000000000 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAzureModule.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Volo.Abp.Modularity; - -namespace Volo.Abp.BlobStoring.Azure -{ - [DependsOn(typeof(AbpBlobStoringModule))] - public class AbpBlobStoringAzureModule : AbpModule - { - - } -} diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs index 17831b6f4b..6acb91aea9 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProvider.cs @@ -36,7 +36,7 @@ namespace Volo.Abp.BlobStoring.Aws if (configuration.CreateContainerIfNotExists) { - await amazonS3Client.PutBucketAsync(containerName); + await CreateContainerIfNotExists(amazonS3Client, containerName); } await amazonS3Client.PutObjectAsync(new PutObjectRequest @@ -108,7 +108,6 @@ namespace Volo.Abp.BlobStoring.Aws protected virtual async Task GetAmazonS3Client(BlobProviderArgs args) { var configuration = args.Configuration.GetAwsConfiguration(); - return await AmazonS3ClientFactory.GetAmazonS3Client(configuration); } @@ -137,6 +136,17 @@ namespace Volo.Abp.BlobStoring.Aws return true; } + protected virtual async Task CreateContainerIfNotExists(AmazonS3Client amazonS3Client, string containerName) + { + if (!await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, containerName)) + { + await amazonS3Client.PutBucketAsync(new PutBucketRequest + { + BucketName = containerName + }); + } + } + private static string GetContainerName(BlobProviderArgs args) { var configuration = args.Configuration.GetAwsConfiguration(); diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs index fde0f10fc9..62695610ff 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs @@ -9,43 +9,43 @@ namespace Volo.Abp.BlobStoring.Aws { public string AccessKeyId { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.AccessKeyId); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.AccessKeyId, value); } public string SecretAccessKey { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.SecretAccessKey); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.SecretAccessKey, value); } - public bool UseAwsCredentials + public bool UseCredentials { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseAwsCredentials); - set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseAwsCredentials, value); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseCredentials,false); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseCredentials, value); } public bool UseTemporaryCredentials { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseTemporaryCredentials,false); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryCredentials, value); } public bool UseTemporaryFederatedCredentials { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials,false); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.UseTemporaryFederatedCredentials, value); } public string ProfileName { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ProfileName); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.ProfileName); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfileName, value); } public string ProfilesLocation { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.ProfilesLocation); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.ProfilesLocation, value); } @@ -66,13 +66,13 @@ namespace Volo.Abp.BlobStoring.Aws public string Policy { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Policy); + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.Policy); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Policy, value); } - public RegionEndpoint Region + public string Region { - get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Region); + get => _containerConfiguration.GetConfiguration(AwsBlobProviderConfigurationNames.Region); set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.Region, Check.NotNull(value, nameof(value))); } diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs index f27efc1552..600aab5bd7 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs @@ -4,7 +4,7 @@ { public const string AccessKeyId = "Aws.AccessKeyId"; public const string SecretAccessKey = "Aws.SecretAccessKey"; - public const string UseAwsCredentials = "Aws.UseAWSCredentials"; + public const string UseCredentials = "Aws.UseCredentials"; public const string UseTemporaryCredentials = "Aws.UseTemporaryCredentials"; public const string UseTemporaryFederatedCredentials = "Aws.UseTemporaryFederatedCredentials"; public const string ProfileName = "Aws.ProfileName"; diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs similarity index 55% rename from framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs rename to framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs index dbc4783978..d45cb2d958 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AssumeRoleCredentialsCacheItem.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs @@ -1,13 +1,11 @@ using System; -using Volo.Abp.Caching; namespace Volo.Abp.BlobStoring.Aws { [Serializable] - [CacheName("TemporaryCredentials")] - public class TemporaryCredentialsCacheItem + public class AwsTemporaryCredentialsCacheItem { - public const string Key = "AwsTemporaryCredentialsCache"; + public const string Key = "AwsBlobTemporaryCredentialsCache"; public string AccessKeyId { get; set; } @@ -15,12 +13,12 @@ namespace Volo.Abp.BlobStoring.Aws public string SessionToken { get; set; } - public TemporaryCredentialsCacheItem() + public AwsTemporaryCredentialsCacheItem() { } - public TemporaryCredentialsCacheItem(string accessKeyId,string secretAccessKey,string sessionToken) + public AwsTemporaryCredentialsCacheItem(string accessKeyId,string secretAccessKey,string sessionToken) { AccessKeyId = accessKeyId; SecretAccessKey = secretAccessKey; diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs index febda577fc..7ae6e4c004 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs @@ -1,5 +1,6 @@ using System; using System.Threading.Tasks; +using Amazon; using Amazon.Runtime; using Amazon.Runtime.CredentialManagement; using Amazon.S3; @@ -13,9 +14,9 @@ namespace Volo.Abp.BlobStoring.Aws { public class DefaultAmazonS3ClientFactory : IAmazonS3ClientFactory, ITransientDependency { - protected IDistributedCache Cache { get; } + protected IDistributedCache Cache { get; } - public DefaultAmazonS3ClientFactory(IDistributedCache cache) + public DefaultAmazonS3ClientFactory(IDistributedCache cache) { Cache = cache; } @@ -23,31 +24,41 @@ namespace Volo.Abp.BlobStoring.Aws public virtual async Task GetAmazonS3Client( AwsBlobProviderConfiguration configuration) { - if (configuration.UseAwsCredentials) + var region = RegionEndpoint.GetBySystemName(configuration.Region); + + if (configuration.UseCredentials) { - return new AmazonS3Client(GetAwsCredentials(configuration), configuration.Region); + var awsCredentials = GetAwsCredentials(configuration); + return awsCredentials == null + ? new AmazonS3Client(region) + : new AmazonS3Client(GetAwsCredentials(configuration), region); } if (configuration.UseTemporaryCredentials) { - return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), configuration.Region); + return new AmazonS3Client(await GetTemporaryCredentialsAsync(configuration), region); } if (configuration.UseTemporaryFederatedCredentials) { return new AmazonS3Client(await GetTemporaryFederatedCredentialsAsync(configuration), - configuration.Region); + region); } Check.NotNullOrWhiteSpace(configuration.AccessKeyId, nameof(configuration.AccessKeyId)); Check.NotNullOrWhiteSpace(configuration.SecretAccessKey, nameof(configuration.SecretAccessKey)); - return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey); + return new AmazonS3Client(configuration.AccessKeyId, configuration.SecretAccessKey, configuration.Region); } protected virtual AWSCredentials GetAwsCredentials( AwsBlobProviderConfiguration configuration) { + if (configuration.ProfileName.IsNullOrWhiteSpace()) + { + return null; + } + var chain = new CredentialProfileStoreChain(configuration.ProfilesLocation); if (chain.TryGetAWSCredentials(configuration.ProfileName, out var awsCredentials)) @@ -61,7 +72,7 @@ namespace Volo.Abp.BlobStoring.Aws protected virtual async Task GetTemporaryCredentialsAsync( AwsBlobProviderConfiguration configuration) { - var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key); + var temporaryCredentialsCache = await Cache.GetAsync(AwsTemporaryCredentialsCacheItem.Key); if (temporaryCredentialsCache == null) { @@ -74,7 +85,10 @@ namespace Volo.Abp.BlobStoring.Aws } else { - stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration)); + var awsCredentials = GetAwsCredentials(configuration); + stsClient = awsCredentials == null + ? new AmazonSecurityTokenServiceClient() + : new AmazonSecurityTokenServiceClient(awsCredentials); } using (stsClient) @@ -105,8 +119,9 @@ namespace Volo.Abp.BlobStoring.Aws AwsBlobProviderConfiguration configuration) { Check.NotNullOrWhiteSpace(configuration.Name, nameof(configuration.Name)); + Check.NotNullOrWhiteSpace(configuration.Policy, nameof(configuration.Policy)); - var temporaryCredentialsCache = await Cache.GetAsync(TemporaryCredentialsCacheItem.Key); + var temporaryCredentialsCache = await Cache.GetAsync(AwsTemporaryCredentialsCacheItem.Key); if (temporaryCredentialsCache == null) { @@ -119,7 +134,10 @@ namespace Volo.Abp.BlobStoring.Aws } else { - stsClient = new AmazonSecurityTokenServiceClient(GetAwsCredentials(configuration)); + var awsCredentials = GetAwsCredentials(configuration); + stsClient = awsCredentials == null + ? new AmazonSecurityTokenServiceClient() + : new AmazonSecurityTokenServiceClient(awsCredentials); } using (stsClient) @@ -148,15 +166,15 @@ namespace Volo.Abp.BlobStoring.Aws return sessionCredentials; } - private async Task SetTemporaryCredentialsCache( + private async Task SetTemporaryCredentialsCache( Credentials credentials, int durationSeconds) { - var temporaryCredentialsCache = new TemporaryCredentialsCacheItem(credentials.AccessKeyId, + var temporaryCredentialsCache = new AwsTemporaryCredentialsCacheItem(credentials.AccessKeyId, credentials.SecretAccessKey, credentials.SessionToken); - await Cache.SetAsync(TemporaryCredentialsCacheItem.Key, temporaryCredentialsCache, + await Cache.SetAsync(AwsTemporaryCredentialsCacheItem.Key, temporaryCredentialsCache, new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(durationSeconds - 10) diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs new file mode 100644 index 0000000000..9ff3511e6f --- /dev/null +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNameCalculator.cs @@ -0,0 +1,22 @@ +using Volo.Abp.DependencyInjection; +using Volo.Abp.MultiTenancy; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAwsBlobNameCalculator : IAwsBlobNameCalculator, ITransientDependency + { + protected ICurrentTenant CurrentTenant { get; } + + public DefaultAwsBlobNameCalculator(ICurrentTenant currentTenant) + { + CurrentTenant = currentTenant; + } + + public virtual string Calculate(BlobProviderArgs args) + { + return CurrentTenant.Id == null + ? $"host/{args.BlobName}" + : $"tenants/{CurrentTenant.Id.Value.ToString("D")}/{args.BlobName}"; + } + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj new file mode 100644 index 0000000000..2a4910e32c --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo.Abp.BlobStoring.Aws.Tests.csproj @@ -0,0 +1,19 @@ + + + + + + netcoreapp3.1 + + 9f0d2c00-80c1-435b-bfab-2c39c8249091 + + + + + + + + + + + diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs new file mode 100644 index 0000000000..eef12cb0a1 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestBase.cs @@ -0,0 +1,20 @@ +using Volo.Abp.Testing; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AbpBlobStoringAwsTestCommonBase : AbpIntegratedTest + { + protected override void SetAbpApplicationCreationOptions(AbpApplicationCreationOptions options) + { + options.UseAutofac(); + } + } + + public class AbpBlobStoringAwsTestBase : AbpIntegratedTest + { + protected override void SetAbpApplicationCreationOptions(AbpApplicationCreationOptions options) + { + options.UseAutofac(); + } + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs new file mode 100644 index 0000000000..dc9dda1b76 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs @@ -0,0 +1,91 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Amazon.S3.Model; +using Amazon.S3.Util; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Volo.Abp.Modularity; +using Volo.Abp.Threading; + +namespace Volo.Abp.BlobStoring.Aws +{ + /// + /// This module will not try to connect to aws. + /// + [DependsOn( + typeof(AbpBlobStoringAwsModule), + typeof(AbpBlobStoringTestModule) + )] + public class AbpBlobStoringAwsTestCommonModule : AbpModule + { + } + + [DependsOn( + typeof(AbpBlobStoringAwsTestCommonModule) + )] + public class AbpBlobStoringAwsTestModule : AbpModule + { + private const string UserSecretsId = "9f0d2c00-80c1-435b-bfab-2c39c8249091"; + + private readonly string _randomContainerName = "abp-aws-test-container-" + Guid.NewGuid().ToString("N"); + + private AwsBlobProviderConfiguration _configuration; + + public override void ConfigureServices(ServiceConfigurationContext context) + { + context.Services.ReplaceConfiguration(ConfigurationHelper.BuildConfiguration(builderAction: builder => + { + builder.AddUserSecrets(UserSecretsId); + })); + + var configuration = context.Services.GetConfiguration(); + var accessKeyId = configuration["Aws:AccessKeyId"]; + var secretAccessKey = configuration["Aws:SecretAccessKey"]; + var region = configuration["Aws:Region"]; + + Configure(options => + { + options.Containers.ConfigureAll((containerName, containerConfiguration) => + { + containerConfiguration.UseAws(aws => + { + aws.AccessKeyId = accessKeyId; + aws.SecretAccessKey = secretAccessKey; + aws.Region = region; + aws.CreateContainerIfNotExists = true; + + _configuration = aws; + }); + }); + }); + } + + public override void OnApplicationShutdown(ApplicationShutdownContext context) + { + AsyncHelper.RunSync(() => DeleteBucketAsync(context)); + } + + private async Task DeleteBucketAsync(ApplicationShutdownContext context) + { + var amazonS3Client = await context.ServiceProvider.GetService() + .GetAmazonS3Client(_configuration); + + if (await AmazonS3Util.DoesS3BucketExistV2Async(amazonS3Client, _randomContainerName)) + { + var blobs = await amazonS3Client.ListObjectsAsync(_randomContainerName); + + if (blobs.S3Objects.Any()) + { + await amazonS3Client.DeleteObjectsAsync(new DeleteObjectsRequest + { + BucketName = _randomContainerName, + Objects = blobs.S3Objects.Select(o => new KeyVersion {Key = o.Key}).ToList() + }); + } + + await amazonS3Client.DeleteBucketAsync(_randomContainerName); + } + } + } +} \ No newline at end of file diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs new file mode 100644 index 0000000000..662949e7e3 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobContainer_Tests.cs @@ -0,0 +1,16 @@ +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + /* + //Please set the correct connection string in secrets.json and continue the test. + + public class AwsBlobContainer_Tests : BlobContainer_Tests + { + public AwsBlobContainer_Tests() + { + + } + } + */ +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs new file mode 100644 index 0000000000..e4fbe714f8 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AwsBlobNameCalculator_Tests.cs @@ -0,0 +1,57 @@ +using System; +using Shouldly; +using Volo.Abp.MultiTenancy; +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class AwsBlobNameCalculatorTests : AbpBlobStoringAwsTestCommonBase + { + private readonly IAwsBlobNameCalculator _calculator; + private readonly ICurrentTenant _currentTenant; + + private const string AwsContainerName = "/"; + private const string AwsSeparator = "/"; + + public AwsBlobNameCalculatorTests() + { + _calculator = GetRequiredService(); + _currentTenant = GetRequiredService(); + } + + [Fact] + public void Default_Settings() + { + _calculator.Calculate( + GetArgs("my-container", "my-blob") + ).ShouldBe($"host{AwsSeparator}my-blob"); + } + + [Fact] + public void Default_Settings_With_TenantId() + { + var tenantId = Guid.NewGuid(); + + using (_currentTenant.Change(tenantId)) + { + _calculator.Calculate( + GetArgs("my-container", "my-blob") + ).ShouldBe($"tenants{AwsSeparator}{tenantId:D}{AwsSeparator}my-blob"); + } + } + + private static BlobProviderArgs GetArgs( + string containerName, + string blobName) + { + return new BlobProviderGetArgs( + containerName, + new BlobContainerConfiguration().UseAws(x => + { + x.ContainerName = containerName; + }), + blobName + ); + } + } +} diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs new file mode 100644 index 0000000000..7798c52419 --- /dev/null +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/DefaultAwsBlobNamingNormalizerProvider_Tests.cs @@ -0,0 +1,57 @@ +using Shouldly; +using Xunit; + +namespace Volo.Abp.BlobStoring.Aws +{ + public class DefaultAwsBlobNamingNormalizerProviderTests : AbpBlobStoringAwsTestCommonBase + { + private readonly IBlobNamingNormalizer _blobNamingNormalizer; + + public DefaultAwsBlobNamingNormalizerProviderTests() + { + _blobNamingNormalizer = GetRequiredService(); + } + + [Fact] + public void NormalizeContainerName_Lowercase() + { + var filename = "ThisIsMyContainerName"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("thisismycontainername"); + } + + [Fact] + public void NormalizeContainerName_Only_Letters_Numbers_Dash() + { + var filename = ",./this-i,./s-my-c,./ont,./ai+*/.=!@#$n^&*er-name.+/"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("this-is-my-container-name"); + } + + [Fact] + public void NormalizeContainerName_Dash() + { + var filename = "-this--is----my-container----name-"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.ShouldBe("this-is-my-container-name"); + } + + + [Fact] + public void NormalizeContainerName_Min_Length() + { + var filename = "a"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.Length.ShouldBeGreaterThanOrEqualTo(3); + } + + + [Fact] + public void NormalizeContainerName_Max_Length() + { + var filename = "abpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabpabp"; + filename = _blobNamingNormalizer.NormalizeContainerName(filename); + filename.Length.ShouldBeLessThanOrEqualTo(63); + } + } +} From 281988d4c32a5d999565d5d18a46fe4583deba49 Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Wed, 22 Jul 2020 08:34:00 +0800 Subject: [PATCH 3/5] Update common.ps1 --- nupkg/common.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/nupkg/common.ps1 b/nupkg/common.ps1 index 6d79ad6b37..99244d5d6c 100644 --- a/nupkg/common.ps1 +++ b/nupkg/common.ps1 @@ -61,6 +61,7 @@ $projects = ( "framework/src/Volo.Abp.BlobStoring.Aliyun", "framework/src/Volo.Abp.BlobStoring.Azure", "framework/src/Volo.Abp.BlobStoring.Minio", + "framework/src/Volo.Abp.BlobStoring.Aws", "framework/src/Volo.Abp.Caching", "framework/src/Volo.Abp.Caching.StackExchangeRedis", "framework/src/Volo.Abp.Castle.Core", From ae78f0dcda6e9a633d9193cf983c3925fa86bff0 Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Wed, 22 Jul 2020 15:24:35 +0800 Subject: [PATCH 4/5] Improve --- .../Volo.Abp.BlobStoring.Aws.csproj | 2 + .../Aws/AbpBlobStoringAwsModule.cs | 8 +++- .../Aws/AwsBlobProviderConfiguration.cs | 13 ++++-- .../Aws/AwsBlobProviderConfigurationNames.cs | 1 + .../Aws/AwsTemporaryCredentialsCacheItem.cs | 2 - .../Aws/DefaultAmazonS3ClientFactory.cs | 42 +++++++++++-------- .../Aws/AbpBlobStoringAwsTestModule.cs | 1 + 7 files changed, 43 insertions(+), 26 deletions(-) diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj index d30601d1e8..814158b064 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj @@ -19,6 +19,8 @@ + + diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs index 84a18a5ca9..6f2e5dc330 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs @@ -1,4 +1,5 @@ -using Volo.Abp.Caching; +using Microsoft.Extensions.DependencyInjection; +using Volo.Abp.Caching; using Volo.Abp.Modularity; namespace Volo.Abp.BlobStoring.Aws @@ -7,6 +8,9 @@ namespace Volo.Abp.BlobStoring.Aws typeof(AbpCachingModule))] public class AbpBlobStoringAwsModule : AbpModule { - + public override void ConfigureServices(ServiceConfigurationContext context) + { + context.Services.AddDataProtection(); + } } } diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs index 62695610ff..c2d3297f4e 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfiguration.cs @@ -1,7 +1,4 @@ -using Amazon; -using Amazon.Runtime; -using Amazon.Runtime.CredentialManagement; -using Amazon.S3; +using System; namespace Volo.Abp.BlobStoring.Aws { @@ -97,11 +94,19 @@ namespace Volo.Abp.BlobStoring.Aws set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.CreateContainerIfNotExists, value); } + private readonly string _temporaryCredentialsCacheKey; + public string TemporaryCredentialsCacheKey + { + get => _containerConfiguration.GetConfigurationOrDefault(AwsBlobProviderConfigurationNames.TemporaryCredentialsCacheKey, _temporaryCredentialsCacheKey); + set => _containerConfiguration.SetConfiguration(AwsBlobProviderConfigurationNames.TemporaryCredentialsCacheKey, value); + } + private readonly BlobContainerConfiguration _containerConfiguration; public AwsBlobProviderConfiguration(BlobContainerConfiguration containerConfiguration) { _containerConfiguration = containerConfiguration; + _temporaryCredentialsCacheKey = Guid.NewGuid().ToString("N"); } } } diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs index 600aab5bd7..24dfca53d2 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsBlobProviderConfigurationNames.cs @@ -10,6 +10,7 @@ public const string ProfileName = "Aws.ProfileName"; public const string ProfilesLocation = "Aws.ProfilesLocation"; public const string DurationSeconds = "Aws.DurationSeconds"; + public const string TemporaryCredentialsCacheKey = "Aws.TemporaryCredentialsCacheKey"; public const string Name = "Aws.Name"; public const string Policy = "Aws.Policy"; public const string Region = "Aws.Region"; diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs index d45cb2d958..887acddc5d 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AwsTemporaryCredentialsCacheItem.cs @@ -5,8 +5,6 @@ namespace Volo.Abp.BlobStoring.Aws [Serializable] public class AwsTemporaryCredentialsCacheItem { - public const string Key = "AwsBlobTemporaryCredentialsCache"; - public string AccessKeyId { get; set; } public string SecretAccessKey { get; set; } diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs index 7ae6e4c004..717ef2e05d 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs @@ -6,6 +6,7 @@ using Amazon.Runtime.CredentialManagement; using Amazon.S3; using Amazon.SecurityToken; using Amazon.SecurityToken.Model; +using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Caching.Distributed; using Volo.Abp.Caching; using Volo.Abp.DependencyInjection; @@ -16,9 +17,13 @@ namespace Volo.Abp.BlobStoring.Aws { protected IDistributedCache Cache { get; } - public DefaultAmazonS3ClientFactory(IDistributedCache cache) + protected IDataProtector DataProtector { get; } + + public DefaultAmazonS3ClientFactory(IDistributedCache cache, + IDataProtectionProvider dataProtectionProvider) { Cache = cache; + DataProtector = dataProtectionProvider.CreateProtector(nameof(AwsTemporaryCredentialsCacheItem)); } public virtual async Task GetAmazonS3Client( @@ -72,7 +77,7 @@ namespace Volo.Abp.BlobStoring.Aws protected virtual async Task GetTemporaryCredentialsAsync( AwsBlobProviderConfiguration configuration) { - var temporaryCredentialsCache = await Cache.GetAsync(AwsTemporaryCredentialsCacheItem.Key); + var temporaryCredentialsCache = await Cache.GetAsync(configuration.TemporaryCredentialsCacheKey); if (temporaryCredentialsCache == null) { @@ -104,14 +109,14 @@ namespace Volo.Abp.BlobStoring.Aws var credentials = sessionTokenResponse.Credentials; temporaryCredentialsCache = - await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds); + await SetTemporaryCredentialsCache(configuration, credentials); } } var sessionCredentials = new SessionAWSCredentials( - temporaryCredentialsCache.AccessKeyId, - temporaryCredentialsCache.SecretAccessKey, - temporaryCredentialsCache.SessionToken); + DataProtector.Unprotect(temporaryCredentialsCache.AccessKeyId), + DataProtector.Unprotect(temporaryCredentialsCache.SecretAccessKey), + DataProtector.Unprotect(temporaryCredentialsCache.SessionToken)); return sessionCredentials; } @@ -121,7 +126,7 @@ namespace Volo.Abp.BlobStoring.Aws Check.NotNullOrWhiteSpace(configuration.Name, nameof(configuration.Name)); Check.NotNullOrWhiteSpace(configuration.Policy, nameof(configuration.Policy)); - var temporaryCredentialsCache = await Cache.GetAsync(AwsTemporaryCredentialsCacheItem.Key); + var temporaryCredentialsCache = await Cache.GetAsync(configuration.TemporaryCredentialsCacheKey); if (temporaryCredentialsCache == null) { @@ -155,29 +160,30 @@ namespace Volo.Abp.BlobStoring.Aws var credentials = federationTokenResponse.Credentials; temporaryCredentialsCache = - await SetTemporaryCredentialsCache(credentials, configuration.DurationSeconds); + await SetTemporaryCredentialsCache(configuration, credentials); } } var sessionCredentials = new SessionAWSCredentials( - temporaryCredentialsCache.AccessKeyId, - temporaryCredentialsCache.SecretAccessKey, - temporaryCredentialsCache.SessionToken); + DataProtector.Unprotect(temporaryCredentialsCache.AccessKeyId), + DataProtector.Unprotect(temporaryCredentialsCache.SecretAccessKey), + DataProtector.Unprotect(temporaryCredentialsCache.SessionToken)); return sessionCredentials; } private async Task SetTemporaryCredentialsCache( - Credentials credentials, - int durationSeconds) + AwsBlobProviderConfiguration configuration, + Credentials credentials) { - var temporaryCredentialsCache = new AwsTemporaryCredentialsCacheItem(credentials.AccessKeyId, - credentials.SecretAccessKey, - credentials.SessionToken); + var temporaryCredentialsCache = new AwsTemporaryCredentialsCacheItem( + DataProtector.Protect(credentials.AccessKeyId), + DataProtector.Protect(credentials.SecretAccessKey), + DataProtector.Protect(credentials.SessionToken)); - await Cache.SetAsync(AwsTemporaryCredentialsCacheItem.Key, temporaryCredentialsCache, + await Cache.SetAsync(configuration.TemporaryCredentialsCacheKey, temporaryCredentialsCache, new DistributedCacheEntryOptions { - AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(durationSeconds - 10) + AbsoluteExpirationRelativeToNow = TimeSpan.FromSeconds(configuration.DurationSeconds - 10) }); return temporaryCredentialsCache; diff --git a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs index dc9dda1b76..d176a864e7 100644 --- a/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs +++ b/framework/test/Volo.Abp.BlobStoring.Aws.Tests/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsTestModule.cs @@ -54,6 +54,7 @@ namespace Volo.Abp.BlobStoring.Aws aws.SecretAccessKey = secretAccessKey; aws.Region = region; aws.CreateContainerIfNotExists = true; + aws.ContainerName = _randomContainerName; _configuration = aws; }); From 1aaabc7e900e79b3c0991f5048d8101c6b005563 Mon Sep 17 00:00:00 2001 From: liangshiwei Date: Wed, 22 Jul 2020 15:47:01 +0800 Subject: [PATCH 5/5] Used IStringEncryptionService to encrypt data --- .../Volo.Abp.BlobStoring.Aws.csproj | 2 -- .../Aws/AbpBlobStoringAwsModule.cs | 4 +-- .../Aws/DefaultAmazonS3ClientFactory.cs | 29 ++++++++++--------- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj index 814158b064..d30601d1e8 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo.Abp.BlobStoring.Aws.csproj @@ -19,8 +19,6 @@ - - diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs index 6f2e5dc330..28426321c8 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/AbpBlobStoringAwsModule.cs @@ -1,5 +1,4 @@ -using Microsoft.Extensions.DependencyInjection; -using Volo.Abp.Caching; +using Volo.Abp.Caching; using Volo.Abp.Modularity; namespace Volo.Abp.BlobStoring.Aws @@ -10,7 +9,6 @@ namespace Volo.Abp.BlobStoring.Aws { public override void ConfigureServices(ServiceConfigurationContext context) { - context.Services.AddDataProtection(); } } } diff --git a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs index 717ef2e05d..78ef55aebb 100644 --- a/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs +++ b/framework/src/Volo.Abp.BlobStoring.Aws/Volo/Abp/BlobStoring/Aws/DefaultAmazonS3ClientFactory.cs @@ -6,10 +6,10 @@ using Amazon.Runtime.CredentialManagement; using Amazon.S3; using Amazon.SecurityToken; using Amazon.SecurityToken.Model; -using Microsoft.AspNetCore.DataProtection; using Microsoft.Extensions.Caching.Distributed; using Volo.Abp.Caching; using Volo.Abp.DependencyInjection; +using Volo.Abp.Security.Encryption; namespace Volo.Abp.BlobStoring.Aws { @@ -17,13 +17,14 @@ namespace Volo.Abp.BlobStoring.Aws { protected IDistributedCache Cache { get; } - protected IDataProtector DataProtector { get; } + protected IStringEncryptionService StringEncryptionService { get; } - public DefaultAmazonS3ClientFactory(IDistributedCache cache, - IDataProtectionProvider dataProtectionProvider) + public DefaultAmazonS3ClientFactory( + IDistributedCache cache, + IStringEncryptionService stringEncryptionService) { Cache = cache; - DataProtector = dataProtectionProvider.CreateProtector(nameof(AwsTemporaryCredentialsCacheItem)); + StringEncryptionService = stringEncryptionService; } public virtual async Task GetAmazonS3Client( @@ -114,9 +115,9 @@ namespace Volo.Abp.BlobStoring.Aws } var sessionCredentials = new SessionAWSCredentials( - DataProtector.Unprotect(temporaryCredentialsCache.AccessKeyId), - DataProtector.Unprotect(temporaryCredentialsCache.SecretAccessKey), - DataProtector.Unprotect(temporaryCredentialsCache.SessionToken)); + StringEncryptionService.Decrypt(temporaryCredentialsCache.AccessKeyId), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SecretAccessKey), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SessionToken)); return sessionCredentials; } @@ -165,9 +166,9 @@ namespace Volo.Abp.BlobStoring.Aws } var sessionCredentials = new SessionAWSCredentials( - DataProtector.Unprotect(temporaryCredentialsCache.AccessKeyId), - DataProtector.Unprotect(temporaryCredentialsCache.SecretAccessKey), - DataProtector.Unprotect(temporaryCredentialsCache.SessionToken)); + StringEncryptionService.Decrypt(temporaryCredentialsCache.AccessKeyId), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SecretAccessKey), + StringEncryptionService.Decrypt(temporaryCredentialsCache.SessionToken)); return sessionCredentials; } @@ -176,9 +177,9 @@ namespace Volo.Abp.BlobStoring.Aws Credentials credentials) { var temporaryCredentialsCache = new AwsTemporaryCredentialsCacheItem( - DataProtector.Protect(credentials.AccessKeyId), - DataProtector.Protect(credentials.SecretAccessKey), - DataProtector.Protect(credentials.SessionToken)); + StringEncryptionService.Encrypt(credentials.AccessKeyId), + StringEncryptionService.Encrypt(credentials.SecretAccessKey), + StringEncryptionService.Encrypt(credentials.SessionToken)); await Cache.SetAsync(configuration.TemporaryCredentialsCacheKey, temporaryCredentialsCache, new DistributedCacheEntryOptions