From 3bcaf829d5eb9f367291b3c8c07618785b37ed15 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Wed, 13 Nov 2019 16:28:39 +0100 Subject: [PATCH] Feature/s3 (#446) Amazon S3 for assets. --- backend/Squidex.sln | 15 ++ .../Schemas/FieldExtensions.cs | 5 - .../Assets/AmazonS3AssetStore.cs | 202 ++++++++++++++++++ .../Squidex.Infrastructure.Amazon.csproj | 22 ++ .../Squidex/Config/Domain/AssetServices.cs | 13 ++ backend/src/Squidex/Squidex.csproj | 1 + backend/src/Squidex/appsettings.json | 32 ++- .../Assets/AmazonS3AssetStoreFixture.cs | 20 ++ .../Assets/AmazonS3AssetStoreTests.cs | 44 ++++ .../Squidex.Infrastructure.Tests.csproj | 1 + 10 files changed, 349 insertions(+), 6 deletions(-) create mode 100644 backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs create mode 100644 backend/src/Squidex.Infrastructure.Amazon/Squidex.Infrastructure.Amazon.csproj create mode 100644 backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreFixture.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreTests.cs diff --git a/backend/Squidex.sln b/backend/Squidex.sln index c6332f516..88fd5bb3e 100644 --- a/backend/Squidex.sln +++ b/backend/Squidex.sln @@ -65,6 +65,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{7EDE8C EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Web", "src\Squidex.Web\Squidex.Web.csproj", "{5B2D251F-46E3-486A-AE16-E3FE06B559ED}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Amazon", "src\Squidex.Infrastructure.Amazon\Squidex.Infrastructure.Amazon.csproj", "{32DA4B56-7EFA-4E34-A29D-30E00579A894}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -339,6 +341,18 @@ Global {5B2D251F-46E3-486A-AE16-E3FE06B559ED}.Release|x64.Build.0 = Release|Any CPU {5B2D251F-46E3-486A-AE16-E3FE06B559ED}.Release|x86.ActiveCfg = Release|Any CPU {5B2D251F-46E3-486A-AE16-E3FE06B559ED}.Release|x86.Build.0 = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|Any CPU.Build.0 = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|x64.ActiveCfg = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|x64.Build.0 = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|x86.ActiveCfg = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Debug|x86.Build.0 = Debug|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|Any CPU.ActiveCfg = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|Any CPU.Build.0 = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x64.ActiveCfg = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x64.Build.0 = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.ActiveCfg = Release|Any CPU + {32DA4B56-7EFA-4E34-A29D-30E00579A894}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -367,6 +381,7 @@ Global {7E8CC864-4C6E-496F-A672-9F9AD8874835} = {7EDE8CF1-B1E4-4005-B154-834B944E0D7A} {F3C41B82-6A67-409A-B7FE-54543EE4F38B} = {FB8BC3A2-2010-4C3C-A87D-D4A98C05EE52} {5B2D251F-46E3-486A-AE16-E3FE06B559ED} = {7EDE8CF1-B1E4-4005-B154-834B944E0D7A} + {32DA4B56-7EFA-4E34-A29D-30E00579A894} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {02F2E872-3141-44F5-BD6A-33CD84E9FE08} diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldExtensions.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldExtensions.cs index 12cd23788..ab55f6a08 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Schemas/FieldExtensions.cs @@ -19,11 +19,6 @@ namespace Squidex.Domain.Apps.Core.Schemas return NamedIdStatic.Of(field.Id, field.Name); } - public static IEnumerable NonHidden(this FieldCollection fields, bool withHidden = false) where T : IField - { - return fields.Ordered.ForApi(withHidden); - } - public static IEnumerable ForApi(this IEnumerable fields, bool withHidden = false) where T : IField { return fields.Where(x => IsForApi(x, withHidden)); diff --git a/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs b/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs new file mode 100644 index 000000000..52dfe7cb2 --- /dev/null +++ b/backend/src/Squidex.Infrastructure.Amazon/Assets/AmazonS3AssetStore.cs @@ -0,0 +1,202 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.IO; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Amazon; +using Amazon.S3; +using Amazon.S3.Model; +using Amazon.S3.Transfer; + +namespace Squidex.Infrastructure.Assets +{ + public sealed class AmazonS3AssetStore : DisposableObjectBase, IAssetStore, IInitializable + { + private const int BufferSize = 81920; + private readonly string accessKey; + private readonly string secretKey; + private readonly string bucketName; + private readonly string? bucketFolder; + private readonly RegionEndpoint bucketRegion; + private TransferUtility transferUtility; + private IAmazonS3 s3Client; + + public AmazonS3AssetStore(string regionName, string bucketName, string? bucketFolder, string accessKey, string secretKey) + { + Guard.NotNullOrEmpty(bucketName); + Guard.NotNullOrEmpty(accessKey); + Guard.NotNullOrEmpty(secretKey); + + this.bucketName = bucketName; + this.bucketFolder = bucketFolder; + this.accessKey = accessKey; + this.secretKey = secretKey; + + bucketRegion = RegionEndpoint.GetBySystemName(regionName); + } + + protected override void DisposeObject(bool disposing) + { + if (disposing) + { + s3Client?.Dispose(); + + transferUtility?.Dispose(); + } + } + + public async Task InitializeAsync(CancellationToken ct = default) + { + try + { + s3Client = new AmazonS3Client( + accessKey, + secretKey, + bucketRegion); + + transferUtility = new TransferUtility(s3Client); + + var exists = await s3Client.DoesS3BucketExistAsync(bucketName); + + if (!exists) + { + throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '${bucketName}'."); + } + } + catch (AmazonS3Exception ex) + { + throw new ConfigurationException($"Cannot connect to Amazon S3 bucket '${bucketName}'.", ex); + } + } + + public string? GeneratePublicUrl(string fileName) + { + return null; + } + + public async Task CopyAsync(string sourceFileName, string targetFileName, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(sourceFileName); + Guard.NotNullOrEmpty(targetFileName); + + try + { + await EnsureNotExistsAsync(targetFileName, ct); + + var request = new CopyObjectRequest + { + SourceBucket = bucketName, + SourceKey = GetKey(sourceFileName), + DestinationBucket = bucketName, + DestinationKey = GetKey(targetFileName) + }; + + await s3Client.CopyObjectAsync(request, ct); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + throw new AssetNotFoundException(sourceFileName, ex); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) + { + throw new AssetAlreadyExistsException(targetFileName); + } + } + + public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(fileName); + Guard.NotNull(stream); + + try + { + var request = new GetObjectRequest { BucketName = bucketName, Key = GetKey(fileName) }; + + using (var response = await s3Client.GetObjectAsync(request, ct)) + { + await response.ResponseStream.CopyToAsync(stream, BufferSize, ct); + } + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + throw new AssetNotFoundException(fileName, ex); + } + } + + public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(fileName); + Guard.NotNull(stream); + + try + { + if (!overwrite) + { + await EnsureNotExistsAsync(fileName, ct); + } + + var request = new TransferUtilityUploadRequest + { + AutoCloseStream = false, + BucketName = bucketName, + InputStream = stream, + Key = GetKey(fileName) + }; + + await transferUtility.UploadAsync(request, ct); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.PreconditionFailed) + { + throw new AssetAlreadyExistsException(fileName); + } + } + + public async Task DeleteAsync(string fileName) + { + Guard.NotNullOrEmpty(fileName); + + try + { + var request = new DeleteObjectRequest { BucketName = bucketName, Key = fileName }; + + await s3Client.DeleteObjectAsync(request); + } + catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + return; + } + } + + private string GetKey(string fileName) + { + if (!string.IsNullOrWhiteSpace(bucketFolder)) + { + return $"{bucketFolder}/{fileName}"; + } + else + { + return fileName; + } + } + + private async Task EnsureNotExistsAsync(string fileName, CancellationToken ct) + { + try + { + await s3Client.GetObjectAsync(bucketName, GetKey(fileName), ct); + } + catch + { + return; + } + + throw new AssetAlreadyExistsException(fileName); + } + } +} \ No newline at end of file diff --git a/backend/src/Squidex.Infrastructure.Amazon/Squidex.Infrastructure.Amazon.csproj b/backend/src/Squidex.Infrastructure.Amazon/Squidex.Infrastructure.Amazon.csproj new file mode 100644 index 000000000..ea1f31f54 --- /dev/null +++ b/backend/src/Squidex.Infrastructure.Amazon/Squidex.Infrastructure.Amazon.csproj @@ -0,0 +1,22 @@ + + + netcoreapp3.0 + Squidex.Infrastructure + 8.0 + enable + + + + + + + + + + + ..\..\Squidex.ruleset + + + + + \ No newline at end of file diff --git a/backend/src/Squidex/Config/Domain/AssetServices.cs b/backend/src/Squidex/Config/Domain/AssetServices.cs index 2e9251998..8d05b8b67 100644 --- a/backend/src/Squidex/Config/Domain/AssetServices.cs +++ b/backend/src/Squidex/Config/Domain/AssetServices.cs @@ -83,6 +83,19 @@ namespace Squidex.Config.Domain services.AddSingletonAs(c => new AzureBlobAssetStore(connectionString, containerName)) .As(); }, + ["AmazonS3"] = () => + { + var regionName = config.GetRequiredValue("assetStore:amazonS3:regionName"); + + var bucketName = config.GetRequiredValue("assetStore:amazonS3:bucket"); + var bucketFolder = config.GetRequiredValue("assetStore:amazonS3:bucketFolder"); + + var accessKey = config.GetRequiredValue("assetStore:amazonS3:accessKey"); + var secretKey = config.GetRequiredValue("assetStore:amazonS3:secretKey"); + + services.AddSingletonAs(c => new AmazonS3AssetStore(regionName, bucketName, bucketFolder, accessKey, secretKey)) + .As(); + }, ["MongoDb"] = () => { var mongoConfiguration = config.GetRequiredValue("assetStore:mongoDb:configuration"); diff --git a/backend/src/Squidex/Squidex.csproj b/backend/src/Squidex/Squidex.csproj index 241dc5da1..7598abc1f 100644 --- a/backend/src/Squidex/Squidex.csproj +++ b/backend/src/Squidex/Squidex.csproj @@ -22,6 +22,7 @@ + diff --git a/backend/src/Squidex/appsettings.json b/backend/src/Squidex/appsettings.json index f6156d974..c93152858 100644 --- a/backend/src/Squidex/appsettings.json +++ b/backend/src/Squidex/appsettings.json @@ -239,7 +239,7 @@ /* * Define the type of the read store. * - * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob, FTP (not recommended). + * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob, AmazonS3, FTP (not recommended). */ "type": "Folder", "folder": { @@ -264,6 +264,36 @@ */ "connectionString": "UseDevelopmentStorage=true" }, + "AmazonS3": { + /* + * The name of your bucket. + */ + "bucketName": "squidex-assets", + + /* + * The optional folder within the bucket. + */ + "bucketFolder": "squidex-assets", + + /* + * The region name of your bucket. + */ + "regionName": "eu-central-1", + + /* + * The access key for your user. + * + * Read More: https://supsystic.com/documentation/id-secret-access-key-amazon-s3/ + */ + "accessKey": "", + + /* + * The secret key for your user. + * + * Read More: https://supsystic.com/documentation/id-secret-access-key-amazon-s3/ + */ + "secretKey": "" + }, "mongoDb": { /* * The connection string to your Mongo Server. diff --git a/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreFixture.cs b/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreFixture.cs new file mode 100644 index 000000000..3e3fcec04 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreFixture.cs @@ -0,0 +1,20 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Infrastructure.Assets +{ + public sealed class AmazonS3AssetStoreFixture + { + public AmazonS3AssetStore AssetStore { get; } + + public AmazonS3AssetStoreFixture() + { + AssetStore = new AmazonS3AssetStore("eu-central-1", "squidex-test", "squidex-assets", "secret", "secret"); + AssetStore.InitializeAsync().Wait(); + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreTests.cs new file mode 100644 index 000000000..454f4d1e5 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/Assets/AmazonS3AssetStoreTests.cs @@ -0,0 +1,44 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Xunit; + +namespace Squidex.Infrastructure.Assets +{ + [Trait("Category", "Dependencies")] + public class AmazonS3AssetStoreTests : AssetStoreTests, IClassFixture + { + private readonly AmazonS3AssetStoreFixture fixture; + + public AmazonS3AssetStoreTests(AmazonS3AssetStoreFixture fixture) + { + this.fixture = fixture; + } + + public override AmazonS3AssetStore CreateStore() + { + return fixture.AssetStore; + } + + [Fact] + public async Task Should_throw_exception_for_invalid_config() + { + var sut = new AmazonS3AssetStore("invalid", "invalid", null, "invalid", "invalid"); + + await Assert.ThrowsAsync(() => sut.InitializeAsync()); + } + + [Fact] + public void Should_calculate_source_url() + { + var url = Sut.GeneratePublicUrl(FileName); + + Assert.Null(url); + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj b/backend/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj index 0aa3172b0..7f8108d7e 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj +++ b/backend/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj @@ -11,6 +11,7 @@ +