diff --git a/Squidex.sln b/Squidex.sln index 606cec592..6be93d123 100644 --- a/Squidex.sln +++ b/Squidex.sln @@ -1,6 +1,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 -VisualStudioVersion = 15.0.26228.12 +VisualStudioVersion = 15.0.26403.3 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex", "src\Squidex\Squidex.csproj", "{61F6BBCE-A080-4400-B194-70E2F5D2096E}" EndProject @@ -34,6 +34,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Redi EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.RabbitMq", "src\Squidex.Infrastructure.RabbitMq\Squidex.Infrastructure.RabbitMq.csproj", "{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.GoogleCloud", "src\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj", "{945871B1-77B8-43FB-B53C-27CF385AB756}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -160,6 +162,18 @@ Global {C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x64.Build.0 = Release|Any CPU {C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.ActiveCfg = Release|Any CPU {C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.Build.0 = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.Build.0 = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.ActiveCfg = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.Build.0 = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.ActiveCfg = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.Build.0 = Debug|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.ActiveCfg = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.Build.0 = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.ActiveCfg = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.Build.0 = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.ActiveCfg = Release|Any CPU + {945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -178,5 +192,6 @@ Global {8B074219-F69A-4E41-83C6-12EE1E647779} = {4C6B06C2-6D77-4E0E-AE32-D7050236433A} {D7166C56-178A-4457-B56A-C615C7450DEE} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} {C1E5BBB6-6B6A-4DE5-B19D-0538304DE343} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} + {945871B1-77B8-43FB-B53C-27CF385AB756} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} EndGlobalSection EndGlobal diff --git a/src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs b/src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs new file mode 100644 index 000000000..03536e108 --- /dev/null +++ b/src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs @@ -0,0 +1,90 @@ +// ========================================================================== +// GoogleCloudAssetStore.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using Google; +using Google.Cloud.Storage.V1; +using Squidex.Infrastructure.Assets; + +namespace Squidex.Infrastructure.GoogleCloud +{ + public sealed class GoogleCloudAssetStore : IAssetStore, IExternalSystem + { + private readonly string bucketName; + private StorageClient storageClient; + + public GoogleCloudAssetStore(string bucketName) + { + Guard.NotNullOrEmpty(bucketName, nameof(bucketName)); + + this.bucketName = bucketName; + } + + public void Connect() + { + try + { + storageClient = StorageClient.Create(); + + storageClient.GetBucket(bucketName); + } + catch (Exception ex) + { + throw new ConfigurationException($"Cannot connect to google cloud bucket '${bucketName}'.", ex); + } + } + + public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream) + { + var objectName = GetObjectName(id, version, suffix); + + try + { + await storageClient.DownloadObjectAsync(bucketName, objectName, stream); + } + catch (GoogleApiException ex) + { + if (ex.HttpStatusCode == HttpStatusCode.NotFound) + { + throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + } + else + { + throw; + } + } + } + + public async Task UploadAsync(Guid id, long version, string suffix, Stream stream) + { + var objectName = GetObjectName(id, version, suffix); + + await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream); + } + + private string GetObjectName(Guid id, long version, string suffix) + { + if (storageClient == null) + { + throw new InvalidOperationException("No connection established yet."); + } + + var name = $"{id}_{version}"; + + if (!string.IsNullOrWhiteSpace(suffix)) + { + name += "_" + suffix; + } + + return name; + } + } +} diff --git a/src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj b/src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj new file mode 100644 index 000000000..e3b4db52b --- /dev/null +++ b/src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj @@ -0,0 +1,15 @@ + + + netstandard1.6 + + + full + True + + + + + + + + \ No newline at end of file diff --git a/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs b/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs new file mode 100644 index 000000000..46b6d6302 --- /dev/null +++ b/src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// AssetNotFoundException.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; + +namespace Squidex.Infrastructure.Assets +{ + public class AssetNotFoundException : Exception + { + public AssetNotFoundException() + { + } + + public AssetNotFoundException(string message) + : base(message) + { + } + + public AssetNotFoundException(string message, Exception inner) + : base(message, inner) + { + } + } +} diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 90e644c04..8bf6d9666 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -50,28 +50,24 @@ namespace Squidex.Infrastructure.Assets } } - public Task GetAssetAsync(Guid id, long version, string suffix = null) + public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream) { var file = GetFile(id, version, suffix); - Stream stream = null; - try { - if (file.Exists) + using (var fileStream = file.OpenWrite()) { - stream = file.OpenRead(); + await fileStream.CopyToAsync(stream); } } - catch (FileNotFoundException) + catch (FileNotFoundException ex) { - stream = null; + throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); } - - return Task.FromResult(stream); } - public async Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null) + public async Task UploadAsync(Guid id, long version, string suffix, Stream stream) { var file = GetFile(id, version, suffix); diff --git a/src/Squidex.Infrastructure/Assets/IAssetStore.cs b/src/Squidex.Infrastructure/Assets/IAssetStore.cs index e0926ad01..d9cc93460 100644 --- a/src/Squidex.Infrastructure/Assets/IAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/IAssetStore.cs @@ -14,8 +14,8 @@ namespace Squidex.Infrastructure.Assets { public interface IAssetStore { - Task GetAssetAsync(Guid id, long version, string suffix = null); + Task DownloadAsync(Guid id, long version, string suffix, Stream stream); - Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null); + Task UploadAsync(Guid id, long version, string suffix, Stream stream); } } \ No newline at end of file diff --git a/src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs b/src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs index a572a853a..ffe920263 100644 --- a/src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs +++ b/src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs @@ -13,8 +13,8 @@ namespace Squidex.Infrastructure.Assets { public interface IAssetThumbnailGenerator { - Task GetImageInfoAsync(Stream input); + Task GetImageInfoAsync(Stream source); - Task CreateThumbnailAsync(Stream input, int? width, int? height, string mode); + Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode); } } diff --git a/src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs b/src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs index a97e9ff58..2c66b0650 100644 --- a/src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs +++ b/src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs @@ -23,13 +23,13 @@ namespace Squidex.Infrastructure.Assets.ImageSharp Configuration.Default.AddImageFormat(new PngFormat()); } - public Task CreateThumbnailAsync(Stream input, int? width, int? height, string mode) + public Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode) { return Task.Run(() => { if (width == null && height == null) { - return input; + source.CopyTo(destination); } if (!Enum.TryParse(mode, true, out var resizeMode)) @@ -40,9 +40,7 @@ namespace Squidex.Infrastructure.Assets.ImageSharp var w = width ?? int.MaxValue; var h = height ?? int.MaxValue; - var result = new MemoryStream(); - - using (var sourceImage = Image.Load(input)) + using (var sourceImage = Image.Load(source)) { if (w >= sourceImage.Width && h >= sourceImage.Height && resizeMode == ResizeMode.Crop) { @@ -57,23 +55,19 @@ namespace Squidex.Infrastructure.Assets.ImageSharp }; sourceImage.MetaData.Quality = 0; - sourceImage.Resize(options).Save(result); + sourceImage.Resize(options).Save(destination); } - - result.Position = 0; - - return result; }); } - public Task GetImageInfoAsync(Stream input) + public Task GetImageInfoAsync(Stream source) { return Task.Run(() => { ImageInfo imageInfo = null; try { - var image = Image.Load(input); + var image = Image.Load(source); if (image.Width > 0 && image.Height > 0) { diff --git a/src/Squidex.Write/Assets/AssetCommandHandler.cs b/src/Squidex.Write/Assets/AssetCommandHandler.cs index 90d5876ff..b19977798 100644 --- a/src/Squidex.Write/Assets/AssetCommandHandler.cs +++ b/src/Squidex.Write/Assets/AssetCommandHandler.cs @@ -44,7 +44,7 @@ namespace Squidex.Write.Assets c.Create(command); - await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead()); + await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead()); context.Succeed(EntityCreatedResult.Create(c.Id, c.Version)); }); @@ -58,7 +58,7 @@ namespace Squidex.Write.Assets c.Update(command); - await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead()); + await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead()); }); } diff --git a/src/Squidex/Config/Domain/AssetStoreModule.cs b/src/Squidex/Config/Domain/AssetStoreModule.cs index 2ccd20a5c..6f6376353 100644 --- a/src/Squidex/Config/Domain/AssetStoreModule.cs +++ b/src/Squidex/Config/Domain/AssetStoreModule.cs @@ -11,6 +11,7 @@ using Autofac; using Microsoft.Extensions.Configuration; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; +using Squidex.Infrastructure.GoogleCloud; using Squidex.Infrastructure.Log; // ReSharper disable InvertIf @@ -49,9 +50,23 @@ namespace Squidex.Config.Domain .As() .SingleInstance(); } + else if (string.Equals(assetStoreType, "GoogleCloud", StringComparison.OrdinalIgnoreCase)) + { + var bucketName = Configuration.GetValue("assetStore:googleCloud:bucket"); + + if (string.IsNullOrWhiteSpace(bucketName)) + { + throw new ConfigurationException("Configure AssetStore GoogleCloud bucket with 'assetStore:googleCloud:bucket'."); + } + + builder.Register(c => new GoogleCloudAssetStore(bucketName)) + .As() + .As() + .SingleInstance(); + } else { - throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder."); + throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder, GoogleCloud."); } } } diff --git a/src/Squidex/Config/Web/WebModule.cs b/src/Squidex/Config/Web/WebModule.cs index 23e1e5083..25ed74696 100644 --- a/src/Squidex/Config/Web/WebModule.cs +++ b/src/Squidex/Config/Web/WebModule.cs @@ -28,6 +28,10 @@ namespace Squidex.Config.Web builder.RegisterType() .AsSelf() .SingleInstance(); + + builder.RegisterType() + .AsSelf() + .SingleInstance(); } } } diff --git a/src/Squidex/Controllers/Api/Assets/AssetContentController.cs b/src/Squidex/Controllers/Api/Assets/AssetContentController.cs index dd6ed5197..03e1dd947 100644 --- a/src/Squidex/Controllers/Api/Assets/AssetContentController.cs +++ b/src/Squidex/Controllers/Api/Assets/AssetContentController.cs @@ -52,41 +52,53 @@ namespace Squidex.Controllers.Api.Assets return NotFound(); } - Stream content; - - if (asset.IsImage && (width.HasValue || height.HasValue)) + return new FileCallbackResult(asset.MimeType, asset.FileName, async bodyStream => { - var suffix = $"{width}_{height}_{mode}"; - - content = await assetStorage.GetAssetAsync(asset.Id, asset.Version, suffix); - - if (content == null) + if (asset.IsImage && (width.HasValue || height.HasValue)) { - var fullSizeContent = await assetStorage.GetAssetAsync(asset.Id, asset.Version); + var suffix = $"{width}_{height}_{mode}"; - if (fullSizeContent == null) + try { - return NotFound(); + await assetStorage.DownloadAsync(asset.Id, asset.Version, suffix, bodyStream); } + catch (AssetNotFoundException) + { + using (var tempStream1 = GetTempStream()) + { + using (var tempStream2 = GetTempStream()) + { + await assetStorage.DownloadAsync(asset.Id, asset.Version, null, tempStream1); + tempStream1.Position = 0; - content = await assetThumbnailGenerator.CreateThumbnailAsync(fullSizeContent, width, height, mode); + await assetThumbnailGenerator.CreateThumbnailAsync(tempStream1, tempStream2, width, height, mode); + tempStream2.Position = 0; - await assetStorage.UploadAssetAsync(asset.Id, asset.Version, content, suffix); + await assetStorage.UploadAsync(asset.Id, asset.Version, suffix, tempStream2); + tempStream2.Position = 0; - content.Position = 0; + await tempStream2.CopyToAsync(bodyStream); + } + } + + } } - } - else - { - content = await assetStorage.GetAssetAsync(asset.Id, asset.Version); - } - if (content == null) - { - return NotFound(); - } + await assetStorage.DownloadAsync(asset.Id, asset.Version, null, bodyStream); + }); + } - return new FileStreamResult(content, asset.MimeType) { FileDownloadName = asset.FileName }; + private static FileStream GetTempStream() + { + var tempFileName = Path.GetTempFileName(); + + return new FileStream(tempFileName, + FileMode.Create, + FileAccess.ReadWrite, + FileShare.Delete, 1024 * 16, + FileOptions.Asynchronous | + FileOptions.DeleteOnClose | + FileOptions.SequentialScan); } } } diff --git a/src/Squidex/Pipeline/FileCallbackResult.cs b/src/Squidex/Pipeline/FileCallbackResult.cs new file mode 100644 index 000000000..ff88b8259 --- /dev/null +++ b/src/Squidex/Pipeline/FileCallbackResult.cs @@ -0,0 +1,43 @@ +// ========================================================================== +// FileCallbackResult.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; + +namespace Squidex.Pipeline +{ + public class FileCallbackResult : FileResult + { + private readonly Func callback; + + public Func Callback + { + get { return callback; } + } + + public FileCallbackResult(string contentType, string name, Func callback) + : base(contentType) + { + FileDownloadName = name; + + this.callback = callback; + } + + public override Task ExecuteResultAsync(ActionContext context) + { + var executor = context.HttpContext.RequestServices.GetRequiredService(); + + return executor.ExecuteAsync(context, this); + } + } +} + +#pragma warning restore 1573 \ No newline at end of file diff --git a/src/Squidex/Pipeline/FileCallbackResultExecutor.cs b/src/Squidex/Pipeline/FileCallbackResultExecutor.cs new file mode 100644 index 000000000..70068424e --- /dev/null +++ b/src/Squidex/Pipeline/FileCallbackResultExecutor.cs @@ -0,0 +1,38 @@ +// ========================================================================== +// FileCallbackResultExecutor.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Mvc.Internal; +using Microsoft.Extensions.Logging; + +namespace Squidex.Pipeline +{ + public sealed class FileCallbackResultExecutor : FileResultExecutorBase + { + public FileCallbackResultExecutor(ILoggerFactory loggerFactory) + : base(CreateLogger(loggerFactory)) + { + } + + public async Task ExecuteAsync(ActionContext context, FileCallbackResult result) + { + try + { + SetHeadersAndLog(context, result); + + await result.Callback(context.HttpContext.Response.Body); + } + catch + { + context.HttpContext.Response.Headers.Clear(); + context.HttpContext.Response.StatusCode = 404; + } + } + } +} \ No newline at end of file diff --git a/src/Squidex/Squidex.csproj b/src/Squidex/Squidex.csproj index 336c1e2c4..c2a6313b6 100644 --- a/src/Squidex/Squidex.csproj +++ b/src/Squidex/Squidex.csproj @@ -15,6 +15,10 @@ + + + + PreserveNewest @@ -23,6 +27,7 @@ + diff --git a/src/Squidex/appsettings.json b/src/Squidex/appsettings.json index 0aa40630b..ce1cdc1fd 100644 --- a/src/Squidex/appsettings.json +++ b/src/Squidex/appsettings.json @@ -14,8 +14,11 @@ "assetStore": { "type": "Folder", "folder": { - "path": "Assets" - } + "path": "Assets" + }, + "googleCloud": { + "bucket": "squidex-assets" + } }, "eventStore": { "type": "MongoDb", diff --git a/tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs b/tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs index 18a770f2d..1a2ec8de3 100644 --- a/tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs +++ b/tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs @@ -44,7 +44,7 @@ namespace Squidex.Write.Assets [Fact] public async Task Create_should_create_asset() { - assetStore.Setup(x => x.UploadAssetAsync(assetId, 0, stream, null)).Returns(TaskHelper.Done).Verifiable(); + assetStore.Setup(x => x.UploadAsync(assetId, 0, null, stream)).Returns(TaskHelper.Done).Verifiable(); assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable(); var context = CreateContextForCommand(new CreateAsset { AssetId = assetId, File = file }); @@ -63,7 +63,7 @@ namespace Squidex.Write.Assets [Fact] public async Task Update_should_update_domain_object() { - assetStore.Setup(x => x.UploadAssetAsync(assetId, 1, stream, null)).Returns(TaskHelper.Done).Verifiable(); + assetStore.Setup(x => x.UploadAsync(assetId, 1, null, stream)).Returns(TaskHelper.Done).Verifiable(); assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable(); CreateAsset();