Google Cloud File Storage

Squidex.sln

@@ -1,6 +1,6 @@
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26228.12
+VisualStudioVersion = 15.0.26403.3
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex", "src\Squidex\Squidex.csproj", "{61F6BBCE-A080-4400-B194-70E2F5D2096E}"
 EndProject
@@ -34,6 +34,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Redi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.RabbitMq", "src\Squidex.Infrastructure.RabbitMq\Squidex.Infrastructure.RabbitMq.csproj", "{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.GoogleCloud", "src\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj", "{945871B1-77B8-43FB-B53C-27CF385AB756}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -160,6 +162,18 @@ Global
 		{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x64.Build.0 = Release|Any CPU
 		{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.ActiveCfg = Release|Any CPU
 		{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.Build.0 = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.Build.0 = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.Build.0 = Debug|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.Build.0 = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.ActiveCfg = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.Build.0 = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.ActiveCfg = Release|Any CPU
+		{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -178,5 +192,6 @@ Global
 		{8B074219-F69A-4E41-83C6-12EE1E647779} = {4C6B06C2-6D77-4E0E-AE32-D7050236433A}
 		{D7166C56-178A-4457-B56A-C615C7450DEE} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
 		{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
+		{945871B1-77B8-43FB-B53C-27CF385AB756} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
 	EndGlobalSection
 EndGlobal

src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs

@@ -0,0 +1,90 @@
+// ==========================================================================
+//  GoogleCloudAssetStore.cs
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex Group
+//  All rights reserved.
+// ==========================================================================
+
+using System;
+using System.IO;
+using System.Net;
+using System.Threading.Tasks;
+using Google;
+using Google.Cloud.Storage.V1;
+using Squidex.Infrastructure.Assets;
+
+namespace Squidex.Infrastructure.GoogleCloud
+{
+    public sealed class GoogleCloudAssetStore : IAssetStore, IExternalSystem
+    {
+        private readonly string bucketName;
+        private StorageClient storageClient;
+
+        public GoogleCloudAssetStore(string bucketName)
+        {
+            Guard.NotNullOrEmpty(bucketName, nameof(bucketName));
+
+            this.bucketName = bucketName;
+        }
+
+        public void Connect()
+        {
+            try
+            {
+                storageClient = StorageClient.Create();
+
+                storageClient.GetBucket(bucketName);
+            }
+            catch (Exception ex)
+            {
+                throw new ConfigurationException($"Cannot connect to google cloud bucket '${bucketName}'.", ex);
+            }
+        }
+
+        public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream)
+        {
+            var objectName = GetObjectName(id, version, suffix);
+
+            try
+            {
+                await storageClient.DownloadObjectAsync(bucketName, objectName, stream);
+            }
+            catch (GoogleApiException ex)
+            {
+                if (ex.HttpStatusCode == HttpStatusCode.NotFound)
+                {
+                    throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
+                }
+                else
+                {
+                    throw;
+                }
+            }
+        }
+
+        public async Task UploadAsync(Guid id, long version, string suffix, Stream stream)
+        {
+            var objectName = GetObjectName(id, version, suffix);
+
+            await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream);
+        }
+
+        private string GetObjectName(Guid id, long version, string suffix)
+        {
+            if (storageClient == null)
+            {
+                throw new InvalidOperationException("No connection established yet.");
+            }
+
+            var name = $"{id}_{version}";
+
+            if (!string.IsNullOrWhiteSpace(suffix))
+            {
+                name += "_" + suffix;
+            }
+
+            return name;
+        }
+    }
+}

src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard1.6</TargetFramework>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DebugType>full</DebugType>
+    <DebugSymbols>True</DebugSymbols>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Google.Cloud.Storage.V1" Version="1.0.0" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
+  </ItemGroup>
+</Project>

src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs

@@ -0,0 +1,29 @@
+// ==========================================================================
+//  AssetNotFoundException.cs
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex Group
+//  All rights reserved.
+// ==========================================================================
+
+using System;
+
+namespace Squidex.Infrastructure.Assets
+{
+    public class AssetNotFoundException : Exception
+    {
+        public AssetNotFoundException()
+        {
+        }
+
+        public AssetNotFoundException(string message) 
+            : base(message)
+        {
+        }
+
+        public AssetNotFoundException(string message, Exception inner) 
+            : base(message, inner)
+        {
+        }
+    }
+}

src/Squidex.Infrastructure/Assets/FolderAssetStore.cs

@@ -50,28 +50,24 @@ namespace Squidex.Infrastructure.Assets
             }
         }
 
-        public Task<Stream> GetAssetAsync(Guid id, long version, string suffix = null)
+        public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream)
         {
             var file = GetFile(id, version, suffix);
 
-            Stream stream = null;
-
             try
             {
-                if (file.Exists)
+                using (var fileStream = file.OpenWrite())
                 {
-                    stream = file.OpenRead();
+                    await fileStream.CopyToAsync(stream);
                 }
             }
-            catch (FileNotFoundException)
+            catch (FileNotFoundException ex)
             {
-                stream = null;
+                throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
             }
-
-            return Task.FromResult(stream);
         }
 
-        public async Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null)
+        public async Task UploadAsync(Guid id, long version, string suffix, Stream stream)
         {
             var file = GetFile(id, version, suffix);
 

src/Squidex.Infrastructure/Assets/IAssetStore.cs

@@ -14,8 +14,8 @@ namespace Squidex.Infrastructure.Assets
 {
     public interface IAssetStore
     {
-        Task<Stream> GetAssetAsync(Guid id, long version, string suffix = null);
+        Task DownloadAsync(Guid id, long version, string suffix, Stream stream);
 
-        Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null);
+        Task UploadAsync(Guid id, long version, string suffix, Stream stream);
     }
 }

src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs

@@ -13,8 +13,8 @@ namespace Squidex.Infrastructure.Assets
 {
     public interface IAssetThumbnailGenerator
     {
-        Task<ImageInfo> GetImageInfoAsync(Stream input);
+        Task<ImageInfo> GetImageInfoAsync(Stream source);
 
-        Task<Stream> CreateThumbnailAsync(Stream input, int? width, int? height, string mode);
+        Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode);
     }
 }

src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs

@@ -23,13 +23,13 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
             Configuration.Default.AddImageFormat(new PngFormat());
         }
 
-        public Task<Stream> CreateThumbnailAsync(Stream input, int? width, int? height, string mode)
+        public Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode)
         {
             return Task.Run(() =>
             {
                 if (width == null && height == null)
                 {
-                    return input;
+                    source.CopyTo(destination);
                 }
 
                 if (!Enum.TryParse<ResizeMode>(mode, true, out var resizeMode))
@@ -40,9 +40,7 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
                 var w = width ?? int.MaxValue;
                 var h = height ?? int.MaxValue;
 
-                var result = new MemoryStream();
+                using (var sourceImage = Image.Load(source))
-
-                using (var sourceImage = Image.Load(input))
                 {
                     if (w >= sourceImage.Width && h >= sourceImage.Height && resizeMode == ResizeMode.Crop)
                     {
@@ -57,23 +55,19 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
                         };
 
                     sourceImage.MetaData.Quality = 0;
-                    sourceImage.Resize(options).Save(result);
+                    sourceImage.Resize(options).Save(destination);
                 }
-
-                result.Position = 0;
-
-                return result;
             });
         }
 
-        public Task<ImageInfo> GetImageInfoAsync(Stream input)
+        public Task<ImageInfo> GetImageInfoAsync(Stream source)
         {
             return Task.Run(() =>
             {
                 ImageInfo imageInfo = null;
                 try
                 {
-                    var image = Image.Load(input);
+                    var image = Image.Load(source);
 
                     if (image.Width > 0 && image.Height > 0)
                     {

src/Squidex.Write/Assets/AssetCommandHandler.cs

@@ -44,7 +44,7 @@ namespace Squidex.Write.Assets
 
                 c.Create(command);
 
-                await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead());
+                await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead());
 
                 context.Succeed(EntityCreatedResult.Create(c.Id, c.Version));
             });
@@ -58,7 +58,7 @@ namespace Squidex.Write.Assets
 
                 c.Update(command);
 
-                await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead());
+                await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead());
             });
         }
 

src/Squidex/Config/Domain/AssetStoreModule.cs

@@ -11,6 +11,7 @@ using Autofac;
 using Microsoft.Extensions.Configuration;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Assets;
+using Squidex.Infrastructure.GoogleCloud;
 using Squidex.Infrastructure.Log;
 
 // ReSharper disable InvertIf
@@ -49,9 +50,23 @@ namespace Squidex.Config.Domain
                     .As<IExternalSystem>()
                     .SingleInstance();
             }
+            else if (string.Equals(assetStoreType, "GoogleCloud", StringComparison.OrdinalIgnoreCase))
+            {
+                var bucketName = Configuration.GetValue<string>("assetStore:googleCloud:bucket");
+
+                if (string.IsNullOrWhiteSpace(bucketName))
+                {
+                    throw new ConfigurationException("Configure AssetStore GoogleCloud bucket with 'assetStore:googleCloud:bucket'.");
+                }
+
+                builder.Register(c => new GoogleCloudAssetStore(bucketName))
+                    .As<IAssetStore>()
+                    .As<IExternalSystem>()
+                    .SingleInstance();
+            }
             else
             {
-                throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder.");
+                throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder, GoogleCloud.");
             }
         }
     }

src/Squidex/Config/Web/WebModule.cs

@@ -28,6 +28,10 @@ namespace Squidex.Config.Web
             builder.RegisterType<AppFilterAttribute>()
                 .AsSelf()
                 .SingleInstance();
+
+            builder.RegisterType<FileCallbackResultExecutor>()
+                .AsSelf()
+                .SingleInstance();
         }
     }
 }

src/Squidex/Controllers/Api/Assets/AssetContentController.cs

@@ -52,41 +52,53 @@ namespace Squidex.Controllers.Api.Assets
                 return NotFound();
             }
 
-            Stream content;
+            return new FileCallbackResult(asset.MimeType, asset.FileName, async bodyStream =>
-
-            if (asset.IsImage && (width.HasValue || height.HasValue))
             {
-                var suffix = $"{width}_{height}_{mode}";
+                if (asset.IsImage && (width.HasValue || height.HasValue))
-
-                content = await assetStorage.GetAssetAsync(asset.Id, asset.Version, suffix);
-
-                if (content == null)
                 {
-                    var fullSizeContent = await assetStorage.GetAssetAsync(asset.Id, asset.Version);
+                    var suffix = $"{width}_{height}_{mode}";
 
-                    if (fullSizeContent == null)
+                    try
                     {
-                        return NotFound();
+                        await assetStorage.DownloadAsync(asset.Id, asset.Version, suffix, bodyStream);
                     }
+                    catch (AssetNotFoundException)
+                    {
+                        using (var tempStream1 = GetTempStream())
+                        {
+                            using (var tempStream2 = GetTempStream())
+                            {
+                                await assetStorage.DownloadAsync(asset.Id, asset.Version, null, tempStream1);
+                                tempStream1.Position = 0;
 
-                    content = await assetThumbnailGenerator.CreateThumbnailAsync(fullSizeContent, width, height, mode);
+                                await assetThumbnailGenerator.CreateThumbnailAsync(tempStream1, tempStream2, width, height, mode);
+                                tempStream2.Position = 0;
 
-                    await assetStorage.UploadAssetAsync(asset.Id, asset.Version, content, suffix);
+                                await assetStorage.UploadAsync(asset.Id, asset.Version, suffix, tempStream2);
+                                tempStream2.Position = 0;
 
-                    content.Position = 0;
+                                await tempStream2.CopyToAsync(bodyStream);
+                            }
+                        }
+
+                    }
                 }
-            }
-            else
-            {
-                content = await assetStorage.GetAssetAsync(asset.Id, asset.Version);
-            }
 
-            if (content == null)
+                await assetStorage.DownloadAsync(asset.Id, asset.Version, null, bodyStream);
-            {
+            });
-                return NotFound();
+        }
-            }
 
-            return new FileStreamResult(content, asset.MimeType) { FileDownloadName = asset.FileName };
+        private static FileStream GetTempStream()
+        {
+            var tempFileName = Path.GetTempFileName();
+
+            return new FileStream(tempFileName,
+                FileMode.Create,
+                FileAccess.ReadWrite,
+                FileShare.Delete, 1024 * 16,
+                FileOptions.Asynchronous |
+                FileOptions.DeleteOnClose |
+                FileOptions.SequentialScan);
         }
     }
 }

src/Squidex/Pipeline/FileCallbackResult.cs

@@ -0,0 +1,43 @@
+// ==========================================================================
+//  FileCallbackResult.cs
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex Group
+//  All rights reserved.
+// ==========================================================================
+
+using System;
+using System.IO;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace Squidex.Pipeline
+{
+    public class FileCallbackResult : FileResult
+    {
+        private readonly Func<Stream, Task> callback;
+
+        public Func<Stream, Task> Callback
+        {
+            get { return callback; }
+        }
+
+        public FileCallbackResult(string contentType, string name, Func<Stream, Task> callback)
+            : base(contentType)
+        {
+            FileDownloadName = name;
+
+            this.callback = callback;
+        }
+
+        public override Task ExecuteResultAsync(ActionContext context)
+        {
+            var executor = context.HttpContext.RequestServices.GetRequiredService<FileCallbackResultExecutor>();
+
+            return executor.ExecuteAsync(context, this);
+        }
+    }
+}
+
+#pragma warning restore 1573

src/Squidex/Pipeline/FileCallbackResultExecutor.cs

@@ -0,0 +1,38 @@
+// ==========================================================================
+//  FileCallbackResultExecutor.cs
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex Group
+//  All rights reserved.
+// ==========================================================================
+
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.Internal;
+using Microsoft.Extensions.Logging;
+
+namespace Squidex.Pipeline
+{
+    public sealed class FileCallbackResultExecutor : FileResultExecutorBase
+    {
+        public FileCallbackResultExecutor(ILoggerFactory loggerFactory)
+            : base(CreateLogger<VirtualFileResultExecutor>(loggerFactory))
+        {
+        }
+
+        public async Task ExecuteAsync(ActionContext context, FileCallbackResult result)
+        {
+            try
+            {
+                SetHeadersAndLog(context, result);
+
+                await result.Callback(context.HttpContext.Response.Body);
+            }
+            catch
+            {
+                context.HttpContext.Response.Headers.Clear();
+                context.HttpContext.Response.StatusCode = 404;
+            }
+        }
+    }
+}

src/Squidex/Squidex.csproj

@@ -15,6 +15,10 @@
 
   <ItemGroup>
     <EmbeddedResource Include="Config\Identity\Cert\*.*;Docs\*.md" />
+    <EmbeddedResource Remove="Assets\**" />
+    <Compile Remove="Assets\**" />
+    <Content Remove="Assets\**" />
+    <None Remove="Assets\**" />
 	  <None Update="dockerfile">
       <CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
     </None>
@@ -23,6 +27,7 @@
   <ItemGroup>
     <ProjectReference Include="..\Squidex.Core\Squidex.Core.csproj" />
     <ProjectReference Include="..\Squidex.Events\Squidex.Events.csproj" />
+    <ProjectReference Include="..\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj" />
     <ProjectReference Include="..\Squidex.Infrastructure.RabbitMq\Squidex.Infrastructure.RabbitMq.csproj" />
     <ProjectReference Include="..\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
     <ProjectReference Include="..\Squidex.Infrastructure.MongoDb\Squidex.Infrastructure.MongoDb.csproj" />

src/Squidex/appsettings.json

@@ -14,8 +14,11 @@
   "assetStore": {
     "type": "Folder",
     "folder": {
-      "path": "Assets" 
+      "path": "Assets"
-    } 
+    },
+    "googleCloud": {
+      "bucket": "squidex-assets"
+    }
   },
   "eventStore": {
     "type": "MongoDb",

tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs

@@ -44,7 +44,7 @@ namespace Squidex.Write.Assets
         [Fact]
         public async Task Create_should_create_asset()
         {
-            assetStore.Setup(x => x.UploadAssetAsync(assetId, 0, stream, null)).Returns(TaskHelper.Done).Verifiable();
+            assetStore.Setup(x => x.UploadAsync(assetId, 0, null, stream)).Returns(TaskHelper.Done).Verifiable();
             assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable();
 
             var context = CreateContextForCommand(new CreateAsset { AssetId = assetId, File = file });
@@ -63,7 +63,7 @@ namespace Squidex.Write.Assets
         [Fact]
         public async Task Update_should_update_domain_object()
         {
-            assetStore.Setup(x => x.UploadAssetAsync(assetId, 1, stream, null)).Returns(TaskHelper.Done).Verifiable();
+            assetStore.Setup(x => x.UploadAsync(assetId, 1, null, stream)).Returns(TaskHelper.Done).Verifiable();
             assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable();
 
             CreateAsset();