Browse Source

Google Cloud File Storage

pull/65/head
Sebastian Stehle 9 years ago
parent
commit
546caf627d
  1. 17
      Squidex.sln
  2. 90
      src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs
  3. 15
      src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj
  4. 29
      src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs
  5. 16
      src/Squidex.Infrastructure/Assets/FolderAssetStore.cs
  6. 4
      src/Squidex.Infrastructure/Assets/IAssetStore.cs
  7. 4
      src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs
  8. 18
      src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs
  9. 4
      src/Squidex.Write/Assets/AssetCommandHandler.cs
  10. 17
      src/Squidex/Config/Domain/AssetStoreModule.cs
  11. 4
      src/Squidex/Config/Web/WebModule.cs
  12. 60
      src/Squidex/Controllers/Api/Assets/AssetContentController.cs
  13. 43
      src/Squidex/Pipeline/FileCallbackResult.cs
  14. 38
      src/Squidex/Pipeline/FileCallbackResultExecutor.cs
  15. 5
      src/Squidex/Squidex.csproj
  16. 7
      src/Squidex/appsettings.json
  17. 4
      tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs

17
Squidex.sln

@ -1,6 +1,6 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26228.12
VisualStudioVersion = 15.0.26403.3
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex", "src\Squidex\Squidex.csproj", "{61F6BBCE-A080-4400-B194-70E2F5D2096E}"
EndProject
@ -34,6 +34,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.Redi
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.RabbitMq", "src\Squidex.Infrastructure.RabbitMq\Squidex.Infrastructure.RabbitMq.csproj", "{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.GoogleCloud", "src\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj", "{945871B1-77B8-43FB-B53C-27CF385AB756}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -160,6 +162,18 @@ Global
{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x64.Build.0 = Release|Any CPU
{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.ActiveCfg = Release|Any CPU
{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343}.Release|x86.Build.0 = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|Any CPU.Build.0 = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.ActiveCfg = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x64.Build.0 = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.ActiveCfg = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Debug|x86.Build.0 = Debug|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.ActiveCfg = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|Any CPU.Build.0 = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.ActiveCfg = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x64.Build.0 = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.ActiveCfg = Release|Any CPU
{945871B1-77B8-43FB-B53C-27CF385AB756}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
@ -178,5 +192,6 @@ Global
{8B074219-F69A-4E41-83C6-12EE1E647779} = {4C6B06C2-6D77-4E0E-AE32-D7050236433A}
{D7166C56-178A-4457-B56A-C615C7450DEE} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
{C1E5BBB6-6B6A-4DE5-B19D-0538304DE343} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
{945871B1-77B8-43FB-B53C-27CF385AB756} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF}
EndGlobalSection
EndGlobal

90
src/Squidex.Infrastructure.GoogleCloud/GoogleCloudAssetStore.cs

@ -0,0 +1,90 @@
// ==========================================================================
// GoogleCloudAssetStore.cs
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex Group
// All rights reserved.
// ==========================================================================
using System;
using System.IO;
using System.Net;
using System.Threading.Tasks;
using Google;
using Google.Cloud.Storage.V1;
using Squidex.Infrastructure.Assets;
namespace Squidex.Infrastructure.GoogleCloud
{
public sealed class GoogleCloudAssetStore : IAssetStore, IExternalSystem
{
private readonly string bucketName;
private StorageClient storageClient;
public GoogleCloudAssetStore(string bucketName)
{
Guard.NotNullOrEmpty(bucketName, nameof(bucketName));
this.bucketName = bucketName;
}
public void Connect()
{
try
{
storageClient = StorageClient.Create();
storageClient.GetBucket(bucketName);
}
catch (Exception ex)
{
throw new ConfigurationException($"Cannot connect to google cloud bucket '${bucketName}'.", ex);
}
}
public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream)
{
var objectName = GetObjectName(id, version, suffix);
try
{
await storageClient.DownloadObjectAsync(bucketName, objectName, stream);
}
catch (GoogleApiException ex)
{
if (ex.HttpStatusCode == HttpStatusCode.NotFound)
{
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
}
else
{
throw;
}
}
}
public async Task UploadAsync(Guid id, long version, string suffix, Stream stream)
{
var objectName = GetObjectName(id, version, suffix);
await storageClient.UploadObjectAsync(bucketName, objectName, "application/octet-stream", stream);
}
private string GetObjectName(Guid id, long version, string suffix)
{
if (storageClient == null)
{
throw new InvalidOperationException("No connection established yet.");
}
var name = $"{id}_{version}";
if (!string.IsNullOrWhiteSpace(suffix))
{
name += "_" + suffix;
}
return name;
}
}
}

15
src/Squidex.Infrastructure.GoogleCloud/Squidex.Infrastructure.GoogleCloud.csproj

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netstandard1.6</TargetFramework>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
<DebugType>full</DebugType>
<DebugSymbols>True</DebugSymbols>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Google.Cloud.Storage.V1" Version="1.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
</ItemGroup>
</Project>

29
src/Squidex.Infrastructure/Assets/AssetNotFoundException.cs

@ -0,0 +1,29 @@
// ==========================================================================
// AssetNotFoundException.cs
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex Group
// All rights reserved.
// ==========================================================================
using System;
namespace Squidex.Infrastructure.Assets
{
public class AssetNotFoundException : Exception
{
public AssetNotFoundException()
{
}
public AssetNotFoundException(string message)
: base(message)
{
}
public AssetNotFoundException(string message, Exception inner)
: base(message, inner)
{
}
}
}

16
src/Squidex.Infrastructure/Assets/FolderAssetStore.cs

@ -50,28 +50,24 @@ namespace Squidex.Infrastructure.Assets
}
}
public Task<Stream> GetAssetAsync(Guid id, long version, string suffix = null)
public async Task DownloadAsync(Guid id, long version, string suffix, Stream stream)
{
var file = GetFile(id, version, suffix);
Stream stream = null;
try
{
if (file.Exists)
using (var fileStream = file.OpenWrite())
{
stream = file.OpenRead();
await fileStream.CopyToAsync(stream);
}
}
catch (FileNotFoundException)
catch (FileNotFoundException ex)
{
stream = null;
throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
}
return Task.FromResult(stream);
}
public async Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null)
public async Task UploadAsync(Guid id, long version, string suffix, Stream stream)
{
var file = GetFile(id, version, suffix);

4
src/Squidex.Infrastructure/Assets/IAssetStore.cs

@ -14,8 +14,8 @@ namespace Squidex.Infrastructure.Assets
{
public interface IAssetStore
{
Task<Stream> GetAssetAsync(Guid id, long version, string suffix = null);
Task DownloadAsync(Guid id, long version, string suffix, Stream stream);
Task UploadAssetAsync(Guid id, long version, Stream stream, string suffix = null);
Task UploadAsync(Guid id, long version, string suffix, Stream stream);
}
}

4
src/Squidex.Infrastructure/Assets/IAssetThumbnailGenerator.cs

@ -13,8 +13,8 @@ namespace Squidex.Infrastructure.Assets
{
public interface IAssetThumbnailGenerator
{
Task<ImageInfo> GetImageInfoAsync(Stream input);
Task<ImageInfo> GetImageInfoAsync(Stream source);
Task<Stream> CreateThumbnailAsync(Stream input, int? width, int? height, string mode);
Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode);
}
}

18
src/Squidex.Infrastructure/Assets/ImageSharp/ImageSharpAssetThumbnailGenerator.cs

@ -23,13 +23,13 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
Configuration.Default.AddImageFormat(new PngFormat());
}
public Task<Stream> CreateThumbnailAsync(Stream input, int? width, int? height, string mode)
public Task CreateThumbnailAsync(Stream source, Stream destination, int? width, int? height, string mode)
{
return Task.Run(() =>
{
if (width == null && height == null)
{
return input;
source.CopyTo(destination);
}
if (!Enum.TryParse<ResizeMode>(mode, true, out var resizeMode))
@ -40,9 +40,7 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
var w = width ?? int.MaxValue;
var h = height ?? int.MaxValue;
var result = new MemoryStream();
using (var sourceImage = Image.Load(input))
using (var sourceImage = Image.Load(source))
{
if (w >= sourceImage.Width && h >= sourceImage.Height && resizeMode == ResizeMode.Crop)
{
@ -57,23 +55,19 @@ namespace Squidex.Infrastructure.Assets.ImageSharp
};
sourceImage.MetaData.Quality = 0;
sourceImage.Resize(options).Save(result);
sourceImage.Resize(options).Save(destination);
}
result.Position = 0;
return result;
});
}
public Task<ImageInfo> GetImageInfoAsync(Stream input)
public Task<ImageInfo> GetImageInfoAsync(Stream source)
{
return Task.Run(() =>
{
ImageInfo imageInfo = null;
try
{
var image = Image.Load(input);
var image = Image.Load(source);
if (image.Width > 0 && image.Height > 0)
{

4
src/Squidex.Write/Assets/AssetCommandHandler.cs

@ -44,7 +44,7 @@ namespace Squidex.Write.Assets
c.Create(command);
await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead());
await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead());
context.Succeed(EntityCreatedResult.Create(c.Id, c.Version));
});
@ -58,7 +58,7 @@ namespace Squidex.Write.Assets
c.Update(command);
await assetStore.UploadAssetAsync(c.Id, c.Version, command.File.OpenRead());
await assetStore.UploadAsync(c.Id, c.Version, null, command.File.OpenRead());
});
}

17
src/Squidex/Config/Domain/AssetStoreModule.cs

@ -11,6 +11,7 @@ using Autofac;
using Microsoft.Extensions.Configuration;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Assets;
using Squidex.Infrastructure.GoogleCloud;
using Squidex.Infrastructure.Log;
// ReSharper disable InvertIf
@ -49,9 +50,23 @@ namespace Squidex.Config.Domain
.As<IExternalSystem>()
.SingleInstance();
}
else if (string.Equals(assetStoreType, "GoogleCloud", StringComparison.OrdinalIgnoreCase))
{
var bucketName = Configuration.GetValue<string>("assetStore:googleCloud:bucket");
if (string.IsNullOrWhiteSpace(bucketName))
{
throw new ConfigurationException("Configure AssetStore GoogleCloud bucket with 'assetStore:googleCloud:bucket'.");
}
builder.Register(c => new GoogleCloudAssetStore(bucketName))
.As<IAssetStore>()
.As<IExternalSystem>()
.SingleInstance();
}
else
{
throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder.");
throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder, GoogleCloud.");
}
}
}

4
src/Squidex/Config/Web/WebModule.cs

@ -28,6 +28,10 @@ namespace Squidex.Config.Web
builder.RegisterType<AppFilterAttribute>()
.AsSelf()
.SingleInstance();
builder.RegisterType<FileCallbackResultExecutor>()
.AsSelf()
.SingleInstance();
}
}
}

60
src/Squidex/Controllers/Api/Assets/AssetContentController.cs

@ -52,41 +52,53 @@ namespace Squidex.Controllers.Api.Assets
return NotFound();
}
Stream content;
if (asset.IsImage && (width.HasValue || height.HasValue))
return new FileCallbackResult(asset.MimeType, asset.FileName, async bodyStream =>
{
var suffix = $"{width}_{height}_{mode}";
content = await assetStorage.GetAssetAsync(asset.Id, asset.Version, suffix);
if (content == null)
if (asset.IsImage && (width.HasValue || height.HasValue))
{
var fullSizeContent = await assetStorage.GetAssetAsync(asset.Id, asset.Version);
var suffix = $"{width}_{height}_{mode}";
if (fullSizeContent == null)
try
{
return NotFound();
await assetStorage.DownloadAsync(asset.Id, asset.Version, suffix, bodyStream);
}
catch (AssetNotFoundException)
{
using (var tempStream1 = GetTempStream())
{
using (var tempStream2 = GetTempStream())
{
await assetStorage.DownloadAsync(asset.Id, asset.Version, null, tempStream1);
tempStream1.Position = 0;
content = await assetThumbnailGenerator.CreateThumbnailAsync(fullSizeContent, width, height, mode);
await assetThumbnailGenerator.CreateThumbnailAsync(tempStream1, tempStream2, width, height, mode);
tempStream2.Position = 0;
await assetStorage.UploadAssetAsync(asset.Id, asset.Version, content, suffix);
await assetStorage.UploadAsync(asset.Id, asset.Version, suffix, tempStream2);
tempStream2.Position = 0;
content.Position = 0;
await tempStream2.CopyToAsync(bodyStream);
}
}
}
}
}
else
{
content = await assetStorage.GetAssetAsync(asset.Id, asset.Version);
}
if (content == null)
{
return NotFound();
}
await assetStorage.DownloadAsync(asset.Id, asset.Version, null, bodyStream);
});
}
return new FileStreamResult(content, asset.MimeType) { FileDownloadName = asset.FileName };
private static FileStream GetTempStream()
{
var tempFileName = Path.GetTempFileName();
return new FileStream(tempFileName,
FileMode.Create,
FileAccess.ReadWrite,
FileShare.Delete, 1024 * 16,
FileOptions.Asynchronous |
FileOptions.DeleteOnClose |
FileOptions.SequentialScan);
}
}
}

43
src/Squidex/Pipeline/FileCallbackResult.cs

@ -0,0 +1,43 @@
// ==========================================================================
// FileCallbackResult.cs
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex Group
// All rights reserved.
// ==========================================================================
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
namespace Squidex.Pipeline
{
public class FileCallbackResult : FileResult
{
private readonly Func<Stream, Task> callback;
public Func<Stream, Task> Callback
{
get { return callback; }
}
public FileCallbackResult(string contentType, string name, Func<Stream, Task> callback)
: base(contentType)
{
FileDownloadName = name;
this.callback = callback;
}
public override Task ExecuteResultAsync(ActionContext context)
{
var executor = context.HttpContext.RequestServices.GetRequiredService<FileCallbackResultExecutor>();
return executor.ExecuteAsync(context, this);
}
}
}
#pragma warning restore 1573

38
src/Squidex/Pipeline/FileCallbackResultExecutor.cs

@ -0,0 +1,38 @@
// ==========================================================================
// FileCallbackResultExecutor.cs
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex Group
// All rights reserved.
// ==========================================================================
using System.Threading.Tasks;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Internal;
using Microsoft.Extensions.Logging;
namespace Squidex.Pipeline
{
public sealed class FileCallbackResultExecutor : FileResultExecutorBase
{
public FileCallbackResultExecutor(ILoggerFactory loggerFactory)
: base(CreateLogger<VirtualFileResultExecutor>(loggerFactory))
{
}
public async Task ExecuteAsync(ActionContext context, FileCallbackResult result)
{
try
{
SetHeadersAndLog(context, result);
await result.Callback(context.HttpContext.Response.Body);
}
catch
{
context.HttpContext.Response.Headers.Clear();
context.HttpContext.Response.StatusCode = 404;
}
}
}
}

5
src/Squidex/Squidex.csproj

@ -15,6 +15,10 @@
<ItemGroup>
<EmbeddedResource Include="Config\Identity\Cert\*.*;Docs\*.md" />
<EmbeddedResource Remove="Assets\**" />
<Compile Remove="Assets\**" />
<Content Remove="Assets\**" />
<None Remove="Assets\**" />
<None Update="dockerfile">
<CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
</None>
@ -23,6 +27,7 @@
<ItemGroup>
<ProjectReference Include="..\Squidex.Core\Squidex.Core.csproj" />
<ProjectReference Include="..\Squidex.Events\Squidex.Events.csproj" />
<ProjectReference Include="..\Squidex.Infrastructure.GoogleCloud\Squidex.Infrastructure.GoogleCloud.csproj" />
<ProjectReference Include="..\Squidex.Infrastructure.RabbitMq\Squidex.Infrastructure.RabbitMq.csproj" />
<ProjectReference Include="..\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
<ProjectReference Include="..\Squidex.Infrastructure.MongoDb\Squidex.Infrastructure.MongoDb.csproj" />

7
src/Squidex/appsettings.json

@ -14,8 +14,11 @@
"assetStore": {
"type": "Folder",
"folder": {
"path": "Assets"
}
"path": "Assets"
},
"googleCloud": {
"bucket": "squidex-assets"
}
},
"eventStore": {
"type": "MongoDb",

4
tests/Squidex.Write.Tests/Assets/AssetCommandHandlerTests.cs

@ -44,7 +44,7 @@ namespace Squidex.Write.Assets
[Fact]
public async Task Create_should_create_asset()
{
assetStore.Setup(x => x.UploadAssetAsync(assetId, 0, stream, null)).Returns(TaskHelper.Done).Verifiable();
assetStore.Setup(x => x.UploadAsync(assetId, 0, null, stream)).Returns(TaskHelper.Done).Verifiable();
assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable();
var context = CreateContextForCommand(new CreateAsset { AssetId = assetId, File = file });
@ -63,7 +63,7 @@ namespace Squidex.Write.Assets
[Fact]
public async Task Update_should_update_domain_object()
{
assetStore.Setup(x => x.UploadAssetAsync(assetId, 1, stream, null)).Returns(TaskHelper.Done).Verifiable();
assetStore.Setup(x => x.UploadAsync(assetId, 1, null, stream)).Returns(TaskHelper.Done).Verifiable();
assetThumbnailGenerator.Setup(x => x.GetImageInfoAsync(stream)).Returns(Task.FromResult(image)).Verifiable();
CreateAsset();

Loading…
Cancel
Save