From 71749dfef8af0c7a672f165b685dd379128680e2 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Wed, 19 Jul 2017 17:18:26 +0100 Subject: [PATCH] added azure blob storage support - needs testing --- Squidex.sln | 15 ++ .../AzureBlobAssetStore.cs | 147 ++++++++++++++++++ .../AzureStorageException.cs | 29 ++++ .../BlobContainerProvider.cs | 33 ++++ .../IBlobContainerProvider.cs | 18 +++ .../IStorageAccountManager.cs | 19 +++ ...Squidex.Infrastructure.AzureStorage.csproj | 15 ++ .../StorageAccountManager.cs | 47 ++++++ src/Squidex/Config/Domain/AssetStoreModule.cs | 32 ++++ src/Squidex/Squidex.csproj | 1 + src/Squidex/appsettings.json | 10 ++ 11 files changed, 366 insertions(+) create mode 100644 src/Squidex.Infrastructure.AzureStorage/AzureBlobAssetStore.cs create mode 100644 src/Squidex.Infrastructure.AzureStorage/AzureStorageException.cs create mode 100644 src/Squidex.Infrastructure.AzureStorage/BlobContainerProvider.cs create mode 100644 src/Squidex.Infrastructure.AzureStorage/IBlobContainerProvider.cs create mode 100644 src/Squidex.Infrastructure.AzureStorage/IStorageAccountManager.cs create mode 100644 src/Squidex.Infrastructure.AzureStorage/Squidex.Infrastructure.AzureStorage.csproj create mode 100644 src/Squidex.Infrastructure.AzureStorage/StorageAccountManager.cs diff --git a/Squidex.sln b/Squidex.sln index 86c2ab6fe..6d652542f 100644 --- a/Squidex.sln +++ b/Squidex.sln @@ -58,6 +58,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Domain.Users.Tests" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Infrastructure.GetEventStore", "src\Squidex.Infrastructure.GetEventStore\Squidex.Infrastructure.GetEventStore.csproj", "{EF75E488-1324-4E18-A1BD-D3A05AE67B1F}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.AzureStorage", "src\Squidex.Infrastructure.AzureStorage\Squidex.Infrastructure.AzureStorage.csproj", "{7931187E-A1E6-4F89-8BC8-20A1E445579F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -280,6 +282,18 @@ Global {EF75E488-1324-4E18-A1BD-D3A05AE67B1F}.Release|x64.Build.0 = Release|Any CPU {EF75E488-1324-4E18-A1BD-D3A05AE67B1F}.Release|x86.ActiveCfg = Release|Any CPU {EF75E488-1324-4E18-A1BD-D3A05AE67B1F}.Release|x86.Build.0 = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|x64.ActiveCfg = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|x64.Build.0 = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|x86.ActiveCfg = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Debug|x86.Build.0 = Debug|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|Any CPU.Build.0 = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|x64.ActiveCfg = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|x64.Build.0 = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|x86.ActiveCfg = Release|Any CPU + {7931187E-A1E6-4F89-8BC8-20A1E445579F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -308,5 +322,6 @@ Global {27CF800D-890F-4882-BF05-44EC3233537D} = {C0D540F0-9158-4528-BFD8-BEAE6EAE45EA} {42184546-E3CB-4D4F-9495-43979B9C63B9} = {C0D540F0-9158-4528-BFD8-BEAE6EAE45EA} {EF75E488-1324-4E18-A1BD-D3A05AE67B1F} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} + {7931187E-A1E6-4F89-8BC8-20A1E445579F} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} EndGlobalSection EndGlobal diff --git a/src/Squidex.Infrastructure.AzureStorage/AzureBlobAssetStore.cs b/src/Squidex.Infrastructure.AzureStorage/AzureBlobAssetStore.cs new file mode 100644 index 000000000..5b3085186 --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/AzureBlobAssetStore.cs @@ -0,0 +1,147 @@ +// ========================================================================== +// AzureBlobAssetStore.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Squidex.Infrastructure.Assets; + +namespace Squidex.Infrastructure.AzureStorage +{ + public class AzureBlobAssetStore : IAssetStore, IExternalSystem + { + private readonly IBlobContainerProvider blobContainerProvider; + private readonly string containerName; + private CloudBlobContainer blobContainer; + private const string AssetVersion = "AssetVersion"; + private const string AssetId = "AssetId"; + + public AzureBlobAssetStore(IBlobContainerProvider blobContainerProvider, string containerName) + { + Guard.NotNullOrEmpty(containerName, nameof(containerName)); + Guard.NotNull(blobContainerProvider, nameof(blobContainerProvider)); + + this.blobContainerProvider = blobContainerProvider; + this.containerName = containerName; + } + + public async Task DownloadAsync(string id, long version, string suffix, Stream stream) + { + var blobName = GetObjectName(id, suffix); + var blob = blobContainer.GetBlockBlobReference(blobName); + + if (!await blob.ExistsAsync()) + return; + + // look for the requested version + // first check if the original blob has the requested version + if (blob.Metadata.TryGetValue(AssetVersion, out string verionStr)) + { + if (long.TryParse(verionStr, out long blobVersion)) + { + // if not, then look for that snapshot which has the requested version number. + if (blobVersion != version) + { + var snapshotBlob = await FindSnapshotAsync(id, version); + blob = snapshotBlob ?? blob; + } + } + } + + await blob.DownloadToStreamAsync(stream); + } + + public async Task UploadAsync(string id, long version, string suffix, Stream stream) + { + var blobName = GetObjectName(id, suffix); + var blob = blobContainer.GetBlockBlobReference(blobName); + + if (await blob.ExistsAsync()) + { + // if it's already exist create a snapshot, and we overwrite the source blob of the snapshot. + // NOTE: not sure if there is an + await CreateVersioningSnapshotAsync(blob, id, version); + } + + if (!blob.Metadata.ContainsKey(AssetVersion)) + blob.Metadata.Add(AssetVersion, version.ToString()); + else + blob.Metadata[AssetVersion] = version.ToString(); + + blob.Metadata[AssetId] = id; + + await blob.UploadFromStreamAsync(stream); + await blob.SetMetadataAsync(); + } + + public async void Connect() + { + blobContainer = await blobContainerProvider.GetContainerAsync(containerName); + } + + private string GetObjectName(string id, string suffix) + { + Guard.NotNullOrEmpty(id, nameof(id)); + + if (blobContainer == null) + { + throw new InvalidOperationException("No connection established yet."); + } + + var name = $"{id}"; + + if (!string.IsNullOrWhiteSpace(suffix)) + { + name += "_" + suffix; + } + + return name; + } + + private async Task CreateVersioningSnapshotAsync(CloudBlockBlob blob, string id, long version) + { + var metadata = new Dictionary(); + metadata.Add(AssetVersion, version.ToString()); + metadata.Add(AssetId, id); + await blob.CreateSnapshotAsync(metadata, + null, null, null); + } + + private async Task FindSnapshotAsync(string id, long requestedVersion) + { + CloudBlockBlob resultBlob = null; + BlobContinuationToken token = null; + do + { + var listingResult = await blobContainer.ListBlobsSegmentedAsync(null, true, + BlobListingDetails.Snapshots, 10, token, null, null); + token = listingResult.ContinuationToken; + + foreach (CloudBlob snapshotBlob in listingResult.Results.Cast()) + { + if (snapshotBlob.Metadata.TryGetValue(AssetVersion, out string snapshotVersionStr) + && snapshotBlob.Metadata.TryGetValue(AssetId, out string snapshotAssetId) + && long.TryParse(snapshotVersionStr, out long snapshotVersion) + && snapshotVersion == requestedVersion + && snapshotAssetId == id) + { + resultBlob = snapshotBlob as CloudBlockBlob; + break; + } + } + } + while (token != null); + + return resultBlob; + } + } +} diff --git a/src/Squidex.Infrastructure.AzureStorage/AzureStorageException.cs b/src/Squidex.Infrastructure.AzureStorage/AzureStorageException.cs new file mode 100644 index 000000000..9121551cd --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/AzureStorageException.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// AzureStorageException.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using Microsoft.WindowsAzure.Storage; + +namespace Squidex.Infrastructure.AzureStorage +{ + [Serializable] + public class AzureStorageException : StorageException + { + public AzureStorageException() + { + } + + public AzureStorageException(string message) : base(message) + { + } + + public AzureStorageException(string message, Exception inner) : base(message, inner) + { + } + } +} diff --git a/src/Squidex.Infrastructure.AzureStorage/BlobContainerProvider.cs b/src/Squidex.Infrastructure.AzureStorage/BlobContainerProvider.cs new file mode 100644 index 000000000..e67ae845f --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/BlobContainerProvider.cs @@ -0,0 +1,33 @@ +// ========================================================================== +// BlobContainerProvider.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Squidex.Infrastructure.AzureStorage +{ + public class BlobContainerProvider : IBlobContainerProvider + { + private readonly IStorageAccountManager accountManager; + + public BlobContainerProvider(IStorageAccountManager accountManager) + { + this.accountManager = accountManager; + } + + public async Task GetContainerAsync(string name) + { + var client = accountManager.CreateCloudBlobClient(); + var saneName = name.Replace("@", "-").Replace(".", "-"); + var container = client.GetContainerReference(saneName); + await container.CreateIfNotExistsAsync(); + + return container; + } + } +} diff --git a/src/Squidex.Infrastructure.AzureStorage/IBlobContainerProvider.cs b/src/Squidex.Infrastructure.AzureStorage/IBlobContainerProvider.cs new file mode 100644 index 000000000..4a14ff169 --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/IBlobContainerProvider.cs @@ -0,0 +1,18 @@ +// ========================================================================== +// IBlobContainerProvider.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Squidex.Infrastructure.AzureStorage +{ + public interface IBlobContainerProvider + { + Task GetContainerAsync(string name); + } +} diff --git a/src/Squidex.Infrastructure.AzureStorage/IStorageAccountManager.cs b/src/Squidex.Infrastructure.AzureStorage/IStorageAccountManager.cs new file mode 100644 index 000000000..f38c2d3c8 --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/IStorageAccountManager.cs @@ -0,0 +1,19 @@ +// ========================================================================== +// IStorageAccountManager.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Squidex.Infrastructure.AzureStorage +{ + public interface IStorageAccountManager + { + CloudBlobClient CreateCloudBlobClient(); + + string GetSharedAccessSignature(); + } +} diff --git a/src/Squidex.Infrastructure.AzureStorage/Squidex.Infrastructure.AzureStorage.csproj b/src/Squidex.Infrastructure.AzureStorage/Squidex.Infrastructure.AzureStorage.csproj new file mode 100644 index 000000000..a1aa5eb3a --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/Squidex.Infrastructure.AzureStorage.csproj @@ -0,0 +1,15 @@ + + + + netstandard1.6 + + + + + + + + + + + \ No newline at end of file diff --git a/src/Squidex.Infrastructure.AzureStorage/StorageAccountManager.cs b/src/Squidex.Infrastructure.AzureStorage/StorageAccountManager.cs new file mode 100644 index 000000000..3de1822ce --- /dev/null +++ b/src/Squidex.Infrastructure.AzureStorage/StorageAccountManager.cs @@ -0,0 +1,47 @@ +// ========================================================================== +// IStorageAccountManager.cs +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex Group +// All rights reserved. +// ========================================================================== + +using System; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; + +namespace Squidex.Infrastructure.AzureStorage +{ + public class StorageAccountManager : IStorageAccountManager + { + private readonly CloudStorageAccount storageAccount; + + public StorageAccountManager(string storageAccountConnectionString) + { + try + { + storageAccount = CloudStorageAccount.Parse(storageAccountConnectionString); + } + catch (Exception ex) + when (ex is FormatException || ex is ArgumentException) + { + throw new AzureStorageException("Invalid storage account information provided. Please confirm the AccountName and AccountKey are valid in the app settings file."); + } + } + + public CloudBlobClient CreateCloudBlobClient() + { + return storageAccount.CreateCloudBlobClient(); + } + + public string GetSharedAccessSignature() + { + return storageAccount.GetSharedAccessSignature(new SharedAccessAccountPolicy() + { + SharedAccessStartTime = DateTimeOffset.UtcNow, + SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddHours(1), + Permissions = SharedAccessAccountPermissions.Read | SharedAccessAccountPermissions.List + }); + } + } +} \ No newline at end of file diff --git a/src/Squidex/Config/Domain/AssetStoreModule.cs b/src/Squidex/Config/Domain/AssetStoreModule.cs index 6f6376353..11cd51113 100644 --- a/src/Squidex/Config/Domain/AssetStoreModule.cs +++ b/src/Squidex/Config/Domain/AssetStoreModule.cs @@ -13,6 +13,7 @@ using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; using Squidex.Infrastructure.GoogleCloud; using Squidex.Infrastructure.Log; +using Squidex.Infrastructure.AzureStorage; // ReSharper disable InvertIf @@ -64,6 +65,37 @@ namespace Squidex.Config.Domain .As() .SingleInstance(); } + else if (string.Equals(assetStoreType, "AzureBlobStorage", StringComparison.OrdinalIgnoreCase)) + { + var containerName = Configuration.GetValue("assetStore:azureStorage:containerName"); + // NOTE: here it can be improved - if we use keyvault the secret key won't be in the app settings, rather the app has to run + // in the same active directory as the keyvault. + var connectionString = Configuration.GetValue("assetStore:azureStorage:connectionString"); + + if (string.IsNullOrWhiteSpace(containerName)) + { + throw new ConfigurationException("Configure AssetStore AzureStorage container with 'assetStore:azureStorage:containerName'."); + } + + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ConfigurationException( + "Configure AssetStore AzureStorage connection string with 'assetStore:azureStorage:connectionString'."); + } + + builder.Register(c => new StorageAccountManager(connectionString)) + .As() + .SingleInstance(); + + builder.RegisterType() + .As() + .SingleInstance(); + + builder.Register(c => new AzureBlobAssetStore(c.Resolve(), containerName)) + .As() + .As() + .SingleInstance(); + } else { throw new ConfigurationException($"Unsupported value '{assetStoreType}' for 'assetStore:type', supported: Folder, GoogleCloud."); diff --git a/src/Squidex/Squidex.csproj b/src/Squidex/Squidex.csproj index 54def6afb..4558e0ced 100644 --- a/src/Squidex/Squidex.csproj +++ b/src/Squidex/Squidex.csproj @@ -29,6 +29,7 @@ + diff --git a/src/Squidex/appsettings.json b/src/Squidex/appsettings.json index 5ba587f8d..37e0429cf 100644 --- a/src/Squidex/appsettings.json +++ b/src/Squidex/appsettings.json @@ -51,6 +51,16 @@ * The name of the bucket in google cloud store. */ "bucket": "squidex-assets" + }, + "azureStorage": { + /* + * The name of the container in the Azure Blob Storage + */ + "container": "squidex-assets", + /* + * The connection string to the azure storage service. + */ + "connectionString": "" } },