diff --git a/Squidex.sln b/Squidex.sln index f3c7e803d..56f8252b0 100644 --- a/Squidex.sln +++ b/Squidex.sln @@ -63,6 +63,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Migrate_01", "tools\Migrate EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Squidex.Tests", "tests\Squidex.Tests\Squidex.Tests.csproj", "{7E8CC864-4C6E-496F-A672-9F9AD8874835}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Squidex.Infrastructure.MongoGridFs", "src\Squidex.Infrastructure.MongoGridFs\Squidex.Infrastructure.MongoGridFs.csproj", "{2498B95C-0A55-4352-B32F-0860BE08317F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -321,6 +323,18 @@ Global {7E8CC864-4C6E-496F-A672-9F9AD8874835}.Release|x64.Build.0 = Release|Any CPU {7E8CC864-4C6E-496F-A672-9F9AD8874835}.Release|x86.ActiveCfg = Release|Any CPU {7E8CC864-4C6E-496F-A672-9F9AD8874835}.Release|x86.Build.0 = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|x64.ActiveCfg = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|x64.Build.0 = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|x86.ActiveCfg = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Debug|x86.Build.0 = Debug|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|Any CPU.Build.0 = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|x64.ActiveCfg = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|x64.Build.0 = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|x86.ActiveCfg = Release|Any CPU + {2498B95C-0A55-4352-B32F-0860BE08317F}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -349,6 +363,7 @@ Global {AA003372-CD8D-4DBC-962C-F61E0C93CF05} = {C9809D59-6665-471E-AD87-5AC624C65892} {7DA5B308-D950-4496-93D5-21D6C4D91644} = {C9809D59-6665-471E-AD87-5AC624C65892} {A4823E14-C0E5-4A4D-B28F-27424C25C3C7} = {94207AA6-4923-4183-A558-E0F8196B8CA3} + {2498B95C-0A55-4352-B32F-0860BE08317F} = {8CF53B92-5EB1-461D-98F8-70DA9B603FBF} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {02F2E872-3141-44F5-BD6A-33CD84E9FE08} diff --git a/src/Squidex.Infrastructure.MongoGridFs/Assets/MongoGridFsAssetStore.cs b/src/Squidex.Infrastructure.MongoGridFs/Assets/MongoGridFsAssetStore.cs new file mode 100644 index 000000000..09a3b12ca --- /dev/null +++ b/src/Squidex.Infrastructure.MongoGridFs/Assets/MongoGridFsAssetStore.cs @@ -0,0 +1,219 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; + +namespace Squidex.Infrastructure.Assets +{ + public class MongoGridFsAssetStore : IAssetStore, IInitializable + { + private readonly string path; + private readonly IGridFSBucket bucket; + private readonly DirectoryInfo directory; + + public MongoGridFsAssetStore(IGridFSBucket bucket, string path) + { + Guard.NotNull(bucket, nameof(bucket)); + Guard.NotNullOrEmpty(path, nameof(path)); + + this.bucket = bucket; + this.path = path; + + directory = new DirectoryInfo(path); + } + + public void Initialize() + { + try + { + // test bucket + bucket.Database.ListCollections(); + + if (!directory.Exists) + { + directory.Create(); + } + } + catch (MongoException ex) + { + throw new ConfigurationException( + $"Cannot connect to Mongo GridFS bucket '${bucket.Options.BucketName}'.", ex); + } + catch (IOException ex) + { + if (!directory.Exists) + { + throw new ConfigurationException($"Cannot access directory '{directory.FullName}'", ex); + } + } + } + + public string GenerateSourceUrl(string id, long version, string suffix) + { + var file = GetFile(id, version, suffix); + + return file.FullName; + } + + public async Task CopyAsync(string name, string id, long version, string suffix, + CancellationToken ct = default(CancellationToken)) + { + try + { + var file = GetFile(name); + + using (var stream = new MemoryStream()) + { + await bucket.DownloadToStreamByNameAsync(file.Name, stream, cancellationToken: ct); + await bucket.UploadFromStreamAsync(file.Name, stream, cancellationToken: ct); + } + + file.CopyTo(GetPath(id, version, suffix)); + } + catch (FileNotFoundException ex) + { + throw new AssetNotFoundException($"Asset {name} not found.", ex); + } + catch (GridFSException ex) + { + throw new AssetNotFoundException($"Asset {name} not found.", ex); + } + } + + public async Task DownloadAsync(string id, long version, string suffix, Stream stream, + CancellationToken ct = default(CancellationToken)) + { + var file = GetFile(id, version, suffix); + + try + { + if (file.Exists) + { + using (var fileStream = file.OpenRead()) + { + await fileStream.CopyToAsync(stream); + } + } + else + { + // file not found locally + // read from GridFS + await bucket.DownloadToStreamByNameAsync(file.Name, stream, cancellationToken: ct); + + // add to local assets + using (var fileStream = file.OpenWrite()) + { + await stream.CopyToAsync(fileStream); + } + } + } + catch (Exception ex) + { + throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex); + } + } + + public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken)) + => UploadFileCoreAsync(GetFile(name), stream); + + public Task UploadAsync(string id, long version, string suffix, Stream stream, + CancellationToken ct = default(CancellationToken)) + => UploadFileCoreAsync(GetFile(id, version, suffix), stream); + + public Task DeleteAsync(string name) + => DeleteCoreAsync(GetFile(name)); + + public Task DeleteAsync(string id, long version, string suffix) + => DeleteCoreAsync(GetFile(id, version, suffix)); + + private async Task DeleteCoreAsync(FileInfo file) + { + try + { + var filter = + Builders.Filter.And( + Builders.Filter.Eq(x => x.Filename, file.Name) + ); + using (var cursor = await bucket.FindAsync(filter)) + { + await cursor.ForEachAsync(fileInfo => bucket.DeleteAsync(fileInfo.Id)); + } + + file.Delete(); + } + catch (FileNotFoundException ex) + { + throw new AssetNotFoundException($"Asset {file.Name} not found.", ex); + } + catch (GridFSException ex) + { + throw new GridFSException( + $"Cannot delete file {file.Name} into Mongo GridFS bucket '{bucket.Options.BucketName}'.", ex); + } + } + + private async Task UploadFileCoreAsync(FileInfo file, Stream stream, + CancellationToken ct = default(CancellationToken)) + { + try + { + // upload file to GridFS first + await bucket.UploadFromStreamAsync(file.Name, stream, cancellationToken: ct); + + // create file locally + // even if this stage will fail, file will be recreated on the next Download call + using (var fileStream = file.OpenWrite()) + { + await stream.CopyToAsync(fileStream); + } + } + catch (IOException ex) + { + throw new IOException($"Cannot write file '{file.Name}' into directory '{directory.FullName}'.", ex); + } + catch (GridFSException ex) + { + throw new GridFSException( + $"Cannot upload file {file.Name} into Mongo GridFS bucket '{bucket.Options.BucketName}'.", + ex); + } + } + + private FileInfo GetFile(string id, long version, string suffix) + { + Guard.NotNullOrEmpty(id, nameof(id)); + + return GetFile(GetPath(id, version, suffix)); + } + + private FileInfo GetFile(string name) + { + Guard.NotNullOrEmpty(name, nameof(name)); + + return new FileInfo(GetPath(name)); + } + + private string GetPath(string name) + { + return Path.Combine(directory.FullName, name); + } + + private string GetPath(string id, long version, string suffix) + { + return Path.Combine(directory.FullName, + string.Join("_", + new[] { id, version.ToString(), suffix }.ToList().Where(x => !string.IsNullOrWhiteSpace(x)))); + } + } +} \ No newline at end of file diff --git a/src/Squidex.Infrastructure.MongoGridFs/Squidex.Infrastructure.MongoGridFs.csproj b/src/Squidex.Infrastructure.MongoGridFs/Squidex.Infrastructure.MongoGridFs.csproj new file mode 100644 index 000000000..30d246b26 --- /dev/null +++ b/src/Squidex.Infrastructure.MongoGridFs/Squidex.Infrastructure.MongoGridFs.csproj @@ -0,0 +1,25 @@ + + + netstandard2.0 + Squidex.Infrastructure + + + full + True + + + + + + + + + ..\..\Squidex.ruleset + + + + + + + + \ No newline at end of file diff --git a/src/Squidex/Config/Domain/AssetServices.cs b/src/Squidex/Config/Domain/AssetServices.cs index 59297c8b6..efe72e194 100644 --- a/src/Squidex/Config/Domain/AssetServices.cs +++ b/src/Squidex/Config/Domain/AssetServices.cs @@ -7,6 +7,8 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; using Squidex.Infrastructure.Assets.ImageSharp; @@ -44,6 +46,29 @@ namespace Squidex.Config.Domain services.AddSingletonAs(c => new AzureBlobAssetStore(connectionString, containerName)) .As() .As(); + }, + ["MongoDb"] = () => + { + var connectionString = config.GetRequiredValue("assetStore:mongoDb:connectionString"); + var mongoDatabaseName = config.GetRequiredValue("assetStore:mongoDb:database"); + var mongoGridFsBucketName = config.GetRequiredValue("assetStore:mongoDb:bucket"); + var localPath = config.GetRequiredValue("assetStore:mongoDb:path"); + + services.AddSingletonAs(c => + { + var mongoClient = Singletons.GetOrAdd(connectionString, s => new MongoClient(s)); + var mongoDatabase = mongoClient.GetDatabase(mongoDatabaseName); + var gridFsbucket = new GridFSBucket(mongoDatabase, new GridFSBucketOptions() + { + BucketName = mongoGridFsBucketName, + ChunkSizeBytes = 255 * 1024 + // Defaults to 255KB, provisionary set here to avoid future changes in default values + }); + + return new MongoGridFsAssetStore(gridFsbucket, localPath); + }) + .As() + .As(); } }); diff --git a/src/Squidex/Squidex.csproj b/src/Squidex/Squidex.csproj index 040d73240..d733deff4 100644 --- a/src/Squidex/Squidex.csproj +++ b/src/Squidex/Squidex.csproj @@ -44,6 +44,7 @@ + @@ -76,6 +77,7 @@ + diff --git a/src/Squidex/appsettings.json b/src/Squidex/appsettings.json index b72f5ec5c..278c3761e 100644 --- a/src/Squidex/appsettings.json +++ b/src/Squidex/appsettings.json @@ -85,6 +85,26 @@ */ "connectionString": "UseDevelopmentStorage=true" }, + "mongoDb": { + /* + * The connection string to your Mongo Server. + * + * Read More: https://docs.mongodb.com/manual/reference/connection-string/ + */ + "connectionString": "mongodb://localhost", + /* + * The name of the event store database. + */ + "database": "SquidexAssets", + /* + * The name of the Mongo Grid FS bucket. + */ + "bucket": "fs", + /* + * The relative or absolute path to the folder to store the local copy of assets. + */ + "path": "Assets" + }, /* * Allow to expose the url in graph ql url. */ diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index f8f0923d8..8f1f66ecd 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -31,7 +31,7 @@ namespace Squidex.Infrastructure.Assets public abstract void Dispose(); [Fact] - public Task Should_throw_exception_if_asset_to_download_is_not_found() + public virtual Task Should_throw_exception_if_asset_to_download_is_not_found() { ((IInitializable)Sut).Initialize(); @@ -111,7 +111,7 @@ namespace Squidex.Infrastructure.Assets await Sut.DeleteAsync(tempId, 0, null); } - private static string Id() + protected static string Id() { return Guid.NewGuid().ToString(); } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs new file mode 100644 index 000000000..bd05add97 --- /dev/null +++ b/tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs @@ -0,0 +1,101 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FakeItEasy; +using MongoDB.Driver; +using MongoDB.Driver.GridFS; +using Xunit; + +namespace Squidex.Infrastructure.Assets +{ + public class MongoGridFsAssetStoreTests : AssetStoreTests + { + private readonly string testFolder = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + private readonly IGridFSBucket bucket = A.Fake(); + + public override MongoGridFsAssetStore CreateStore() + { + return new MongoGridFsAssetStore(bucket, testFolder); + } + + public override void Dispose() + { + if (Directory.Exists(testFolder)) + { + Directory.Delete(testFolder, true); + } + } + + [Fact] + public void Should_calculate_source_url() + { + Sut.Initialize(); + + var id = Id(); + + Assert.Equal(Path.Combine(testFolder, $"{id}_1"), Sut.GenerateSourceUrl(id, 1, null)); + } + + [Fact] + public override Task Should_throw_exception_if_asset_to_download_is_not_found() + { + var id = Id(); + + A.CallTo(() => + bucket.DownloadToStreamByNameAsync($"{id}_1_suffix", A.Ignored, null, + A.Ignored)) + .Throws(); + + ((IInitializable)Sut).Initialize(); + + return Assert.ThrowsAsync(() => + Sut.DownloadAsync(id, 1, "suffix", new MemoryStream())); + } + + [Fact] + public async Task Should_try_to_download_asset_if_is_not_found_locally() + { + var id = Id(); + using (var stream = new MemoryStream()) + { + ((IInitializable)Sut).Initialize(); + + await Sut.DownloadAsync(id, 1, "suffix", stream); + + A.CallTo(() => + bucket.DownloadToStreamByNameAsync($"{id}_1_suffix", stream, + A.Ignored, + A.Ignored)) + .MustHaveHappened(); + } + } + + [Fact] + public async Task Should_try_to_upload_asset_and_save_locally() + { + var id = Id(); + var file = new FileInfo(testFolder + Path.DirectorySeparatorChar + $"{id}_1_suffix"); + using (var stream = new MemoryStream(new byte[] { 0x1, 0x2, 0x3, 0x4 })) + { + ((IInitializable)Sut).Initialize(); + + await Sut.UploadAsync(id, 1, "suffix", stream); + + A.CallTo(() => + bucket.UploadFromStreamAsync($"{id}_1_suffix", stream, A.Ignored, + A.Ignored)) + .MustHaveHappened(); + + Assert.True(file.Exists); + } + } + } +} \ No newline at end of file diff --git a/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj b/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj index 0e366e697..3042125aa 100644 --- a/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj +++ b/tests/Squidex.Infrastructure.Tests/Squidex.Infrastructure.Tests.csproj @@ -8,6 +8,7 @@ +