Merge branch 'master' of https://github.com/razims/squidex

# Conflicts:
#	src/Squidex/Squidex.csproj

README.md

@@ -32,9 +32,10 @@ Current Version 1.8.0. Roadmap: https://trello.com/b/KakM4F3S/squidex-roadmap
 
 ### Contributors
 
-* [pushrbx](https://pushrbx.net/): Azure Store Support.
+* [pushrbx](https://pushrbx.net/): Azure Store support.
 * [cpmstars](https://www.cpmstars.com): Asset support for rich editor.
 * [civicplus](https://www.civicplus.com/) ([Avd6977](https://github.com/Avd6977), [dsbegnoce](https://github.com/dsbegnoche)): Google Maps support, custom regex patterns and a lot of small improvements.
+* [razims](https://github.com/razims): GridFS support.
 
 ## Contributing
 

src/Squidex.Domain.Apps.Entities/Backup/EventStreamWriter.cs

@@ -59,7 +59,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
                 var attachmentPath = $"attachments/{attachmentFolder}/{writtenEvents}.blob";
                 var attachmentEntry = archive.GetEntry(attachmentPath) ?? archive.CreateEntry(attachmentPath);
 
-                using (var stream = eventEntry.Open())
+                using (var stream = attachmentEntry.Open())
                 {
                     await attachment(stream);
                 }

src/Squidex.Domain.Apps.Entities/Contents/GraphQL/CachingGraphQLService.cs

@@ -11,7 +11,6 @@ using System.Threading.Tasks;
 using Microsoft.Extensions.Caching.Memory;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Domain.Apps.Entities.Assets;
-using Squidex.Domain.Apps.Entities.Assets.Repositories;
 using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Contents.GraphQL

src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs

@@ -0,0 +1,122 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.GridFS;
+
+namespace Squidex.Infrastructure.Assets
+{
+    public sealed class MongoGridFsAssetStore : IAssetStore, IInitializable
+    {
+        private const int BufferSize = 81920;
+        private readonly IGridFSBucket<string> bucket;
+
+        public MongoGridFsAssetStore(IGridFSBucket<string> bucket)
+        {
+            Guard.NotNull(bucket, nameof(bucket));
+
+            this.bucket = bucket;
+        }
+
+        public void Initialize()
+        {
+            try
+            {
+                bucket.Database.ListCollections();
+            }
+            catch (MongoException ex)
+            {
+                throw new ConfigurationException($"Cannot connect to Mongo GridFS bucket '${bucket.Options.BucketName}'.", ex);
+            }
+        }
+
+        public string GenerateSourceUrl(string id, long version, string suffix)
+        {
+            return "UNSUPPORTED";
+        }
+
+        public async Task CopyAsync(string name, string id, long version, string suffix, CancellationToken ct = default(CancellationToken))
+        {
+            try
+            {
+                var target = GetFileName(id, version, suffix);
+
+                using (var readStream = await bucket.OpenDownloadStreamAsync(name, cancellationToken: ct))
+                {
+                    await bucket.UploadFromStreamAsync(target, target, readStream, cancellationToken: ct);
+                }
+            }
+            catch (GridFSFileNotFoundException ex)
+            {
+                throw new AssetNotFoundException($"Asset {name} not found.", ex);
+            }
+        }
+
+        public async Task DownloadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
+        {
+            try
+            {
+                var name = GetFileName(id, version, suffix);
+
+                using (var readStream = await bucket.OpenDownloadStreamAsync(name, cancellationToken: ct))
+                {
+                    await readStream.CopyToAsync(stream, BufferSize);
+                }
+            }
+            catch (GridFSFileNotFoundException ex)
+            {
+                throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
+            }
+        }
+
+        public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
+        {
+            return UploadFileCoreAsync(name, stream, ct);
+        }
+
+        public Task UploadAsync(string id, long version, string suffix, Stream stream, CancellationToken ct = default(CancellationToken))
+        {
+            return UploadFileCoreAsync(GetFileName(id, version, suffix), stream, ct);
+        }
+
+        public Task DeleteAsync(string name)
+        {
+            return DeleteCoreAsync(name);
+        }
+
+        public Task DeleteAsync(string id, long version, string suffix)
+        {
+            return DeleteCoreAsync(GetFileName(id, version, suffix));
+        }
+
+        private async Task DeleteCoreAsync(string id)
+        {
+            try
+            {
+                await bucket.DeleteAsync(id);
+            }
+            catch (GridFSFileNotFoundException)
+            {
+                return;
+            }
+        }
+
+        private Task UploadFileCoreAsync(string id, Stream stream, CancellationToken ct = default(CancellationToken))
+        {
+            return bucket.UploadFromStreamAsync(id, id, stream, cancellationToken: ct);
+        }
+
+        private static string GetFileName(string id, long version, string suffix)
+        {
+            return string.Join("_", new[] { id, version.ToString(), suffix }.Where(x => !string.IsNullOrWhiteSpace(x)));
+        }
+    }
+}

src/Squidex.Infrastructure.MongoDb/MongoDb/OData/PropertyBuilder.cs

@@ -7,7 +7,6 @@
 
 using System.Linq;
 using Microsoft.OData.UriParser;
-using MongoDB.Driver;
 
 namespace Squidex.Infrastructure.MongoDb.OData
 {

src/Squidex.Infrastructure.MongoDb/Squidex.Infrastructure.MongoDb.csproj

@@ -13,6 +13,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.OData.Core" Version="7.5.0" />
     <PackageReference Include="MongoDB.Driver" Version="2.7.0" />
+    <PackageReference Include="MongoDB.Driver.GridFS" Version="2.7.0" />
     <PackageReference Include="RefactoringEssentials" Version="5.6.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.0.2" />
     <PackageReference Include="System.Threading.Tasks.Dataflow" Version="4.9.0" />

src/Squidex.Infrastructure.MongoGridFs/Assets/MongoGridFsAssetStore.cs

@@ -0,0 +1,235 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Threading;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using MongoDB.Driver.Core.Bindings;
+using MongoDB.Driver.GridFS;
+
+namespace Squidex.Infrastructure.Assets
+{
+    public class MongoGridFsAssetStore : IAssetStore, IInitializable
+    {
+        public const int ChunkSizeBytes = 255 * 1024;
+        private const int BufferSize = 81920;
+
+        private readonly string path;
+        private readonly IGridFSBucket<string> bucket;
+        private readonly DirectoryInfo directory;
+
+        public MongoGridFsAssetStore(IGridFSBucket<string> bucket, string path)
+        {
+            Guard.NotNull(bucket, nameof(bucket));
+            Guard.NotNullOrEmpty(path, nameof(path));
+
+            this.bucket = bucket;
+            this.path = path;
+
+            directory = new DirectoryInfo(path);
+        }
+
+        public void Initialize()
+        {
+            try
+            {
+                // test bucket
+                bucket.Database.ListCollections();
+
+                if (!directory.Exists)
+                {
+                    directory.Create();
+                }
+            }
+            catch (MongoException ex)
+            {
+                throw new ConfigurationException(
+                    $"Cannot connect to Mongo GridFS bucket '${bucket.Options.BucketName}'.", ex);
+            }
+            catch (IOException ex)
+            {
+                if (!directory.Exists)
+                {
+                    throw new ConfigurationException($"Cannot access directory '{directory.FullName}'", ex);
+                }
+            }
+        }
+
+        public string GenerateSourceUrl(string id, long version, string suffix)
+        {
+            var file = GetFile(id, version, suffix);
+
+            return file.FullName;
+        }
+
+        public async Task CopyAsync(string name, string id, long version, string suffix,
+            CancellationToken ct = default(CancellationToken))
+        {
+            try
+            {
+                var file = GetFile(name);
+                var toFile = GetFile(id, version, suffix);
+
+                file.CopyTo(toFile.FullName);
+
+                using (var readStream = await bucket.OpenDownloadStreamAsync(file.Name, cancellationToken: ct))
+                {
+                    using (var writeStream =
+                        await bucket.OpenUploadStreamAsync(toFile.Name, toFile.Name, cancellationToken: ct))
+                    {
+                        var buffer = new byte[ChunkSizeBytes];
+                        int bytesRead;
+                        while ((bytesRead = await readStream.ReadAsync(buffer, 0, buffer.Length, ct)) > 0)
+                        {
+                            await writeStream.WriteAsync(buffer, 0, bytesRead, ct);
+                        }
+
+                        await writeStream.CloseAsync(ct);
+                    }
+                }
+            }
+            catch (FileNotFoundException ex)
+            {
+                throw new AssetNotFoundException($"Asset {name} not found.", ex);
+            }
+            catch (GridFSException ex)
+            {
+                throw new AssetNotFoundException($"Asset {name} not found.", ex);
+            }
+        }
+
+        public async Task DownloadAsync(string id, long version, string suffix, Stream stream,
+            CancellationToken ct = default(CancellationToken))
+        {
+            var file = GetFile(id, version, suffix);
+
+            try
+            {
+                if (file.Exists)
+                {
+                    using (var fileStream = file.OpenRead())
+                    {
+                        await fileStream.CopyToAsync(stream, BufferSize, ct);
+                    }
+                }
+                else
+                {
+                    // file not found locally
+                    // read from GridFS
+                    using (var readStream = await bucket.OpenDownloadStreamAsync(file.Name, cancellationToken: ct))
+                    {
+                        using (var fileStream = file.OpenWrite())
+                        {
+                            var buffer = new byte[BufferSize];
+                            int bytesRead;
+                            while ((bytesRead = await readStream.ReadAsync(buffer, 0, buffer.Length, ct)) > 0)
+                            {
+                                await fileStream.WriteAsync(buffer, 0, bytesRead, ct);
+                                await stream.WriteAsync(buffer, 0, bytesRead, ct);
+                            }
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                throw new AssetNotFoundException($"Asset {id}, {version} not found.", ex);
+            }
+        }
+
+        public Task UploadAsync(string name, Stream stream, CancellationToken ct = default(CancellationToken))
+            => UploadFileCoreAsync(GetFile(name), stream, ct);
+
+        public Task UploadAsync(string id, long version, string suffix, Stream stream,
+            CancellationToken ct = default(CancellationToken))
+            => UploadFileCoreAsync(GetFile(id, version, suffix), stream, ct);
+
+        public Task DeleteAsync(string name)
+            => DeleteCoreAsync(GetFile(name));
+
+        public Task DeleteAsync(string id, long version, string suffix)
+            => DeleteCoreAsync(GetFile(id, version, suffix));
+
+        private async Task DeleteCoreAsync(FileInfo file, CancellationToken ct = default(CancellationToken))
+        {
+            try
+            {
+                file.Delete();
+                await bucket.DeleteAsync(file.Name, ct);
+            }
+            catch (FileNotFoundException ex)
+            {
+                throw new AssetNotFoundException($"Asset {file.Name} not found.", ex);
+            }
+            catch (GridFSException ex)
+            {
+                throw new GridFSException(
+                    $"Cannot delete file {file.Name} into Mongo GridFS bucket '{bucket.Options.BucketName}'.", ex);
+            }
+        }
+
+        private async Task UploadFileCoreAsync(FileInfo file, Stream stream,
+            CancellationToken ct = default(CancellationToken))
+        {
+            try
+            {
+                // upload file to GridFS first
+                await bucket.UploadFromStreamAsync(file.Name, file.Name, stream, cancellationToken: ct);
+
+                // reset stream position
+                stream.Position = 0;
+
+                // create file locally
+                // even if this stage will fail, file will be recreated on the next Download call
+                using (var fileStream = file.OpenWrite())
+                {
+                    await stream.CopyToAsync(fileStream, BufferSize, ct);
+                }
+            }
+            catch (IOException ex)
+            {
+                throw new IOException($"Cannot write file '{file.Name}' into directory '{directory.FullName}'.", ex);
+            }
+            catch (GridFSException ex)
+            {
+                throw new GridFSException(
+                    $"Cannot upload file {file.Name} into Mongo GridFS bucket '{bucket.Options.BucketName}'.",
+                    ex);
+            }
+        }
+
+        private FileInfo GetFile(string id, long version, string suffix)
+        {
+            Guard.NotNullOrEmpty(id, nameof(id));
+
+            return GetFile(GetPath(id, version, suffix));
+        }
+
+        private FileInfo GetFile(string name)
+        {
+            Guard.NotNullOrEmpty(name, nameof(name));
+
+            return new FileInfo(GetPath(name));
+        }
+
+        private string GetPath(string name)
+        {
+            return Path.Combine(directory.FullName, name);
+        }
+
+        private string GetPath(string id, long version, string suffix)
+        {
+            return Path.Combine(directory.FullName,
+                string.Join("_",
+                    new[] { id, version.ToString(), suffix }.ToList().Where(x => !string.IsNullOrWhiteSpace(x))));
+        }
+    }
+}

src/Squidex.Infrastructure.MongoGridFs/Squidex.Infrastructure.MongoGridFs.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <RootNamespace>Squidex.Infrastructure</RootNamespace>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <DebugType>full</DebugType>
+    <DebugSymbols>True</DebugSymbols>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="MongoDB.Driver" Version="2.5.1" />
+    <PackageReference Include="MongoDB.Driver.GridFS" Version="2.5.1" />
+    <PackageReference Include="RefactoringEssentials" Version="5.6.0" />
+    <PackageReference Include="StyleCop.Analyzers" Version="1.0.2" />
+  </ItemGroup>
+  <PropertyGroup>
+    <CodeAnalysisRuleSet>..\..\Squidex.ruleset</CodeAnalysisRuleSet>
+  </PropertyGroup>
+  <ItemGroup>
+    <AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Squidex.Infrastructure\Squidex.Infrastructure.csproj" />
+  </ItemGroup>
+</Project>

src/Squidex/Areas/Api/Controllers/Contents/ContentsController.cs

@@ -6,7 +6,6 @@
 // ==========================================================================
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;

src/Squidex/Config/Domain/AssetServices.cs

@@ -7,6 +7,8 @@
 
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
+using MongoDB.Driver;
+using MongoDB.Driver.GridFS;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Assets;
 using Squidex.Infrastructure.Assets.ImageSharp;
@@ -44,6 +46,27 @@ namespace Squidex.Config.Domain
                     services.AddSingletonAs(c => new AzureBlobAssetStore(connectionString, containerName))
                         .As<IAssetStore>()
                         .As<IInitializable>();
+                },
+                ["MongoDb"] = () =>
+                {
+                    var mongoConfiguration = config.GetRequiredValue("assetStore:mongoDb:configuration");
+                    var mongoDatabaseName = config.GetRequiredValue("assetStore:mongoDb:database");
+                    var mongoGridFsBucketName = config.GetRequiredValue("assetStore:mongoDb:bucket");
+
+                    services.AddSingletonAs(c =>
+                        {
+                            var mongoClient = Singletons<IMongoClient>.GetOrAdd(mongoConfiguration, s => new MongoClient(s));
+                            var mongoDatabase = mongoClient.GetDatabase(mongoDatabaseName);
+
+                            var gridFsbucket = new GridFSBucket<string>(mongoDatabase, new GridFSBucketOptions
+                            {
+                                BucketName = mongoGridFsBucketName
+                            });
+
+                            return new MongoGridFsAssetStore(gridFsbucket);
+                        })
+                        .As<IAssetStore>()
+                        .As<IInitializable>();
                 }
             });
 

src/Squidex/appsettings.json

@@ -78,7 +78,7 @@
     /*
      * Define the type of the read store. 
      * 
-     * Supported: Folder (local folder), GoogleCloud (hosted in Google Cloud only), AzureBlob.
+     * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob.
      */
     "type": "Folder",
     "folder": {
@@ -103,6 +103,22 @@
        */
       "connectionString": "UseDevelopmentStorage=true"
     },
+    "mongoDb": {
+      /*
+       * The connection string to your Mongo Server.
+       * 
+       * Read More: https://docs.mongodb.com/manual/reference/connection-string/
+       */
+      "configuration": "mongodb://localhost",
+      /*
+       * The name of the event store database.
+       */
+      "database": "SquidexAssets",
+      /*
+       * The name of the Mongo Grid FS bucket.
+       */
+      "bucket": "fs"
+    },
     /*
      * Allow to expose the url in graph ql url.
      */

tests/Squidex.Domain.Apps.Core.Tests/Operations/ExtractReferenceIds/ReferenceExtractionTests.cs

@@ -6,7 +6,6 @@
 // ==========================================================================
 
 using System;
-using System.Collections.Generic;
 using System.Linq;
 using Newtonsoft.Json.Linq;
 using Squidex.Domain.Apps.Core.Apps;

tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLQueriesTests.cs

@@ -6,11 +6,9 @@
 // ==========================================================================
 
 using System;
-using System.Collections.Generic;
 using System.Threading.Tasks;
 using FakeItEasy;
 using Squidex.Domain.Apps.Core.Contents;
-using Squidex.Domain.Apps.Entities.Assets;
 using Squidex.Infrastructure;
 using Xunit;
 

tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLTestBase.cs

@@ -20,7 +20,6 @@ using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Core.Schemas;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Domain.Apps.Entities.Assets;
-using Squidex.Domain.Apps.Entities.Assets.Repositories;
 using Squidex.Domain.Apps.Entities.Contents.TestData;
 using Squidex.Domain.Apps.Entities.Schemas;
 using Squidex.Infrastructure;

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Guard/GuardContentTests.cs

@@ -5,7 +5,6 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using System;
 using FakeItEasy;
 using NodaTime;
 using Squidex.Domain.Apps.Core.Contents;

tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagGrainTests.cs

@@ -7,7 +7,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 using FakeItEasy;
 using Squidex.Infrastructure;

tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs

@@ -31,7 +31,7 @@ namespace Squidex.Infrastructure.Assets
         public abstract void Dispose();
 
         [Fact]
-        public Task Should_throw_exception_if_asset_to_download_is_not_found()
+        public virtual Task Should_throw_exception_if_asset_to_download_is_not_found()
         {
             ((IInitializable)Sut).Initialize();
 
@@ -64,7 +64,7 @@ namespace Squidex.Infrastructure.Assets
         }
 
         [Fact]
-        public async Task Should_commit_temporary_file()
+        public virtual async Task Should_commit_temporary_file()
         {
             ((IInitializable)Sut).Initialize();
 
@@ -111,7 +111,7 @@ namespace Squidex.Infrastructure.Assets
             await Sut.DeleteAsync(tempId, 0, null);
         }
 
-        private static string Id()
+        protected static string Id()
         {
             return Guid.NewGuid().ToString();
         }

tests/Squidex.Infrastructure.Tests/Assets/AzureBlobAssetStoreTests.cs

@@ -8,6 +8,8 @@
 using System;
 using Xunit;
 
+#pragma warning disable xUnit1000 // Test classes must be public
+
 namespace Squidex.Infrastructure.Assets
 {
     internal class AzureBlobAssetStoreTests : AssetStoreTests<AzureBlobAssetStore>
@@ -21,7 +23,7 @@ namespace Squidex.Infrastructure.Assets
         {
         }
 
-        // [Fact]
+        [Fact]
         public void Should_calculate_source_url()
         {
             Sut.Initialize();

tests/Squidex.Infrastructure.Tests/Assets/GoogleCloudAssetStoreTests.cs

@@ -8,6 +8,8 @@
 using System;
 using Xunit;
 
+#pragma warning disable xUnit1000 // Test classes must be public
+
 namespace Squidex.Infrastructure.Assets
 {
     internal class GoogleCloudAssetStoreTests : AssetStoreTests<GoogleCloudAssetStore>
@@ -21,7 +23,7 @@ namespace Squidex.Infrastructure.Assets
         {
         }
 
-        // [Fact]
+        [Fact]
         public void Should_calculate_source_url()
         {
             Sut.Initialize();

tests/Squidex.Infrastructure.Tests/Assets/MongoGridFsAssetStoreTests.cs

@@ -0,0 +1,51 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschränkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using MongoDB.Driver;
+using MongoDB.Driver.GridFS;
+using Xunit;
+
+#pragma warning disable xUnit1000 // Test classes must be public
+
+namespace Squidex.Infrastructure.Assets
+{
+    internal class MongoGridFSAssetStoreTests : AssetStoreTests<MongoGridFsAssetStore>
+    {
+        private static readonly IMongoClient MongoClient;
+        private static readonly IMongoDatabase MongoDatabase;
+        private static readonly IGridFSBucket<string> GridFSBucket;
+
+        static MongoGridFSAssetStoreTests()
+        {
+            MongoClient = new MongoClient("mongodb://localhost");
+            MongoDatabase = MongoClient.GetDatabase("Test");
+
+            GridFSBucket = new GridFSBucket<string>(MongoDatabase, new GridFSBucketOptions
+            {
+                BucketName = "fs"
+            });
+        }
+
+        public override MongoGridFsAssetStore CreateStore()
+        {
+            return new MongoGridFsAssetStore(GridFSBucket);
+        }
+
+        public override void Dispose()
+        {
+        }
+
+        [Fact]
+        public void Should_not_calculate_source_url()
+        {
+            Sut.Initialize();
+
+            Assert.Equal("UNSUPPORTED", Sut.GenerateSourceUrl(Guid.NewGuid().ToString(), 1, null));
+        }
+    }
+}