Manually merged FTP. Closes #360

README.md

@@ -34,9 +34,10 @@ Current Version v2.0.4. Roadmap: https://trello.com/b/KakM4F3S/squidex-roadmap
 
 ### Contributors
 
-* [pushrbx](https://pushrbx.net/): Azure Store support.
-* [cpmstars](https://www.cpmstars.com): Asset support for rich editor.
 * [civicplus](https://www.civicplus.com/) ([Avd6977](https://github.com/Avd6977), [dsbegnoce](https://github.com/dsbegnoche)): Google Maps support, custom regex patterns and a lot of small improvements.
+* [cpmstars](https://www.cpmstars.com): Asset support for rich editor.
+* [guohai](https://github.com/seamys): FTP asset store support, Email rule support, custom editors and bug fixes.
+* [pushrbx](https://pushrbx.net/): Azure Store support.
 * [razims](https://github.com/razims): GridFS support.
 
 ## Contributing

src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs

@@ -46,6 +46,8 @@ namespace Squidex.Infrastructure.Assets
 
         public async Task CopyAsync(string sourceFileName, string targetFileName, CancellationToken ct = default)
         {
+            Guard.NotNullOrEmpty(targetFileName, nameof(targetFileName));
+
             try
             {
                 var sourceName = GetFileName(sourceFileName, nameof(sourceFileName));
@@ -63,6 +65,8 @@ namespace Squidex.Infrastructure.Assets
 
         public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default)
         {
+            Guard.NotNull(stream, nameof(stream));
+
             try
             {
                 var name = GetFileName(fileName, nameof(fileName));
@@ -80,6 +84,8 @@ namespace Squidex.Infrastructure.Assets
 
         public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default)
         {
+            Guard.NotNull(stream, nameof(stream));
+
             try
             {
                 var name = GetFileName(fileName, nameof(fileName));

src/Squidex.Infrastructure/Assets/FTPAssetStore.cs

@@ -0,0 +1,158 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschränkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using FluentFTP;
+using Squidex.Infrastructure.Log;
+
+namespace Squidex.Infrastructure.Assets
+{
+    public sealed class FTPAssetStore : IAssetStore, IInitializable
+    {
+        private readonly string path;
+        private readonly ISemanticLog log;
+        private readonly Func<IFtpClient> factory;
+
+        public FTPAssetStore(Func<IFtpClient> factory, string path, ISemanticLog log)
+        {
+            Guard.NotNull(factory, nameof(factory));
+            Guard.NotNullOrEmpty(path, nameof(path));
+            Guard.NotNull(log, nameof(log));
+
+            this.factory = factory;
+            this.path = path;
+
+            this.log = log;
+        }
+
+        public string GeneratePublicUrl(string fileName)
+        {
+            return null;
+        }
+
+        public async Task InitializeAsync(CancellationToken ct = default)
+        {
+            using (var client = factory())
+            {
+                await client.ConnectAsync(ct);
+
+                if (!await client.DirectoryExistsAsync(path, ct))
+                {
+                    await client.CreateDirectoryAsync(path, ct);
+                }
+            }
+
+            log.LogInformation(w => w
+                .WriteProperty("action", "FTPAssetStoreConfigured")
+                .WriteProperty("path", path));
+        }
+
+        public async Task CopyAsync(string sourceFileName, string targetFileName, CancellationToken ct = default)
+        {
+            Guard.NotNullOrEmpty(sourceFileName, nameof(sourceFileName));
+            Guard.NotNullOrEmpty(targetFileName, nameof(targetFileName));
+
+            using (var client = GetFtpClient())
+            {
+                var tempPath = Path.Combine(Path.GetTempPath(), Path.GetTempFileName());
+
+                using (var stream = new FileStream(tempPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose))
+                {
+                    await DownloadAsync(client, sourceFileName, stream, ct);
+                    await UploadAsync(client, targetFileName, stream, false, ct);
+                }
+            }
+        }
+
+        public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default)
+        {
+            Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
+
+            using (var client = GetFtpClient())
+            {
+                await DownloadAsync(client, fileName, stream, ct);
+            }
+        }
+
+        public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default)
+        {
+            Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
+
+            using (var client = GetFtpClient())
+            {
+                await UploadAsync(client, fileName, stream, overwrite, ct);
+            }
+        }
+
+        private static async Task DownloadAsync(IFtpClient client, string fileName, Stream stream, CancellationToken ct)
+        {
+            try
+            {
+                await client.DownloadAsync(stream, fileName, token: ct);
+            }
+            catch (FtpException ex) when (IsNotFound(ex))
+            {
+                throw new AssetNotFoundException(fileName, ex);
+            }
+        }
+
+        private static async Task UploadAsync(IFtpClient client, string fileName, Stream stream, bool overwrite, CancellationToken ct)
+        {
+            if (!overwrite && await client.FileExistsAsync(fileName, ct))
+            {
+                throw new AssetAlreadyExistsException(fileName);
+            }
+
+            await client.UploadAsync(stream, fileName, overwrite ? FtpExists.Overwrite : FtpExists.Skip, true, null, ct);
+        }
+
+        public async Task DeleteAsync(string fileName)
+        {
+            Guard.NotNullOrEmpty(fileName, nameof(fileName));
+
+            using (var client = GetFtpClient())
+            {
+                try
+                {
+                    await client.DeleteFileAsync(fileName);
+                }
+                catch (FtpException ex)
+                {
+                    if (!IsNotFound(ex))
+                    {
+                        throw ex;
+                    }
+                }
+            }
+        }
+
+        private IFtpClient GetFtpClient()
+        {
+            var client = factory();
+
+            client.SetWorkingDirectory(path);
+            client.Connect();
+
+            return client;
+        }
+
+        private static bool IsNotFound(Exception exception)
+        {
+            if (exception is FtpCommandException command)
+            {
+                return command.CompletionCode == "550";
+            }
+
+            return exception.InnerException != null ? IsNotFound(exception.InnerException) : false;
+        }
+    }
+}

src/Squidex.Infrastructure/Assets/FolderAssetStore.cs

@@ -82,7 +82,7 @@ namespace Squidex.Infrastructure.Assets
 
         public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default)
         {
-            Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
 
             var file = GetFile(fileName);
 
@@ -101,7 +101,7 @@ namespace Squidex.Infrastructure.Assets
 
         public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default)
         {
-            Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
 
             var file = GetFile(fileName);
 
@@ -120,8 +120,6 @@ namespace Squidex.Infrastructure.Assets
 
         public Task DeleteAsync(string fileName)
         {
-            Guard.NotNullOrEmpty(fileName, nameof(fileName));
-
             var file = GetFile(fileName);
 
             file.Delete();

src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs

@@ -43,6 +43,7 @@ namespace Squidex.Infrastructure.Assets
         public virtual async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default)
         {
             Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
 
             if (!streams.TryGetValue(fileName, out var sourceStream))
             {
@@ -65,6 +66,7 @@ namespace Squidex.Infrastructure.Assets
         public virtual async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default)
         {
             Guard.NotNullOrEmpty(fileName, nameof(fileName));
+            Guard.NotNull(stream, nameof(stream));
 
             var memoryStream = new MemoryStream();
 

src/Squidex.Infrastructure/Squidex.Infrastructure.csproj

@@ -8,6 +8,7 @@
     <DebugSymbols>True</DebugSymbols>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="FluentFTP" Version="24.0.0" />
     <PackageReference Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Caching.Abstractions" Version="2.2.0" />
     <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="2.2.0" />

src/Squidex/Config/Domain/AssetServices.cs

@@ -5,6 +5,8 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System;
+using FluentFTP;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using MongoDB.Driver;
@@ -68,6 +70,24 @@ namespace Squidex.Config.Domain
                             return new MongoGridFsAssetStore(gridFsbucket);
                         })
                         .As<IAssetStore>();
+                },
+                ["Ftp"] = () =>
+                {
+                    var serverHost = config.GetRequiredValue("assetStore:ftp:serverHost");
+                    var serverPort = config.GetOptionalValue<int>("assetStore:ftp:serverPort", 21);
+
+                    var username = config.GetRequiredValue("assetStore:ftp:username");
+                    var password = config.GetRequiredValue("assetStore:ftp:password");
+
+                    var path = config.GetOptionalValue("assetStore:ftp:path", "/");
+
+                    services.AddSingletonAs(c =>
+                        {
+                            var factory = new Func<FtpClient>(() => new FtpClient(serverHost, serverPort, username, password));
+
+                            return new FTPAssetStore(factory, path, c.GetRequiredService<ISemanticLog>());
+                        })
+                        .As<IAssetStore>();
                 }
             });
 

src/Squidex/appsettings.json

@@ -200,7 +200,7 @@
     /*
      * Define the type of the read store. 
      * 
-     * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob.
+     * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob, FTP (not recommended).
      */
     "type": "Folder",
     "folder": {
@@ -241,6 +241,25 @@
        */
       "bucket": "fs"
     },
+    "ftp": {
+      /*
+       *The host of the ftp service
+       */
+      "serverHost": "",
+      /*
+       *The host of the ftp service
+       */
+      "serverPort": "21",
+      /*
+       * Credentials.
+       */
+      "username": "",
+      "password": "",
+      /*
+       * The relative or absolute path to the folder to store the assets.
+       */
+      "path": "Assets"
+    },
     /*
      * Allow to expose the url in graph ql url.
      */

tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs

@@ -48,6 +48,48 @@ namespace Squidex.Infrastructure.Assets
             await Assert.ThrowsAsync<AssetNotFoundException>(() => Sut.CopyAsync(fileName, sourceFile));
         }
 
+        [Fact]
+        public async Task Should_throw_exception_if_stream_to_download_is_null()
+        {
+            await Assert.ThrowsAsync<ArgumentNullException>(() => Sut.DownloadAsync("File", null));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_stream_to_upload_is_null()
+        {
+            await Assert.ThrowsAsync<ArgumentNullException>(() => Sut.UploadAsync("File", null));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_source_file_name_to_copy_is_empty()
+        {
+            await CheckEmpty(v => Sut.CopyAsync(v, "Target"));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_target_file_name_to_copy_is_empty()
+        {
+            await CheckEmpty(v => Sut.CopyAsync("Source", v));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_file_name_to_delete_is_empty()
+        {
+            await CheckEmpty(v => Sut.DeleteAsync(v));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_file_name_to_download_is_empty()
+        {
+            await CheckEmpty(v => Sut.DownloadAsync(v, new MemoryStream()));
+        }
+
+        [Fact]
+        public async Task Should_throw_exception_if_file_name_to_upload_is_empty()
+        {
+            await CheckEmpty(v => Sut.UploadAsync(v, new MemoryStream()));
+        }
+
         [Fact]
         public async Task Should_write_and_read_file()
         {
@@ -111,5 +153,12 @@ namespace Squidex.Infrastructure.Assets
             await Sut.DeleteAsync(sourceFile);
             await Sut.DeleteAsync(sourceFile);
         }
+
+        private async Task CheckEmpty(Func<string, Task> action)
+        {
+            await Assert.ThrowsAsync<ArgumentNullException>(() => action(null));
+            await Assert.ThrowsAsync<ArgumentException>(() => action(string.Empty));
+            await Assert.ThrowsAsync<ArgumentException>(() => action(" "));
+        }
     }
 }

tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreFixture.cs

@@ -0,0 +1,29 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using FakeItEasy;
+using FluentFTP;
+using Squidex.Infrastructure.Log;
+
+namespace Squidex.Infrastructure.Assets
+{
+    public sealed class FTPAssetStoreFixture : IDisposable
+    {
+        public FTPAssetStore AssetStore { get; }
+
+        public FTPAssetStoreFixture()
+        {
+            AssetStore = new FTPAssetStore(() => new FtpClient("localhost", 21, "test", "test"), "assets", A.Fake<ISemanticLog>());
+            AssetStore.InitializeAsync().Wait();
+        }
+
+        public void Dispose()
+        {
+        }
+    }
+}

tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreTests.cs

@@ -0,0 +1,35 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschränkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Xunit;
+
+namespace Squidex.Infrastructure.Assets
+{
+    [Trait("Category", "Dependencies")]
+    public class FTPAssetStoreTests : AssetStoreTests<FTPAssetStore>, IClassFixture<FTPAssetStoreFixture>
+    {
+        private readonly FTPAssetStoreFixture fixture;
+
+        public FTPAssetStoreTests(FTPAssetStoreFixture fixture)
+        {
+            this.fixture = fixture;
+        }
+
+        public override FTPAssetStore CreateStore()
+        {
+            return fixture.AssetStore;
+        }
+
+        [Fact]
+        public void Should_calculate_source_url()
+        {
+            var url = Sut.GeneratePublicUrl(FileName);
+
+            Assert.Null(url);
+        }
+    }
+}