From 0f856d57c34149c0e9a2076f976f0e2dcdd264ed Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 5 Jun 2019 18:12:14 +0200 Subject: [PATCH] Manually merged FTP. Closes #360 --- README.md | 5 +- .../Assets/MongoGridFsAssetStore.cs | 6 + .../Assets/FTPAssetStore.cs | 158 ++++++++++++++++++ .../Assets/FolderAssetStore.cs | 6 +- .../Assets/MemoryAssetStore.cs | 2 + .../Squidex.Infrastructure.csproj | 1 + src/Squidex/Config/Domain/AssetServices.cs | 20 +++ src/Squidex/appsettings.json | 21 ++- .../Assets/AssetStoreTests.cs | 49 ++++++ .../Assets/FTPAssetStoreFixture.cs | 29 ++++ .../Assets/FTPAssetStoreTests.cs | 35 ++++ 11 files changed, 325 insertions(+), 7 deletions(-) create mode 100644 src/Squidex.Infrastructure/Assets/FTPAssetStore.cs create mode 100644 tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreFixture.cs create mode 100644 tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreTests.cs diff --git a/README.md b/README.md index 2484db7f7..bbf5dfe3b 100644 --- a/README.md +++ b/README.md @@ -34,9 +34,10 @@ Current Version v2.0.4. Roadmap: https://trello.com/b/KakM4F3S/squidex-roadmap ### Contributors -* [pushrbx](https://pushrbx.net/): Azure Store support. -* [cpmstars](https://www.cpmstars.com): Asset support for rich editor. * [civicplus](https://www.civicplus.com/) ([Avd6977](https://github.com/Avd6977), [dsbegnoce](https://github.com/dsbegnoche)): Google Maps support, custom regex patterns and a lot of small improvements. +* [cpmstars](https://www.cpmstars.com): Asset support for rich editor. +* [guohai](https://github.com/seamys): FTP asset store support, Email rule support, custom editors and bug fixes. +* [pushrbx](https://pushrbx.net/): Azure Store support. * [razims](https://github.com/razims): GridFS support. ## Contributing diff --git a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs index 3a86d4fd9..cde15c0da 100644 --- a/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs +++ b/src/Squidex.Infrastructure.MongoDb/Assets/MongoGridFsAssetStore.cs @@ -46,6 +46,8 @@ namespace Squidex.Infrastructure.Assets public async Task CopyAsync(string sourceFileName, string targetFileName, CancellationToken ct = default) { + Guard.NotNullOrEmpty(targetFileName, nameof(targetFileName)); + try { var sourceName = GetFileName(sourceFileName, nameof(sourceFileName)); @@ -63,6 +65,8 @@ namespace Squidex.Infrastructure.Assets public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default) { + Guard.NotNull(stream, nameof(stream)); + try { var name = GetFileName(fileName, nameof(fileName)); @@ -80,6 +84,8 @@ namespace Squidex.Infrastructure.Assets public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default) { + Guard.NotNull(stream, nameof(stream)); + try { var name = GetFileName(fileName, nameof(fileName)); diff --git a/src/Squidex.Infrastructure/Assets/FTPAssetStore.cs b/src/Squidex.Infrastructure/Assets/FTPAssetStore.cs new file mode 100644 index 000000000..8e361de32 --- /dev/null +++ b/src/Squidex.Infrastructure/Assets/FTPAssetStore.cs @@ -0,0 +1,158 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using FluentFTP; +using Squidex.Infrastructure.Log; + +namespace Squidex.Infrastructure.Assets +{ + public sealed class FTPAssetStore : IAssetStore, IInitializable + { + private readonly string path; + private readonly ISemanticLog log; + private readonly Func factory; + + public FTPAssetStore(Func factory, string path, ISemanticLog log) + { + Guard.NotNull(factory, nameof(factory)); + Guard.NotNullOrEmpty(path, nameof(path)); + Guard.NotNull(log, nameof(log)); + + this.factory = factory; + this.path = path; + + this.log = log; + } + + public string GeneratePublicUrl(string fileName) + { + return null; + } + + public async Task InitializeAsync(CancellationToken ct = default) + { + using (var client = factory()) + { + await client.ConnectAsync(ct); + + if (!await client.DirectoryExistsAsync(path, ct)) + { + await client.CreateDirectoryAsync(path, ct); + } + } + + log.LogInformation(w => w + .WriteProperty("action", "FTPAssetStoreConfigured") + .WriteProperty("path", path)); + } + + public async Task CopyAsync(string sourceFileName, string targetFileName, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(sourceFileName, nameof(sourceFileName)); + Guard.NotNullOrEmpty(targetFileName, nameof(targetFileName)); + + using (var client = GetFtpClient()) + { + var tempPath = Path.Combine(Path.GetTempPath(), Path.GetTempFileName()); + + using (var stream = new FileStream(tempPath, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose)) + { + await DownloadAsync(client, sourceFileName, stream, ct); + await UploadAsync(client, targetFileName, stream, false, ct); + } + } + } + + public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); + + using (var client = GetFtpClient()) + { + await DownloadAsync(client, fileName, stream, ct); + } + } + + public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default) + { + Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); + + using (var client = GetFtpClient()) + { + await UploadAsync(client, fileName, stream, overwrite, ct); + } + } + + private static async Task DownloadAsync(IFtpClient client, string fileName, Stream stream, CancellationToken ct) + { + try + { + await client.DownloadAsync(stream, fileName, token: ct); + } + catch (FtpException ex) when (IsNotFound(ex)) + { + throw new AssetNotFoundException(fileName, ex); + } + } + + private static async Task UploadAsync(IFtpClient client, string fileName, Stream stream, bool overwrite, CancellationToken ct) + { + if (!overwrite && await client.FileExistsAsync(fileName, ct)) + { + throw new AssetAlreadyExistsException(fileName); + } + + await client.UploadAsync(stream, fileName, overwrite ? FtpExists.Overwrite : FtpExists.Skip, true, null, ct); + } + + public async Task DeleteAsync(string fileName) + { + Guard.NotNullOrEmpty(fileName, nameof(fileName)); + + using (var client = GetFtpClient()) + { + try + { + await client.DeleteFileAsync(fileName); + } + catch (FtpException ex) + { + if (!IsNotFound(ex)) + { + throw ex; + } + } + } + } + + private IFtpClient GetFtpClient() + { + var client = factory(); + + client.SetWorkingDirectory(path); + client.Connect(); + + return client; + } + + private static bool IsNotFound(Exception exception) + { + if (exception is FtpCommandException command) + { + return command.CompletionCode == "550"; + } + + return exception.InnerException != null ? IsNotFound(exception.InnerException) : false; + } + } +} \ No newline at end of file diff --git a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs index 3df128be3..c9a3e83f0 100644 --- a/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/FolderAssetStore.cs @@ -82,7 +82,7 @@ namespace Squidex.Infrastructure.Assets public async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default) { - Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); var file = GetFile(fileName); @@ -101,7 +101,7 @@ namespace Squidex.Infrastructure.Assets public async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default) { - Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); var file = GetFile(fileName); @@ -120,8 +120,6 @@ namespace Squidex.Infrastructure.Assets public Task DeleteAsync(string fileName) { - Guard.NotNullOrEmpty(fileName, nameof(fileName)); - var file = GetFile(fileName); file.Delete(); diff --git a/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs b/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs index 10fa7fe84..20d9fc363 100644 --- a/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs +++ b/src/Squidex.Infrastructure/Assets/MemoryAssetStore.cs @@ -43,6 +43,7 @@ namespace Squidex.Infrastructure.Assets public virtual async Task DownloadAsync(string fileName, Stream stream, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); if (!streams.TryGetValue(fileName, out var sourceStream)) { @@ -65,6 +66,7 @@ namespace Squidex.Infrastructure.Assets public virtual async Task UploadAsync(string fileName, Stream stream, bool overwrite = false, CancellationToken ct = default) { Guard.NotNullOrEmpty(fileName, nameof(fileName)); + Guard.NotNull(stream, nameof(stream)); var memoryStream = new MemoryStream(); diff --git a/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj b/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj index 882061c58..5877358e2 100644 --- a/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj +++ b/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj @@ -8,6 +8,7 @@ True + diff --git a/src/Squidex/Config/Domain/AssetServices.cs b/src/Squidex/Config/Domain/AssetServices.cs index 63f813aa6..90b5ac434 100644 --- a/src/Squidex/Config/Domain/AssetServices.cs +++ b/src/Squidex/Config/Domain/AssetServices.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using FluentFTP; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using MongoDB.Driver; @@ -68,6 +70,24 @@ namespace Squidex.Config.Domain return new MongoGridFsAssetStore(gridFsbucket); }) .As(); + }, + ["Ftp"] = () => + { + var serverHost = config.GetRequiredValue("assetStore:ftp:serverHost"); + var serverPort = config.GetOptionalValue("assetStore:ftp:serverPort", 21); + + var username = config.GetRequiredValue("assetStore:ftp:username"); + var password = config.GetRequiredValue("assetStore:ftp:password"); + + var path = config.GetOptionalValue("assetStore:ftp:path", "/"); + + services.AddSingletonAs(c => + { + var factory = new Func(() => new FtpClient(serverHost, serverPort, username, password)); + + return new FTPAssetStore(factory, path, c.GetRequiredService()); + }) + .As(); } }); diff --git a/src/Squidex/appsettings.json b/src/Squidex/appsettings.json index 45c7851a7..d8c5ba74f 100644 --- a/src/Squidex/appsettings.json +++ b/src/Squidex/appsettings.json @@ -200,7 +200,7 @@ /* * Define the type of the read store. * - * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob. + * Supported: Folder (local folder), MongoDb (GridFS), GoogleCloud (hosted in Google Cloud only), AzureBlob, FTP (not recommended). */ "type": "Folder", "folder": { @@ -241,6 +241,25 @@ */ "bucket": "fs" }, + "ftp": { + /* + *The host of the ftp service + */ + "serverHost": "", + /* + *The host of the ftp service + */ + "serverPort": "21", + /* + * Credentials. + */ + "username": "", + "password": "", + /* + * The relative or absolute path to the folder to store the assets. + */ + "path": "Assets" + }, /* * Allow to expose the url in graph ql url. */ diff --git a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs index e066217e5..731944ac7 100644 --- a/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs +++ b/tests/Squidex.Infrastructure.Tests/Assets/AssetStoreTests.cs @@ -48,6 +48,48 @@ namespace Squidex.Infrastructure.Assets await Assert.ThrowsAsync(() => Sut.CopyAsync(fileName, sourceFile)); } + [Fact] + public async Task Should_throw_exception_if_stream_to_download_is_null() + { + await Assert.ThrowsAsync(() => Sut.DownloadAsync("File", null)); + } + + [Fact] + public async Task Should_throw_exception_if_stream_to_upload_is_null() + { + await Assert.ThrowsAsync(() => Sut.UploadAsync("File", null)); + } + + [Fact] + public async Task Should_throw_exception_if_source_file_name_to_copy_is_empty() + { + await CheckEmpty(v => Sut.CopyAsync(v, "Target")); + } + + [Fact] + public async Task Should_throw_exception_if_target_file_name_to_copy_is_empty() + { + await CheckEmpty(v => Sut.CopyAsync("Source", v)); + } + + [Fact] + public async Task Should_throw_exception_if_file_name_to_delete_is_empty() + { + await CheckEmpty(v => Sut.DeleteAsync(v)); + } + + [Fact] + public async Task Should_throw_exception_if_file_name_to_download_is_empty() + { + await CheckEmpty(v => Sut.DownloadAsync(v, new MemoryStream())); + } + + [Fact] + public async Task Should_throw_exception_if_file_name_to_upload_is_empty() + { + await CheckEmpty(v => Sut.UploadAsync(v, new MemoryStream())); + } + [Fact] public async Task Should_write_and_read_file() { @@ -111,5 +153,12 @@ namespace Squidex.Infrastructure.Assets await Sut.DeleteAsync(sourceFile); await Sut.DeleteAsync(sourceFile); } + + private async Task CheckEmpty(Func action) + { + await Assert.ThrowsAsync(() => action(null)); + await Assert.ThrowsAsync(() => action(string.Empty)); + await Assert.ThrowsAsync(() => action(" ")); + } } } diff --git a/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreFixture.cs b/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreFixture.cs new file mode 100644 index 000000000..ae2123934 --- /dev/null +++ b/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreFixture.cs @@ -0,0 +1,29 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using FakeItEasy; +using FluentFTP; +using Squidex.Infrastructure.Log; + +namespace Squidex.Infrastructure.Assets +{ + public sealed class FTPAssetStoreFixture : IDisposable + { + public FTPAssetStore AssetStore { get; } + + public FTPAssetStoreFixture() + { + AssetStore = new FTPAssetStore(() => new FtpClient("localhost", 21, "test", "test"), "assets", A.Fake()); + AssetStore.InitializeAsync().Wait(); + } + + public void Dispose() + { + } + } +} diff --git a/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreTests.cs b/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreTests.cs new file mode 100644 index 000000000..1ef2e53d9 --- /dev/null +++ b/tests/Squidex.Infrastructure.Tests/Assets/FTPAssetStoreTests.cs @@ -0,0 +1,35 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschränkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Xunit; + +namespace Squidex.Infrastructure.Assets +{ + [Trait("Category", "Dependencies")] + public class FTPAssetStoreTests : AssetStoreTests, IClassFixture + { + private readonly FTPAssetStoreFixture fixture; + + public FTPAssetStoreTests(FTPAssetStoreFixture fixture) + { + this.fixture = fixture; + } + + public override FTPAssetStore CreateStore() + { + return fixture.AssetStore; + } + + [Fact] + public void Should_calculate_source_url() + { + var url = Sut.GeneratePublicUrl(FileName); + + Assert.Null(url); + } + } +}