diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs index 6092be397..0b932f1ab 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs @@ -14,6 +14,10 @@ namespace Squidex.Domain.Apps.Entities.Assets.Commands { public DomainId? ParentId { get; set; } + public bool Duplicate { get; set; } = true; + + public bool OptimizeValidation { get; set; } + public UpsertAsset() { AssetId = DomainId.NewGuid(); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs index 53d8b1f7d..fcea93bcb 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs @@ -42,80 +42,73 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject { switch (context.Command) { - case CreateAsset createAsset: - { - var tempFile = context.ContextId.ToString(); - - try - { - await EnrichWithHashAndUploadAsync(createAsset, tempFile); - - if (!createAsset.Duplicate) - { - var existing = - await assetQuery.FindByHashAsync(contextProvider.Context, - createAsset.FileHash, - createAsset.File.FileName, - createAsset.File.FileSize); - - if (existing != null) - { - context.Complete(new AssetDuplicate(existing)); + case CreateAsset create: + await UploadWithDuplicateCheckAsync(context, create, create.Duplicate, next); + break; - await next(context); - return; - } - } + case UpsertAsset upsert: + await UploadWithDuplicateCheckAsync(context, upsert, upsert.Duplicate, next); + break; - await EnrichWithMetadataAsync(createAsset); + case MoveAsset move: + await base.HandleAsync(context, next); + break; - await base.HandleAsync(context, next); - } - finally - { - await assetFileStore.DeleteAsync(tempFile); + case UpdateAsset upload: + await UploadAndHandleAsync(context, upload, next); + break; - await createAsset.File.DisposeAsync(); - } + default: + await base.HandleAsync(context, next); + break; + } + } - break; - } + private async Task UploadWithDuplicateCheckAsync(CommandContext context, UploadAssetCommand command, bool duplicate, NextDelegate next) + { + var tempFile = context.ContextId.ToString(); - case MoveAsset move: - { - await base.HandleAsync(context, next); + try + { + await EnrichWithHashAndUploadAsync(command, tempFile); - break; - } + if (!duplicate) + { + var existing = + await assetQuery.FindByHashAsync(contextProvider.Context, + command.FileHash, + command.File.FileName, + command.File.FileSize); - case UpsertAsset upsert: + if (existing != null) { - await UploadAndHandleAsync(context, next, upsert); + context.Complete(new AssetDuplicate(existing)); - break; + await next(context); + return; } + } - case UpdateAsset upload: - { - await UploadAndHandleAsync(context, next, upload); + await EnrichWithMetadataAsync(command); - break; - } + await base.HandleAsync(context, next); + } + finally + { + await assetFileStore.DeleteAsync(tempFile); - default: - await base.HandleAsync(context, next); - break; + await command.File.DisposeAsync(); } } - private async Task UploadAndHandleAsync(CommandContext context, NextDelegate next, UploadAssetCommand upload) + private async Task UploadAndHandleAsync(CommandContext context, UploadAssetCommand command, NextDelegate next) { var tempFile = context.ContextId.ToString(); try { - await EnrichWithHashAndUploadAsync(upload, tempFile); - await EnrichWithMetadataAsync(upload); + await EnrichWithHashAndUploadAsync(command, tempFile); + await EnrichWithMetadataAsync(command); await base.HandleAsync(context, next); } @@ -123,7 +116,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject { await assetFileStore.DeleteAsync(tempFile); - await upload.File.DisposeAsync(); + await command.File.DisposeAsync(); } } diff --git a/backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs b/backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs index 473b27318..a01d0c0c3 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs @@ -248,7 +248,7 @@ namespace Squidex.Areas.Api.Controllers.Assets if (file != null) { - var command = CreateAssetDto.ToCommand(file); + var command = UpsertAssetDto.ToCommand(file); var response = await InvokeCommandAsync(command); @@ -345,42 +345,6 @@ namespace Squidex.Areas.Api.Controllers.Assets return Ok(response); } - /// - /// Replace asset content using tus. - /// - /// The name of the app. - /// The id of the asset. - /// - /// 200 => Asset updated. - /// 400 => Asset request not valid. - /// 413 => Asset exceeds the maximum upload size. - /// 404 => Asset or app not found. - /// - /// - /// Use the tus protocol to upload an asset. - /// - [OpenApiIgnore] - [Route("apps/{app}/assets/{id}/content/tus/{**fileId}")] - [ProducesResponseType(typeof(AssetDto), StatusCodes.Status200OK)] - [AssetRequestSizeLimit] - [ApiPermissionOrAnonymous(Permissions.AppAssetsUpload)] - [ApiCosts(1)] - public async Task PutAssetContentTus(string app, DomainId id) - { - var (result, file) = await assetTusRunner.InvokeAsync(HttpContext, Url.Action(null, new { app, id })!); - - if (file != null) - { - var command = new UpdateAsset { File = file, AssetId = id }; - - var response = await InvokeCommandAsync(command); - - return Ok(response); - } - - return result; - } - /// /// Update an asset. /// diff --git a/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs b/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs index 70b88c35f..5bd199380 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs @@ -38,28 +38,6 @@ namespace Squidex.Areas.Api.Controllers.Assets.Models [FromQuery] public bool Duplicate { get; set; } - public static CreateAsset ToCommand(AssetTusFile file) - { - var command = new CreateAsset { File = file }; - - if (file.Metadata.TryGetValue("id", out var id) && !string.IsNullOrWhiteSpace(id)) - { - command.AssetId = DomainId.Create(id); - } - - if (file.Metadata.TryGetValue("parentId", out var parentId) && !string.IsNullOrWhiteSpace(parentId)) - { - command.ParentId = DomainId.Create(parentId); - } - - if (file.Metadata.TryGetValue("duplicate", out var duplicate) && bool.TryParse(duplicate, out var parsed)) - { - command.Duplicate = parsed; - } - - return command; - } - public CreateAsset ToCommand(AssetFile file) { var command = SimpleMapper.Map(this, new CreateAsset { File = file }); diff --git a/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs b/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs index 20ca9df56..5f28e4510 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs @@ -32,6 +32,43 @@ namespace Squidex.Areas.Api.Controllers.Assets.Models [FromQuery] public bool Duplicate { get; set; } + public static UpsertAsset ToCommand(AssetTusFile file) + { + var command = new UpsertAsset { File = file }; + + bool TryGetString(string key, out string result) + { + result = null!; + + var value = file.Metadata.FirstOrDefault(x => string.Equals(x.Key, key, StringComparison.OrdinalIgnoreCase)).Value; + + if (!string.IsNullOrWhiteSpace(value)) + { + result = value; + return true; + } + + return false; + } + + if (TryGetString("id", out var id)) + { + command.AssetId = DomainId.Create(id); + } + + if (TryGetString("parentId", out var parentId)) + { + command.ParentId = DomainId.Create(parentId); + } + + if (TryGetString("duplicate", out var duplicate) && bool.TryParse(duplicate, out var parsed)) + { + command.Duplicate = parsed; + } + + return command; + } + public UpsertAsset ToCommand(DomainId id, AssetFile file) { return SimpleMapper.Map(this, new UpsertAsset { File = file, AssetId = id }); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs index 3b3dc36dd..a95ba7305 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs @@ -170,12 +170,38 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject [Fact] public async Task Upsert_should_upload_file() { - await HandleAsync(new UpsertAsset { File = file }, CreateAsset(1)); + await HandleAsync(new UpsertAsset { File = file, Duplicate = false }, CreateAsset(1)); AssertAssetHasBeenUploaded(1); AssertMetadataEnriched(); } + [Fact] + public async Task Upsert_should_not_return_duplicate_result_if_file_with_same_hash_found_but_duplicate_allowed() + { + var result = CreateAsset(); + + SetupSameHashAsset(file.FileName, file.FileSize, out _); + + var context = + await HandleAsync(new UpsertAsset { File = file }, + result); + + Assert.Same(result, context.Result()); + } + + [Fact] + public async Task Upsert_should_return_duplicate_result_if_file_with_same_hash_found() + { + SetupSameHashAsset(file.FileName, file.FileSize, out var duplicate); + + var context = + await HandleAsync(new UpsertAsset { File = file, Duplicate = false }, + CreateAsset()); + + Assert.Same(duplicate, context.Result().Asset); + } + [Fact] public async Task Upsert_should_calculate_hash() { diff --git a/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs b/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs index f04bda410..51ee085a2 100644 --- a/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs +++ b/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs @@ -18,6 +18,8 @@ namespace TestSuite.ApiTests { public class AssetTests : IClassFixture { + private ProgressHandler progress = new ProgressHandler(); + public AssetFixture _ { get; } public AssetTests(AssetFixture fixture) @@ -46,42 +48,19 @@ namespace TestSuite.ApiTests // STEP 1: Create asset var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4"); - var reportedException = (Exception)null; - var reportedProgress = new List(); - var reportedAsset = (AssetDto)null; - await using (fileParameter.Data) { - await _.Assets.UploadNewAssetAsync(_.AppName, fileParameter, new AssetUploadOptions - { - ProgressHandler = new AssetDelegatingProgressHandler - { - OnProgressAsync = (@event, _) => - { - reportedProgress.Add(@event.Progress); - return Task.CompletedTask; - }, - OnCompletedAsync = (@event, _) => - { - reportedAsset = @event.Asset; - return Task.CompletedTask; - }, - OnFailedAsync = (@event, _) => - { - reportedException = @event.Exception; - return Task.CompletedTask; - } - } - }); + await _.Assets.UploadAssetAsync(_.AppName, fileParameter, + progress.AsOptions()); } - Assert.NotEmpty(reportedProgress); - Assert.NotNull(reportedAsset); - Assert.Null(reportedException); + Assert.NotEmpty(progress.Progress); + Assert.NotNull(progress.Asset); + Assert.Null(progress.Exception); await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open)) { - var downloaded = await _.DownloadAsync(reportedAsset); + var downloaded = await _.DownloadAsync(progress.Asset); // Should dowload with correct size. Assert.Equal(stream.Length, downloaded.Length); @@ -94,58 +73,25 @@ namespace TestSuite.ApiTests for (var i = 0; i < 5; i++) { // STEP 1: Create asset + progress = new ProgressHandler(); + var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4"); var pausingStream = new PauseStream(fileParameter.Data, 0.5); var pausingFile = new FileParameter(pausingStream, fileParameter.FileName, fileParameter.ContentType); var numUploads = 0; - var reportedException = (Exception)null; - var reportedProgress = new List(); - var reportedAsset = (AssetDto)null; - var fileId = (string)null; await using (pausingFile.Data) { using var cts = new CancellationTokenSource(5000); - while (reportedAsset == null) + while (progress.Asset == null && progress.Exception == null) { pausingStream.Reset(); - await _.Assets.UploadNewAssetAsync(_.AppName, pausingFile, new AssetUploadOptions - { - ProgressHandler = new AssetDelegatingProgressHandler - { - OnCreatedAsync = (@event, _) => - { - fileId = @event.FileId; - return Task.CompletedTask; - }, - OnProgressAsync = (@event, _) => - { - reportedProgress.Add(@event.Progress); - return Task.CompletedTask; - }, - OnCompletedAsync = (@event, _) => - { - reportedAsset = @event.Asset; - return Task.CompletedTask; - }, - OnFailedAsync = (@event, _) => - { - if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase)) - { - reportedException = @event.Exception; - } - - return Task.CompletedTask; - } - }, - FileId = fileId - }, cts.Token); - - Assert.Null(reportedException); + await _.Assets.UploadAssetAsync(_.AppName, pausingFile, + progress.AsOptions(), cts.Token); await Task.Delay(50, cts.Token); @@ -153,14 +99,14 @@ namespace TestSuite.ApiTests } } - Assert.NotEmpty(reportedProgress); - Assert.NotNull(reportedAsset); - Assert.Null(reportedException); + Assert.NotEmpty(progress.Progress); + Assert.NotNull(progress.Asset); + Assert.Null(progress.Exception); Assert.True(numUploads > 1); await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open)) { - var downloaded = await _.DownloadAsync(reportedAsset); + var downloaded = await _.DownloadAsync(progress.Asset); // Should dowload with correct size. Assert.Equal(stream.Length, downloaded.Length); @@ -179,6 +125,20 @@ namespace TestSuite.ApiTests Assert.Equal(id, asset_1.Id); } + [Fact] + public async Task Should_upload_asset_with_custom_id_using_tus() + { + var id = Guid.NewGuid().ToString(); + + // STEP 1: Create asset + var fileParameter = FileParameter.FromPath("Assets/logo-squared.png"); + + await _.Assets.UploadAssetAsync(_.AppName, fileParameter, + progress.AsOptions(id)); + + Assert.Equal(id, progress.Asset?.Id); + } + [Fact] public async Task Should_not_create_asset_with_custom_id_twice() { @@ -237,41 +197,19 @@ namespace TestSuite.ApiTests // STEP 2: Reupload asset var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4"); - var reportedException = (Exception)null; - var reportedProgress = new List(); - var reportedAsset = (AssetDto)null; - await using (fileParameter.Data) { - await _.Assets.UploadExistingAssetAsync(_.AppName, asset_1.Id, fileParameter, new AssetUploadOptions - { - ProgressHandler = new AssetDelegatingProgressHandler - { - OnProgressAsync = (@event, _) => - { - reportedProgress.Add(@event.Progress); - return Task.CompletedTask; - }, - OnCompletedAsync = (@event, _) => - { - reportedAsset = @event.Asset; - return Task.CompletedTask; - }, - OnFailedAsync = (@event, _) => - { - reportedException = @event.Exception; - return Task.CompletedTask; - } - } - }); + await _.Assets.UploadAssetAsync(_.AppName, fileParameter, + progress.AsOptions(asset_1.Id)); } - Assert.NotNull(reportedAsset); - Assert.Equal(Enumerable.Range(1, 100).ToArray(), reportedProgress.ToArray()); + Assert.NotNull(progress.Asset); + Assert.NotEmpty(progress.Progress); + Assert.Null(progress.Exception); await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open)) { - var downloaded = await _.DownloadAsync(reportedAsset); + var downloaded = await _.DownloadAsync(progress.Asset); // Should dowload with correct size. Assert.Equal(stream.Length, downloaded.Length); @@ -288,58 +226,25 @@ namespace TestSuite.ApiTests // STEP 2: Reupload asset + progress = new ProgressHandler(); + var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4"); var pausingStream = new PauseStream(fileParameter.Data, 0.5); var pausingFile = new FileParameter(pausingStream, fileParameter.FileName, fileParameter.ContentType); var numUploads = 0; - var reportedException = (Exception)null; - var reportedProgress = new List(); - var reportedAsset = (AssetDto)null; - var fileId = (string)null; await using (pausingFile.Data) { using var cts = new CancellationTokenSource(5000); - while (reportedAsset == null) + while (progress.Asset == null && progress.Exception == null) { pausingStream.Reset(); - await _.Assets.UploadExistingAssetAsync(_.AppName, asset_1.Id, pausingFile, new AssetUploadOptions - { - ProgressHandler = new AssetDelegatingProgressHandler - { - OnCreatedAsync = (@event, _) => - { - fileId = @event.FileId; - return Task.CompletedTask; - }, - OnProgressAsync = (@event, _) => - { - reportedProgress.Add(@event.Progress); - return Task.CompletedTask; - }, - OnCompletedAsync = (@event, _) => - { - reportedAsset = @event.Asset; - return Task.CompletedTask; - }, - OnFailedAsync = (@event, _) => - { - if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase)) - { - reportedException = @event.Exception; - } - - return Task.CompletedTask; - } - }, - FileId = fileId - }, cts.Token); - - Assert.Null(reportedException); + await _.Assets.UploadAssetAsync(_.AppName, pausingFile, + progress.AsOptions(asset_1.Id), cts.Token); await Task.Delay(50, cts.Token); @@ -347,14 +252,14 @@ namespace TestSuite.ApiTests } } - Assert.NotEmpty(reportedProgress); - Assert.NotNull(reportedAsset); - Assert.Null(reportedException); + Assert.NotEmpty(progress.Progress); + Assert.NotNull(progress.Asset); + Assert.Null(progress.Exception); Assert.True(numUploads > 1); await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open)) { - var downloaded = await _.DownloadAsync(reportedAsset); + var downloaded = await _.DownloadAsync(progress.Asset); // Should dowload with correct size. Assert.Equal(stream.Length, downloaded.Length); @@ -621,6 +526,59 @@ namespace TestSuite.ApiTests Assert.NotEqual(asset_1.FileSize, asset_2.FileSize); } + public class ProgressHandler : IAssetProgressHandler + { + public string FileId { get; private set; } + + public List Progress { get; } = new List(); + + public Exception Exception { get; private set; } + + public AssetDto Asset { get; private set; } + + public AssetUploadOptions AsOptions(string id = null) + { + var options = default(AssetUploadOptions); + options.ProgressHandler = this; + options.FileId = FileId; + options.Id = id; + + return options; + } + + public Task OnCompletedAsync(AssetUploadCompletedEvent @event, + CancellationToken ct) + { + Asset = @event.Asset; + return Task.CompletedTask; + } + + public Task OnCreatedAsync(AssetUploadCreatedEvent @event, + CancellationToken ct) + { + FileId = @event.FileId; + return Task.CompletedTask; + } + + public Task OnProgressAsync(AssetUploadProgressEvent @event, + CancellationToken ct) + { + Progress.Add(@event.Progress); + return Task.CompletedTask; + } + + public Task OnFailedAsync(AssetUploadExceptionEvent @event, + CancellationToken ct) + { + if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase)) + { + Exception = @event.Exception; + } + + return Task.CompletedTask; + } + } + public class PauseStream : DelegateStream { private readonly double pauseAfter = 1; diff --git a/backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj b/backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj index 3b2715d9e..7f474b612 100644 --- a/backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj +++ b/backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj @@ -18,10 +18,10 @@ - + - +