tsai
/
squidex
mirror of https://github.com/Squidex/squidex.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Tus simplified. (#841) 

* Tus simplified.

* Update package.
pull/843/head
Sebastian Stehle 4 years ago
committed by GitHub
parent
85c5a41ba9
commit
3f7b255990
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 216 additions and 256 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs
  2. 99
      backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs
  3. 38
      backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs
  4. 22
      backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs
  5. 37
      backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs
  6. 28
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs
  7. 240
      backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs
  8. 4
      backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj

4
backend/src/Squidex.Domain.Apps.Entities/Assets/Commands/UpsertAsset.cs
View File

@ -14,6 +14,10 @@ namespace Squidex.Domain.Apps.Entities.Assets.Commands
{
public DomainId? ParentId { get; set; }
public bool Duplicate { get; set; } = true;
public bool OptimizeValidation { get; set; }
public UpsertAsset()
{
AssetId = DomainId.NewGuid();

99
backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs
View File

@ -42,80 +42,73 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
{
switch (context.Command)
{
case CreateAsset createAsset:
{
var tempFile = context.ContextId.ToString();
try
{
await EnrichWithHashAndUploadAsync(createAsset, tempFile);
if (!createAsset.Duplicate)
{
var existing =
await assetQuery.FindByHashAsync(contextProvider.Context,
createAsset.FileHash,
createAsset.File.FileName,
createAsset.File.FileSize);
if (existing != null)
{
context.Complete(new AssetDuplicate(existing));
case CreateAsset create:
await UploadWithDuplicateCheckAsync(context, create, create.Duplicate, next);
break;
await next(context);
return;
}
}
case UpsertAsset upsert:
await UploadWithDuplicateCheckAsync(context, upsert, upsert.Duplicate, next);
break;
await EnrichWithMetadataAsync(createAsset);
case MoveAsset move:
await base.HandleAsync(context, next);
break;
await base.HandleAsync(context, next);
}
finally
{
await assetFileStore.DeleteAsync(tempFile);
case UpdateAsset upload:
await UploadAndHandleAsync(context, upload, next);
break;
await createAsset.File.DisposeAsync();
}
default:
await base.HandleAsync(context, next);
break;
}
}
break;
}
private async Task UploadWithDuplicateCheckAsync(CommandContext context, UploadAssetCommand command, bool duplicate, NextDelegate next)
{
var tempFile = context.ContextId.ToString();
case MoveAsset move:
{
await base.HandleAsync(context, next);
try
{
await EnrichWithHashAndUploadAsync(command, tempFile);
break;
}
if (!duplicate)
{
var existing =
await assetQuery.FindByHashAsync(contextProvider.Context,
command.FileHash,
command.File.FileName,
command.File.FileSize);
case UpsertAsset upsert:
if (existing != null)
{
await UploadAndHandleAsync(context, next, upsert);
context.Complete(new AssetDuplicate(existing));
break;
await next(context);
return;
}
}
case UpdateAsset upload:
{
await UploadAndHandleAsync(context, next, upload);
await EnrichWithMetadataAsync(command);
break;
}
await base.HandleAsync(context, next);
}
finally
{
await assetFileStore.DeleteAsync(tempFile);
default:
await base.HandleAsync(context, next);
break;
await command.File.DisposeAsync();
}
}
private async Task UploadAndHandleAsync(CommandContext context, NextDelegate next, UploadAssetCommand upload)
private async Task UploadAndHandleAsync(CommandContext context, UploadAssetCommand command, NextDelegate next)
{
var tempFile = context.ContextId.ToString();
try
{
await EnrichWithHashAndUploadAsync(upload, tempFile);
await EnrichWithMetadataAsync(upload);
await EnrichWithHashAndUploadAsync(command, tempFile);
await EnrichWithMetadataAsync(command);
await base.HandleAsync(context, next);
}
@ -123,7 +116,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
{
await assetFileStore.DeleteAsync(tempFile);
await upload.File.DisposeAsync();
await command.File.DisposeAsync();
}
}

38
backend/src/Squidex/Areas/Api/Controllers/Assets/AssetsController.cs
View File

@ -248,7 +248,7 @@ namespace Squidex.Areas.Api.Controllers.Assets
if (file != null)
{
var command = CreateAssetDto.ToCommand(file);
var command = UpsertAssetDto.ToCommand(file);
var response = await InvokeCommandAsync(command);
@ -345,42 +345,6 @@ namespace Squidex.Areas.Api.Controllers.Assets
return Ok(response);
}
/// <summary>
/// Replace asset content using tus.
/// </summary>
/// <param name="app">The name of the app.</param>
/// <param name="id">The id of the asset.</param>
/// <returns>
/// 200 => Asset updated.
/// 400 => Asset request not valid.
/// 413 => Asset exceeds the maximum upload size.
/// 404 => Asset or app not found.
/// </returns>
/// <remarks>
/// Use the tus protocol to upload an asset.
/// </remarks>
[OpenApiIgnore]
[Route("apps/{app}/assets/{id}/content/tus/{**fileId}")]
[ProducesResponseType(typeof(AssetDto), StatusCodes.Status200OK)]
[AssetRequestSizeLimit]
[ApiPermissionOrAnonymous(Permissions.AppAssetsUpload)]
[ApiCosts(1)]
public async Task<IActionResult> PutAssetContentTus(string app, DomainId id)
{
var (result, file) = await assetTusRunner.InvokeAsync(HttpContext, Url.Action(null, new { app, id })!);
if (file != null)
{
var command = new UpdateAsset { File = file, AssetId = id };
var response = await InvokeCommandAsync(command);
return Ok(response);
}
return result;
}
/// <summary>
/// Update an asset.
/// </summary>

22
backend/src/Squidex/Areas/Api/Controllers/Assets/Models/CreateAssetDto.cs
View File

@ -38,28 +38,6 @@ namespace Squidex.Areas.Api.Controllers.Assets.Models
[FromQuery]
public bool Duplicate { get; set; }
public static CreateAsset ToCommand(AssetTusFile file)
{
var command = new CreateAsset { File = file };
if (file.Metadata.TryGetValue("id", out var id) && !string.IsNullOrWhiteSpace(id))
{
command.AssetId = DomainId.Create(id);
}
if (file.Metadata.TryGetValue("parentId", out var parentId) && !string.IsNullOrWhiteSpace(parentId))
{
command.ParentId = DomainId.Create(parentId);
}
if (file.Metadata.TryGetValue("duplicate", out var duplicate) && bool.TryParse(duplicate, out var parsed))
{
command.Duplicate = parsed;
}
return command;
}
public CreateAsset ToCommand(AssetFile file)
{
var command = SimpleMapper.Map(this, new CreateAsset { File = file });

37
backend/src/Squidex/Areas/Api/Controllers/Assets/Models/UpsertAssetDto.cs
View File

@ -32,6 +32,43 @@ namespace Squidex.Areas.Api.Controllers.Assets.Models
[FromQuery]
public bool Duplicate { get; set; }
public static UpsertAsset ToCommand(AssetTusFile file)
{
var command = new UpsertAsset { File = file };
bool TryGetString(string key, out string result)
{
result = null!;
var value = file.Metadata.FirstOrDefault(x => string.Equals(x.Key, key, StringComparison.OrdinalIgnoreCase)).Value;
if (!string.IsNullOrWhiteSpace(value))
{
result = value;
return true;
}
return false;
}
if (TryGetString("id", out var id))
{
command.AssetId = DomainId.Create(id);
}
if (TryGetString("parentId", out var parentId))
{
command.ParentId = DomainId.Create(parentId);
}
if (TryGetString("duplicate", out var duplicate) && bool.TryParse(duplicate, out var parsed))
{
command.Duplicate = parsed;
}
return command;
}
public UpsertAsset ToCommand(DomainId id, AssetFile file)
{
return SimpleMapper.Map(this, new UpsertAsset { File = file, AssetId = id });

28
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetCommandMiddlewareTests.cs
View File

@ -170,12 +170,38 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
[Fact]
public async Task Upsert_should_upload_file()
{
await HandleAsync(new UpsertAsset { File = file }, CreateAsset(1));
await HandleAsync(new UpsertAsset { File = file, Duplicate = false }, CreateAsset(1));
AssertAssetHasBeenUploaded(1);
AssertMetadataEnriched();
}
[Fact]
public async Task Upsert_should_not_return_duplicate_result_if_file_with_same_hash_found_but_duplicate_allowed()
{
var result = CreateAsset();
SetupSameHashAsset(file.FileName, file.FileSize, out _);
var context =
await HandleAsync(new UpsertAsset { File = file },
result);
Assert.Same(result, context.Result<IEnrichedAssetEntity>());
}
[Fact]
public async Task Upsert_should_return_duplicate_result_if_file_with_same_hash_found()
{
SetupSameHashAsset(file.FileName, file.FileSize, out var duplicate);
var context =
await HandleAsync(new UpsertAsset { File = file, Duplicate = false },
CreateAsset());
Assert.Same(duplicate, context.Result<AssetDuplicate>().Asset);
}
[Fact]
public async Task Upsert_should_calculate_hash()
{

240
backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs
View File

@ -18,6 +18,8 @@ namespace TestSuite.ApiTests
{
public class AssetTests : IClassFixture<AssetFixture>
{
private ProgressHandler progress = new ProgressHandler();
public AssetFixture _ { get; }
public AssetTests(AssetFixture fixture)
@ -46,42 +48,19 @@ namespace TestSuite.ApiTests
// STEP 1: Create asset
var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4");
var reportedException = (Exception)null;
var reportedProgress = new List<int>();
var reportedAsset = (AssetDto)null;
await using (fileParameter.Data)
{
await _.Assets.UploadNewAssetAsync(_.AppName, fileParameter, new AssetUploadOptions
{
ProgressHandler = new AssetDelegatingProgressHandler
{
OnProgressAsync = (@event, _) =>
{
reportedProgress.Add(@event.Progress);
return Task.CompletedTask;
},
OnCompletedAsync = (@event, _) =>
{
reportedAsset = @event.Asset;
return Task.CompletedTask;
},
OnFailedAsync = (@event, _) =>
{
reportedException = @event.Exception;
return Task.CompletedTask;
}
}
});
await _.Assets.UploadAssetAsync(_.AppName, fileParameter,
progress.AsOptions());
}
Assert.NotEmpty(reportedProgress);
Assert.NotNull(reportedAsset);
Assert.Null(reportedException);
Assert.NotEmpty(progress.Progress);
Assert.NotNull(progress.Asset);
Assert.Null(progress.Exception);
await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open))
{
var downloaded = await _.DownloadAsync(reportedAsset);
var downloaded = await _.DownloadAsync(progress.Asset);
// Should dowload with correct size.
Assert.Equal(stream.Length, downloaded.Length);
@ -94,58 +73,25 @@ namespace TestSuite.ApiTests
for (var i = 0; i < 5; i++)
{
// STEP 1: Create asset
progress = new ProgressHandler();
var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4");
var pausingStream = new PauseStream(fileParameter.Data, 0.5);
var pausingFile = new FileParameter(pausingStream, fileParameter.FileName, fileParameter.ContentType);
var numUploads = 0;
var reportedException = (Exception)null;
var reportedProgress = new List<int>();
var reportedAsset = (AssetDto)null;
var fileId = (string)null;
await using (pausingFile.Data)
{
using var cts = new CancellationTokenSource(5000);
while (reportedAsset == null)
while (progress.Asset == null && progress.Exception == null)
{
pausingStream.Reset();
await _.Assets.UploadNewAssetAsync(_.AppName, pausingFile, new AssetUploadOptions
{
ProgressHandler = new AssetDelegatingProgressHandler
{
OnCreatedAsync = (@event, _) =>
{
fileId = @event.FileId;
return Task.CompletedTask;
},
OnProgressAsync = (@event, _) =>
{
reportedProgress.Add(@event.Progress);
return Task.CompletedTask;
},
OnCompletedAsync = (@event, _) =>
{
reportedAsset = @event.Asset;
return Task.CompletedTask;
},
OnFailedAsync = (@event, _) =>
{
if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase))
{
reportedException = @event.Exception;
}
return Task.CompletedTask;
}
},
FileId = fileId
}, cts.Token);
Assert.Null(reportedException);
await _.Assets.UploadAssetAsync(_.AppName, pausingFile,
progress.AsOptions(), cts.Token);
await Task.Delay(50, cts.Token);
@ -153,14 +99,14 @@ namespace TestSuite.ApiTests
}
}
Assert.NotEmpty(reportedProgress);
Assert.NotNull(reportedAsset);
Assert.Null(reportedException);
Assert.NotEmpty(progress.Progress);
Assert.NotNull(progress.Asset);
Assert.Null(progress.Exception);
Assert.True(numUploads > 1);
await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open))
{
var downloaded = await _.DownloadAsync(reportedAsset);
var downloaded = await _.DownloadAsync(progress.Asset);
// Should dowload with correct size.
Assert.Equal(stream.Length, downloaded.Length);
@ -179,6 +125,20 @@ namespace TestSuite.ApiTests
Assert.Equal(id, asset_1.Id);
}
[Fact]
public async Task Should_upload_asset_with_custom_id_using_tus()
{
var id = Guid.NewGuid().ToString();
// STEP 1: Create asset
var fileParameter = FileParameter.FromPath("Assets/logo-squared.png");
await _.Assets.UploadAssetAsync(_.AppName, fileParameter,
progress.AsOptions(id));
Assert.Equal(id, progress.Asset?.Id);
}
[Fact]
public async Task Should_not_create_asset_with_custom_id_twice()
{
@ -237,41 +197,19 @@ namespace TestSuite.ApiTests
// STEP 2: Reupload asset
var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4");
var reportedException = (Exception)null;
var reportedProgress = new List<int>();
var reportedAsset = (AssetDto)null;
await using (fileParameter.Data)
{
await _.Assets.UploadExistingAssetAsync(_.AppName, asset_1.Id, fileParameter, new AssetUploadOptions
{
ProgressHandler = new AssetDelegatingProgressHandler
{
OnProgressAsync = (@event, _) =>
{
reportedProgress.Add(@event.Progress);
return Task.CompletedTask;
},
OnCompletedAsync = (@event, _) =>
{
reportedAsset = @event.Asset;
return Task.CompletedTask;
},
OnFailedAsync = (@event, _) =>
{
reportedException = @event.Exception;
return Task.CompletedTask;
}
}
});
await _.Assets.UploadAssetAsync(_.AppName, fileParameter,
progress.AsOptions(asset_1.Id));
}
Assert.NotNull(reportedAsset);
Assert.Equal(Enumerable.Range(1, 100).ToArray(), reportedProgress.ToArray());
Assert.NotNull(progress.Asset);
Assert.NotEmpty(progress.Progress);
Assert.Null(progress.Exception);
await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open))
{
var downloaded = await _.DownloadAsync(reportedAsset);
var downloaded = await _.DownloadAsync(progress.Asset);
// Should dowload with correct size.
Assert.Equal(stream.Length, downloaded.Length);
@ -288,58 +226,25 @@ namespace TestSuite.ApiTests
// STEP 2: Reupload asset
progress = new ProgressHandler();
var fileParameter = FileParameter.FromPath("Assets/SampleVideo_1280x720_1mb.mp4");
var pausingStream = new PauseStream(fileParameter.Data, 0.5);
var pausingFile = new FileParameter(pausingStream, fileParameter.FileName, fileParameter.ContentType);
var numUploads = 0;
var reportedException = (Exception)null;
var reportedProgress = new List<int>();
var reportedAsset = (AssetDto)null;
var fileId = (string)null;
await using (pausingFile.Data)
{
using var cts = new CancellationTokenSource(5000);
while (reportedAsset == null)
while (progress.Asset == null && progress.Exception == null)
{
pausingStream.Reset();
await _.Assets.UploadExistingAssetAsync(_.AppName, asset_1.Id, pausingFile, new AssetUploadOptions
{
ProgressHandler = new AssetDelegatingProgressHandler
{
OnCreatedAsync = (@event, _) =>
{
fileId = @event.FileId;
return Task.CompletedTask;
},
OnProgressAsync = (@event, _) =>
{
reportedProgress.Add(@event.Progress);
return Task.CompletedTask;
},
OnCompletedAsync = (@event, _) =>
{
reportedAsset = @event.Asset;
return Task.CompletedTask;
},
OnFailedAsync = (@event, _) =>
{
if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase))
{
reportedException = @event.Exception;
}
return Task.CompletedTask;
}
},
FileId = fileId
}, cts.Token);
Assert.Null(reportedException);
await _.Assets.UploadAssetAsync(_.AppName, pausingFile,
progress.AsOptions(asset_1.Id), cts.Token);
await Task.Delay(50, cts.Token);
@ -347,14 +252,14 @@ namespace TestSuite.ApiTests
}
}
Assert.NotEmpty(reportedProgress);
Assert.NotNull(reportedAsset);
Assert.Null(reportedException);
Assert.NotEmpty(progress.Progress);
Assert.NotNull(progress.Asset);
Assert.Null(progress.Exception);
Assert.True(numUploads > 1);
await using (var stream = new FileStream("Assets/SampleVideo_1280x720_1mb.mp4", FileMode.Open))
{
var downloaded = await _.DownloadAsync(reportedAsset);
var downloaded = await _.DownloadAsync(progress.Asset);
// Should dowload with correct size.
Assert.Equal(stream.Length, downloaded.Length);
@ -621,6 +526,59 @@ namespace TestSuite.ApiTests
Assert.NotEqual(asset_1.FileSize, asset_2.FileSize);
}
public class ProgressHandler : IAssetProgressHandler
{
public string FileId { get; private set; }
public List<int> Progress { get; } = new List<int>();
public Exception Exception { get; private set; }
public AssetDto Asset { get; private set; }
public AssetUploadOptions AsOptions(string id = null)
{
var options = default(AssetUploadOptions);
options.ProgressHandler = this;
options.FileId = FileId;
options.Id = id;
return options;
}
public Task OnCompletedAsync(AssetUploadCompletedEvent @event,
CancellationToken ct)
{
Asset = @event.Asset;
return Task.CompletedTask;
}
public Task OnCreatedAsync(AssetUploadCreatedEvent @event,
CancellationToken ct)
{
FileId = @event.FileId;
return Task.CompletedTask;
}
public Task OnProgressAsync(AssetUploadProgressEvent @event,
CancellationToken ct)
{
Progress.Add(@event.Progress);
return Task.CompletedTask;
}
public Task OnFailedAsync(AssetUploadExceptionEvent @event,
CancellationToken ct)
{
if (!@event.Exception.ToString().Contains("PAUSED", StringComparison.OrdinalIgnoreCase))
{
Exception = @event.Exception;
}
return Task.CompletedTask;
}
}
public class PauseStream : DelegateStream
{
private readonly double pauseAfter = 1;

4
backend/tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj
View File

@ -18,10 +18,10 @@
<PackageReference Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="6.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="6.0.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.ClientLibrary" Version="8.6.0" />
<PackageReference Include="Squidex.ClientLibrary" Version="8.8.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="xunit" Version="2.4.1" />
</ItemGroup>

Write Preview
Loading…
Cancel
Save