From cd169a1377aa1324e02ce5280debc3ba0ef2ca5a Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 22 Jun 2021 10:46:06 +0200 Subject: [PATCH] Encoding support for asset text. --- .../Contents/GeoJsonValue.cs | 9 +- .../Assets/AssetExtensions.cs | 30 ++ .../Assets/AssetsFluidExtension.cs | 21 +- .../Assets/AssetsJintExtension.cs | 30 +- .../ObjectPool/DefaultPools.cs | 6 +- .../MemoryStreamPooledObjectPolicy.cs | 36 --- .../Squidex.Infrastructure.csproj | 1 + .../Assets/AssetsFluidExtensionTests.cs | 262 +++++++++++------- .../Assets/AssetsJintExtensionTests.cs | 239 ++++++++++------ 9 files changed, 366 insertions(+), 268 deletions(-) delete mode 100644 backend/src/Squidex.Infrastructure/ObjectPool/MemoryStreamPooledObjectPolicy.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs index d8de7acc9..9dc3c01df 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs @@ -12,6 +12,7 @@ using Squidex.Infrastructure.Json; using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.ObjectPool; using Squidex.Infrastructure.Validation; +using System.IO; namespace Squidex.Domain.Apps.Core.Contents { @@ -28,9 +29,7 @@ namespace Squidex.Domain.Apps.Core.Contents { try { - var stream = DefaultPools.MemoryStream.Get(); - - try + using (var stream = DefaultPools.MemoryStream.GetStream()) { serializer.Serialize(value, stream, true); @@ -40,10 +39,6 @@ namespace Squidex.Domain.Apps.Core.Contents return GeoJsonParseResult.Success; } - finally - { - DefaultPools.MemoryStream.Return(stream); - } } catch { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetExtensions.cs index 44feae209..b31b7cf0e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetExtensions.cs @@ -5,6 +5,12 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using System.Text; +using System.Threading.Tasks; +using Squidex.Infrastructure; +using Squidex.Infrastructure.ObjectPool; + namespace Squidex.Domain.Apps.Entities.Assets { public static class AssetExtensions @@ -20,5 +26,29 @@ namespace Squidex.Domain.Apps.Entities.Assets { return builder.WithBoolean(HeaderNoEnrichment, value); } + + public static async Task GetTextAsync(this IAssetFileStore assetFileStore, DomainId appId, DomainId id, long fileVersion, string? encoding) + { + using (var stream = DefaultPools.MemoryStream.GetStream()) + { + await assetFileStore.DownloadAsync(appId, id, fileVersion, stream); + + stream.Position = 0; + + var bytes = stream.ToArray(); + + switch (encoding?.ToLowerInvariant()) + { + case "base64": + return Convert.ToBase64String(bytes); + case "ascii": + return Encoding.ASCII.GetString(bytes); + case "unicode": + return Encoding.Unicode.GetString(bytes); + default: + return Encoding.UTF8.GetString(bytes); + } + } + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs index ff0c0a1b2..8d204e737 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs @@ -18,7 +18,6 @@ using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Templates; using Squidex.Domain.Apps.Core.ValidateContent; using Squidex.Infrastructure; -using Squidex.Infrastructure.ObjectPool; namespace Squidex.Domain.Apps.Entities.Assets { @@ -122,24 +121,10 @@ namespace Squidex.Domain.Apps.Entities.Assets return ErrorTooBig; } - var tempStream = DefaultPools.MemoryStream.Get(); - try - { - await assetFileStore!.DownloadAsync(appId, id, fileVersion, tempStream); - - tempStream.Position = 0; - - using (var reader = new StreamReader(tempStream, leaveOpen: true)) - { - var text = reader.ReadToEnd(); + var encoding = arguments.At(0).ToStringValue()?.ToUpperInvariant(); + var encoded = await assetFileStore.GetTextAsync(appId, id, fileVersion, encoding); - return new StringValue(text); - } - } - finally - { - DefaultPools.MemoryStream.Return(tempStream); - } + return new StringValue(encoded); } switch (objectValue.ToObjectValue()) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsJintExtension.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsJintExtension.cs index 85011f098..d79a7478e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsJintExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsJintExtension.cs @@ -7,7 +7,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Security.Claims; using System.Threading.Tasks; @@ -19,7 +18,6 @@ using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Scripting; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Infrastructure; -using Squidex.Infrastructure.ObjectPool; using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Assets @@ -27,6 +25,7 @@ namespace Squidex.Domain.Apps.Entities.Assets public sealed class AssetsJintExtension : IJintExtension { private delegate void GetAssetsDelegate(JsValue references, Action callback); + private delegate void GetAssetTextDelegate(JsValue references, Action callback, JsValue encoding); private readonly IServiceProvider serviceProvider; public AssetsJintExtension(IServiceProvider serviceProvider) @@ -62,17 +61,17 @@ namespace Squidex.Domain.Apps.Entities.Assets private void AddAssetText(ExecutionContext context) { - var action = new GetAssetsDelegate((references, callback) => GetAssetText(context, references, callback)); + var action = new GetAssetTextDelegate((references, callback, encoding) => GetText(context, references, callback, encoding)); context.Engine.SetValue("getAssetText", action); } - private void GetAssetText(ExecutionContext context, JsValue input, Action callback) + private void GetText(ExecutionContext context, JsValue input, Action callback, JsValue encoding) { - GetAssetTextCore(context, input, callback).Forget(); + GetTextAsync(context, input, callback, encoding).Forget(); } - private async Task GetAssetTextCore(ExecutionContext context, JsValue input, Action callback) + private async Task GetTextAsync(ExecutionContext context, JsValue input, Action callback, JsValue encoding) { Guard.NotNull(callback, nameof(callback)); @@ -96,24 +95,9 @@ namespace Squidex.Domain.Apps.Entities.Assets { var assetFileStore = serviceProvider.GetRequiredService(); - var tempStream = DefaultPools.MemoryStream.Get(); - try - { - await assetFileStore!.DownloadAsync(appId, id, fileVersion, tempStream, default, context.CancellationToken); - - tempStream.Position = 0; - - using (var reader = new StreamReader(tempStream, leaveOpen: true)) - { - var text = reader.ReadToEnd(); + var encoded = await assetFileStore.GetTextAsync(appId, id, fileVersion, encoding?.ToString()); - callback(JsValue.FromObject(context.Engine, text)); - } - } - finally - { - DefaultPools.MemoryStream.Return(tempStream); - } + callback(JsValue.FromObject(context.Engine, encoded)); } catch (Exception ex) { diff --git a/backend/src/Squidex.Infrastructure/ObjectPool/DefaultPools.cs b/backend/src/Squidex.Infrastructure/ObjectPool/DefaultPools.cs index 7335bc38e..c782f0aff 100644 --- a/backend/src/Squidex.Infrastructure/ObjectPool/DefaultPools.cs +++ b/backend/src/Squidex.Infrastructure/ObjectPool/DefaultPools.cs @@ -5,16 +5,16 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.IO; using System.Text; using Microsoft.Extensions.ObjectPool; +using Microsoft.IO; namespace Squidex.Infrastructure.ObjectPool { public static class DefaultPools { - public static readonly ObjectPool MemoryStream = - new DefaultObjectPool(new MemoryStreamPooledObjectPolicy()); + public static readonly RecyclableMemoryStreamManager MemoryStream = + new RecyclableMemoryStreamManager(); public static readonly ObjectPool StringBuilder = new DefaultObjectPool(new StringBuilderPooledObjectPolicy()); diff --git a/backend/src/Squidex.Infrastructure/ObjectPool/MemoryStreamPooledObjectPolicy.cs b/backend/src/Squidex.Infrastructure/ObjectPool/MemoryStreamPooledObjectPolicy.cs deleted file mode 100644 index 5370cb0d1..000000000 --- a/backend/src/Squidex.Infrastructure/ObjectPool/MemoryStreamPooledObjectPolicy.cs +++ /dev/null @@ -1,36 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.IO; -using Microsoft.Extensions.ObjectPool; - -namespace Squidex.Infrastructure.ObjectPool -{ - public sealed class MemoryStreamPooledObjectPolicy : PooledObjectPolicy - { - public int InitialCapacity { get; set; } = 100; - - public int MaximumRetainedCapacity { get; set; } = 4 * 1024; - - public override MemoryStream Create() - { - return new MemoryStream(InitialCapacity); - } - - public override bool Return(MemoryStream obj) - { - if (obj.Capacity > MaximumRetainedCapacity) - { - return false; - } - - obj.Position = 0; - - return true; - } - } -} diff --git a/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj b/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj index 9d523bccf..f893792ec 100644 --- a/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj +++ b/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj @@ -14,6 +14,7 @@ + all diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs index 8021bd2b3..629b3ec87 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs @@ -53,42 +53,40 @@ namespace Squidex.Domain.Apps.Entities.Assets [Fact] public async Task Should_resolve_assets_in_loop() { - var assetId1 = DomainId.NewGuid(); - var asset1 = CreateAsset(assetId1, 1); - var assetId2 = DomainId.NewGuid(); - var asset2 = CreateAsset(assetId2, 2); + var (vars, assets) = SetupAssetsVars(); - var @event = new EnrichedContentEvent - { - Data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId1, assetId2))), - AppId = appId - }; + var template = @" + {% for id in event.data.assets.iv %} + {% asset 'ref', id %} + Text: {{ ref.fileName }} {{ ref.id }} + {% endfor %} + "; - A.CallTo(() => assetQuery.FindAsync(A._, assetId1, EtagVersion.Any, A._)) - .Returns(asset1); + var expected = $@" + Text: {assets[0].FileName} {assets[0].Id} + Text: {assets[1].FileName} {assets[1].Id} + "; - A.CallTo(() => assetQuery.FindAsync(A._, assetId2, EtagVersion.Any, A._)) - .Returns(asset2); + var result = await sut.RenderAsync(template, vars); - var vars = new TemplateVars - { - ["event"] = @event - }; + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_assets_in_loop_with_filter() + { + var (vars, assets) = SetupAssetsVars(); var template = @" {% for id in event.data.assets.iv %} - {% asset 'ref', id %} + {% assign ref = id | asset %} Text: {{ ref.fileName }} {{ ref.id }} {% endfor %} "; var expected = $@" - Text: file1.jpg {assetId1} - Text: file2.jpg {assetId2} + Text: {assets[0].FileName} {assets[0].Id} + Text: {assets[1].FileName} {assets[1].Id} "; var result = await sut.RenderAsync(template, vars); @@ -97,44 +95,40 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_assets_in_loop_with_filter() + public async Task Should_resolve_asset_text() { - var assetId1 = DomainId.NewGuid(); - var asset1 = CreateAsset(assetId1, 1); - var assetId2 = DomainId.NewGuid(); - var asset2 = CreateAsset(assetId2, 2); + var (vars, asset) = SetupAssetVars(); - var @event = new EnrichedContentEvent - { - Data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId1, assetId2))), - AppId = appId - }; + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); - A.CallTo(() => assetQuery.FindAsync(A._, assetId1, EtagVersion.Any, A._)) - .Returns(asset1); + var template = @" + {% assign ref = event.data.assets.iv[0] | asset %} + Text: {{ ref | assetText }} + "; - A.CallTo(() => assetQuery.FindAsync(A._, assetId2, EtagVersion.Any, A._)) - .Returns(asset2); + var expected = $@" + Text: Hello Asset + "; - var vars = new TemplateVars - { - ["event"] = @event - }; + var result = await sut.RenderAsync(template, vars); + + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_asset_text_with_utf8() + { + var (vars, asset) = SetupAssetVars(); + + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); var template = @" - {% for id in event.data.assets.iv %} - {% assign ref = id | asset %} - Text: {{ ref.fileName }} {{ ref.id }} - {% endfor %} + {% assign ref = event.data.assets.iv[0] | asset %} + Text: {{ ref | assetText: 'utf8' }} "; var expected = $@" - Text: file1.jpg {assetId1} - Text: file2.jpg {assetId2} + Text: Hello Asset "; var result = await sut.RenderAsync(template, vars); @@ -143,40 +137,36 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_asset_text() + public async Task Should_resolve_asset_text_with_unicode() { - var assetId = DomainId.NewGuid(); - var asset = CreateAsset(assetId, 1); + var (vars, asset) = SetupAssetVars(); - var @event = new EnrichedContentEvent - { - Data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId))), - AppId = appId - }; + SetupText(asset.Id, Encoding.Unicode.GetBytes("Hello Asset")); - A.CallTo(() => assetQuery.FindAsync(A._, assetId, EtagVersion.Any, A._)) - .Returns(asset); + var template = @" + {% assign ref = event.data.assets.iv[0] | asset %} + Text: {{ ref | assetText: 'unicode' }} + "; - A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, assetId, asset.FileVersion, A._, A._, A._)) - .Invokes(x => - { - var stream = x.GetArgument(3)!; + var expected = $@" + Text: Hello Asset + "; - stream.Write(Encoding.UTF8.GetBytes("Hello Asset")); - }); + var result = await sut.RenderAsync(template, vars); - var vars = new TemplateVars - { - ["event"] = @event - }; + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_asset_text_with_ascii() + { + var (vars, asset) = SetupAssetVars(); + + SetupText(asset.Id, Encoding.ASCII.GetBytes("Hello Asset")); var template = @" {% assign ref = event.data.assets.iv[0] | asset %} - Text: {{ ref | assetText }} + Text: {{ ref | assetText: 'ascii' }} "; var expected = $@" @@ -189,28 +179,30 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_not_resolve_asset_text_if_too_big() + public async Task Should_resolve_asset_text_with_base64() { - var assetId = DomainId.NewGuid(); - var asset = CreateAsset(assetId, 1, 1_000_000); + var (vars, asset) = SetupAssetVars(); - var @event = new EnrichedContentEvent - { - Data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId))), - AppId = appId - }; + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); - A.CallTo(() => assetQuery.FindAsync(A._, assetId, EtagVersion.Any, A._)) - .Returns(asset); + var template = @" + {% assign ref = event.data.assets.iv[0] | asset %} + Text: {{ ref | assetText: 'base64' }} + "; - var vars = new TemplateVars - { - ["event"] = @event - }; + var expected = $@" + Text: SGVsbG8gQXNzZXQ= + "; + + var result = await sut.RenderAsync(template, vars); + + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_not_resolve_asset_text_if_too_big() + { + var (vars, _) = SetupAssetVars(1_000_000); var template = @" {% assign ref = event.data.assets.iv[0] | asset %} @@ -240,13 +232,7 @@ namespace Squidex.Domain.Apps.Entities.Assets AppId = appId }; - A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, @event.Id, @event.FileVersion, A._, A._, A._)) - .Invokes(x => - { - var stream = x.GetArgument(3)!; - - stream.Write(Encoding.UTF8.GetBytes("Hello Asset")); - }); + SetupText(@event.Id, Encoding.UTF8.GetBytes("Hello Asset")); var vars = new TemplateVars { @@ -267,7 +253,7 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_asset_text_from_event_if_too_big() + public async Task Should_not_resolve_asset_text_from_event_if_too_big() { var @event = new EnrichedAssetEvent { @@ -298,6 +284,76 @@ namespace Squidex.Domain.Apps.Entities.Assets .MustNotHaveHappened(); } + private void SetupText(DomainId id, byte[] bytes) + { + A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, id, 0, A._, A._, A._)) + .Invokes(x => + { + var stream = x.GetArgument(3)!; + + stream.Write(bytes); + }); + } + + private (TemplateVars, IAssetEntity) SetupAssetVars(int fileSize = 100) + { + var assetId = DomainId.NewGuid(); + var asset = CreateAsset(assetId, 1, fileSize); + + var @event = new EnrichedContentEvent + { + Data = + new ContentData() + .AddField("assets", + new ContentFieldData() + .AddInvariant(JsonValue.Array(assetId))), + AppId = appId + }; + + A.CallTo(() => assetQuery.FindAsync(A._, assetId, EtagVersion.Any, A._)) + .Returns(asset); + + SetupText(@event.Id, Encoding.UTF8.GetBytes("Hello Asset")); + + var vars = new TemplateVars + { + ["event"] = @event + }; + + return (vars, asset); + } + + private (TemplateVars, IAssetEntity[]) SetupAssetsVars(int fileSize = 100) + { + var assetId1 = DomainId.NewGuid(); + var asset1 = CreateAsset(assetId1, 1, fileSize); + var assetId2 = DomainId.NewGuid(); + var asset2 = CreateAsset(assetId2, 2, fileSize); + + var @event = new EnrichedContentEvent + { + Data = + new ContentData() + .AddField("assets", + new ContentFieldData() + .AddInvariant(JsonValue.Array(assetId1, assetId2))), + AppId = appId + }; + + A.CallTo(() => assetQuery.FindAsync(A._, assetId1, EtagVersion.Any, A._)) + .Returns(asset1); + + A.CallTo(() => assetQuery.FindAsync(A._, assetId2, EtagVersion.Any, A._)) + .Returns(asset2); + + var vars = new TemplateVars + { + ["event"] = @event + }; + + return (vars, new[] { asset1, asset2 }); + } + private IEnrichedAssetEntity CreateAsset(DomainId assetId, int index, int fileSize = 100) { return new AssetEntity diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsJintExtensionTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsJintExtensionTests.cs index f4e749d4e..8096e7380 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsJintExtensionTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsJintExtensionTests.cs @@ -62,32 +62,40 @@ namespace Squidex.Domain.Apps.Entities.Assets [Fact] public async Task Should_resolve_asset() { - var assetId1 = DomainId.NewGuid(); - var asset1 = CreateAsset(assetId1, 1); + var (vars, asset) = SetupAssetVars(); - var user = new ClaimsPrincipal(); + var script = @" + getAsset(data.assets.iv[0], function (assets) { + var result1 = `Text: ${assets[0].fileName} ${assets[0].id}`; - var data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId1))); + complete(`${result1}`); + });"; - A.CallTo(() => assetQuery.QueryAsync( - A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId1), A._)) - .Returns(ResultList.CreateFrom(1, asset1)); + var expected = $@" + Text: {asset.FileName} {asset.Id} + "; - var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + var result = (await sut.ExecuteAsync(vars, script)).ToString(); + + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_assets() + { + var (vars, assets) = SetupAssetsVars(); var script = @" - getAsset(data.assets.iv[0], function (assets) { - var result1 = `Text: ${assets[0].fileName}`; + getAssets(data.assets.iv, function (assets) { + var result1 = `Text: ${assets[0].fileName} ${assets[0].id}`; + var result2 = `Text: ${assets[1].fileName} ${assets[1].id}`; - complete(`${result1}`); + complete(`${result1}\n${result2}`); });"; - var expected = @" - Text: file1.jpg + var expected = $@" + Text: {assets[0].FileName} {assets[0].Id} + Text: {assets[1].FileName} {assets[1].Id} "; var result = (await sut.ExecuteAsync(vars, script)).ToString(); @@ -96,38 +104,48 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_assets() + public async Task Should_resolve_asset_text() { - var assetId1 = DomainId.NewGuid(); - var asset1 = CreateAsset(assetId1, 1); - var assetId2 = DomainId.NewGuid(); - var asset2 = CreateAsset(assetId1, 2); + var (vars, asset) = SetupAssetVars(); - var user = new ClaimsPrincipal(); + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); - var data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId1, assetId2))); + var script = @" + getAssets(data.assets.iv, function (assets) { + getAssetText(assets[0], function (text) { + var result = `Text: ${text}`; - A.CallTo(() => assetQuery.QueryAsync( - A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId1, assetId2), A._)) - .Returns(ResultList.CreateFrom(2, asset1, asset2)); + complete(result); + }); + });"; - var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + var expected = @" + Text: Hello Asset + "; + + var result = (await sut.ExecuteAsync(vars, script)).ToString(); + + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_asset_text_with_utf8() + { + var (vars, asset) = SetupAssetVars(); + + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); var script = @" getAssets(data.assets.iv, function (assets) { - var result1 = `Text: ${assets[0].fileName}`; - var result2 = `Text: ${assets[1].fileName}`; + getAssetText(assets[0], function (text) { + var result = `Text: ${text}`; - complete(`${result1}\n${result2}`); + complete(result); + }, 'utf8'); });"; var expected = @" - Text: file1.jpg - Text: file2.jpg + Text: Hello Asset "; var result = (await sut.ExecuteAsync(vars, script)).ToString(); @@ -136,32 +154,36 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_asset_text() + public async Task Should_resolve_asset_text_with_unicode() { - var assetId = DomainId.NewGuid(); - var asset = CreateAsset(assetId, 1); + var (vars, asset) = SetupAssetVars(); - var user = new ClaimsPrincipal(); + SetupText(asset.Id, Encoding.Unicode.GetBytes("Hello Asset")); - var data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId))); + var script = @" + getAssets(data.assets.iv, function (assets) { + getAssetText(assets[0], function (text) { + var result = `Text: ${text}`; - A.CallTo(() => assetQuery.QueryAsync( - A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId), A._)) - .Returns(ResultList.CreateFrom(2, asset)); + complete(result); + }, 'unicode'); + });"; - A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, assetId, asset.FileVersion, A._, A._, A._)) - .Invokes(x => - { - var stream = x.GetArgument(3)!; + var expected = @" + Text: Hello Asset + "; - stream.Write(Encoding.UTF8.GetBytes("Hello Asset")); - }); + var result = (await sut.ExecuteAsync(vars, script)).ToString(); - var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_resolve_asset_text_with_ascii() + { + var (vars, asset) = SetupAssetVars(); + + SetupText(asset.Id, Encoding.ASCII.GetBytes("Hello Asset")); var script = @" getAssets(data.assets.iv, function (assets) { @@ -169,7 +191,7 @@ namespace Squidex.Domain.Apps.Entities.Assets var result = `Text: ${text}`; complete(result); - }); + }, 'ascii'); });"; var expected = @" @@ -182,24 +204,34 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_not_resolve_asset_text_if_too_big() + public async Task Should_resolve_asset_text_with_base64() { - var assetId = DomainId.NewGuid(); - var asset = CreateAsset(assetId, 1, 1_000_000); + var (vars, asset) = SetupAssetVars(); - var user = new ClaimsPrincipal(); + SetupText(asset.Id, Encoding.UTF8.GetBytes("Hello Asset")); - var data = - new ContentData() - .AddField("assets", - new ContentFieldData() - .AddInvariant(JsonValue.Array(assetId))); + var script = @" + getAssets(data.assets.iv, function (assets) { + getAssetText(assets[0], function (text) { + var result = `Text: ${text}`; - A.CallTo(() => assetQuery.QueryAsync( - A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId), A._)) - .Returns(ResultList.CreateFrom(2, asset)); + complete(result); + }, 'base64'); + });"; - var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + var expected = @" + Text: SGVsbG8gQXNzZXQ= + "; + + var result = (await sut.ExecuteAsync(vars, script)).ToString(); + + Assert.Equal(Cleanup(expected), Cleanup(result)); + } + + [Fact] + public async Task Should_not_resolve_asset_text_if_too_big() + { + var (vars, _) = SetupAssetVars(1_000_000); var script = @" getAssets(data.assets.iv, function (assets) { @@ -233,13 +265,7 @@ namespace Squidex.Domain.Apps.Entities.Assets AppId = appId }; - A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, @event.Id, @event.FileVersion, A._, A._, A._)) - .Invokes(x => - { - var stream = x.GetArgument(3)!; - - stream.Write(Encoding.UTF8.GetBytes("Hello Asset")); - }); + SetupText(@event.Id, Encoding.UTF8.GetBytes("Hello Asset")); var vars = new ScriptVars { @@ -263,7 +289,7 @@ namespace Squidex.Domain.Apps.Entities.Assets } [Fact] - public async Task Should_resolve_asset_text_from_event_if_too_big() + public async Task Should_not_resolve_asset_text_from_event_if_too_big() { var @event = new EnrichedAssetEvent { @@ -297,6 +323,63 @@ namespace Squidex.Domain.Apps.Entities.Assets .MustNotHaveHappened(); } + private void SetupText(DomainId id, byte[] bytes) + { + A.CallTo(() => assetFileStore.DownloadAsync(appId.Id, id, 0, A._, A._, A._)) + .Invokes(x => + { + var stream = x.GetArgument(3)!; + + stream.Write(bytes); + }); + } + + private (ScriptVars, IAssetEntity) SetupAssetVars(int fileSize = 100) + { + var assetId = DomainId.NewGuid(); + var asset = CreateAsset(assetId, 1, fileSize); + + var user = new ClaimsPrincipal(); + + var data = + new ContentData() + .AddField("assets", + new ContentFieldData() + .AddInvariant(JsonValue.Array(assetId))); + + A.CallTo(() => assetQuery.QueryAsync( + A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId), A._)) + .Returns(ResultList.CreateFrom(2, asset)); + + var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + + return (vars, asset); + } + + private (ScriptVars, IAssetEntity[]) SetupAssetsVars(int fileSize = 100) + { + var assetId1 = DomainId.NewGuid(); + var asset1 = CreateAsset(assetId1, 1, fileSize); + var assetId2 = DomainId.NewGuid(); + var asset2 = CreateAsset(assetId1, 2, fileSize); + + var user = new ClaimsPrincipal(); + + var data = + new ContentData() + .AddField("assets", + new ContentFieldData() + .AddInvariant(JsonValue.Array(assetId1, assetId2))); + + A.CallTo(() => assetQuery.QueryAsync( + A.That.Matches(x => x.App.Id == appId.Id && x.User == user), null, A.That.HasIds(assetId1, assetId2), A._)) + .Returns(ResultList.CreateFrom(2, asset1, asset2)); + + var vars = new ScriptVars { Data = data, AppId = appId.Id, User = user }; + + return (vars, new[] { asset1, asset2 }); + } + private IEnrichedAssetEntity CreateAsset(DomainId assetId, int index, int fileSize = 100) { return new AssetEntity