From e7c4333b5a8a9211140210807653bca4fe6a11ad Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 24 May 2020 12:44:38 +0200 Subject: [PATCH 01/14] Nullable fixes. --- .../Migrations/Migrations/PopulateGrainIndexes.cs | 12 ++++++------ .../Contents/IdContentData.cs | 2 +- .../Contents/NamedContentData.cs | 3 ++- .../EventSourcing/Grains/EventConsumerGrain.cs | 4 ++-- .../Json/Objects/JsonObject.cs | 2 +- .../Squidex.Infrastructure/Net/IPAddressComparer.cs | 8 +++++++- .../Orleans/Indexes/UniqueNameIndexGrain.cs | 2 +- .../Reflection/Equality/DeepEqualityComparer.cs | 3 ++- .../Squidex.Infrastructure/Security/Permission.cs | 3 ++- 9 files changed, 24 insertions(+), 15 deletions(-) diff --git a/backend/src/Migrations/Migrations/PopulateGrainIndexes.cs b/backend/src/Migrations/Migrations/PopulateGrainIndexes.cs index 13fffb0c7..ef64223cc 100644 --- a/backend/src/Migrations/Migrations/PopulateGrainIndexes.cs +++ b/backend/src/Migrations/Migrations/PopulateGrainIndexes.cs @@ -55,24 +55,24 @@ namespace Migrations.Migrations bool HasApp(NamedId appId, bool consistent, out Guid id) { - return appsByName.TryGetValue(appId.Name, out id) && (!consistent || id == appId.Id); + return appsByName!.TryGetValue(appId.Name, out id) && (!consistent || id == appId.Id); } HashSet Index(string contributorId) { - return appsByUser.GetOrAddNew(contributorId); + return appsByUser!.GetOrAddNew(contributorId); } void RemoveApp(NamedId appId, bool consistent) { if (HasApp(appId, consistent, out var id)) { - foreach (var apps in appsByUser.Values) + foreach (var apps in appsByUser!.Values) { apps.Remove(id); } - appsByName.Remove(appId.Name); + appsByName!.Remove(appId.Name); } } @@ -125,7 +125,7 @@ namespace Migrations.Migrations HashSet Index(RuleEvent @event) { - return rulesByApp.GetOrAddNew(@event.AppId.Id); + return rulesByApp!.GetOrAddNew(@event.AppId.Id); } await eventStore.QueryAsync(storedEvent => @@ -157,7 +157,7 @@ namespace Migrations.Migrations Dictionary Index(SchemaEvent @event) { - return schemasByApp.GetOrAddNew(@event.AppId.Id); + return schemasByApp!.GetOrAddNew(@event.AppId.Id); } await eventStore.QueryAsync(storedEvent => diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs index 6afddd6b0..7a800ecb6 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs @@ -47,7 +47,7 @@ namespace Squidex.Domain.Apps.Core.Contents return this; } - public bool Equals(IdContentData other) + public bool Equals([AllowNull] IdContentData other) { return base.Equals(other); } diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/NamedContentData.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/NamedContentData.cs index 68b536f5c..84ed62235 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/NamedContentData.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/NamedContentData.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Diagnostics.CodeAnalysis; using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Core.Contents @@ -63,7 +64,7 @@ namespace Squidex.Domain.Apps.Core.Contents return clone; } - public bool Equals(NamedContentData other) + public bool Equals([AllowNull] NamedContentData other) { return base.Equals(other); } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs index 3bb246b27..ce02d6d6a 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs @@ -201,7 +201,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains private async Task ClearAsync() { - var logContext = (actionId: Guid.NewGuid().ToString(), consumer: eventConsumer.Name); + var logContext = (actionId: Guid.NewGuid().ToString(), consumer: eventConsumer!.Name); log.LogDebug(logContext, (ctx, w) => w .WriteProperty("action", "EventConsumerReset") @@ -224,7 +224,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains var eventId = @event.Headers.EventId().ToString(); var eventType = @event.Payload.GetType().Name; - var logContext = (eventId, eventType, consumer: eventConsumer.Name); + var logContext = (eventId, eventType, consumer: eventConsumer!.Name); log.LogDebug(logContext, (ctx, w) => w .WriteProperty("action", "HandleEvent") diff --git a/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs b/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs index 04198efda..a7372d291 100644 --- a/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs +++ b/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs @@ -113,7 +113,7 @@ namespace Squidex.Infrastructure.Json.Objects return Equals(obj as JsonObject); } - public bool Equals(IJsonValue other) + public bool Equals([AllowNull] IJsonValue other) { return Equals(other as JsonObject); } diff --git a/backend/src/Squidex.Infrastructure/Net/IPAddressComparer.cs b/backend/src/Squidex.Infrastructure/Net/IPAddressComparer.cs index be236eaf9..0a5f8f939 100644 --- a/backend/src/Squidex.Infrastructure/Net/IPAddressComparer.cs +++ b/backend/src/Squidex.Infrastructure/Net/IPAddressComparer.cs @@ -6,6 +6,7 @@ // ========================================================================== using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Net; namespace Squidex.Infrastructure.Net @@ -18,8 +19,13 @@ namespace Squidex.Infrastructure.Net { } - public int Compare(IPAddress x, IPAddress y) + public int Compare([AllowNull] IPAddress x, [AllowNull] IPAddress y) { + if (x == null || y == null) + { + return 0; + } + var lbytes = x.GetAddressBytes(); var rbytes = y.GetAddressBytes(); diff --git a/backend/src/Squidex.Infrastructure/Orleans/Indexes/UniqueNameIndexGrain.cs b/backend/src/Squidex.Infrastructure/Orleans/Indexes/UniqueNameIndexGrain.cs index 2a05e3d04..b55354beb 100644 --- a/backend/src/Squidex.Infrastructure/Orleans/Indexes/UniqueNameIndexGrain.cs +++ b/backend/src/Squidex.Infrastructure/Orleans/Indexes/UniqueNameIndexGrain.cs @@ -116,7 +116,7 @@ namespace Squidex.Infrastructure.Orleans.Indexes { state.Value.Names.TryGetValue(name, out var id); - return Task.FromResult(id); + return Task.FromResult(id!); } public Task> GetIdsAsync() diff --git a/backend/src/Squidex.Infrastructure/Reflection/Equality/DeepEqualityComparer.cs b/backend/src/Squidex.Infrastructure/Reflection/Equality/DeepEqualityComparer.cs index 3bb04aef1..4a7ceda20 100644 --- a/backend/src/Squidex.Infrastructure/Reflection/Equality/DeepEqualityComparer.cs +++ b/backend/src/Squidex.Infrastructure/Reflection/Equality/DeepEqualityComparer.cs @@ -6,6 +6,7 @@ // ========================================================================== using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; namespace Squidex.Infrastructure.Reflection.Equality { @@ -19,7 +20,7 @@ namespace Squidex.Infrastructure.Reflection.Equality this.comparer = comparer ?? SimpleEquals.Build(typeof(T)); } - public bool Equals(T x, T y) + public bool Equals([AllowNull] T x, [AllowNull] T y) { return comparer.IsEquals(x, y); } diff --git a/backend/src/Squidex.Infrastructure/Security/Permission.cs b/backend/src/Squidex.Infrastructure/Security/Permission.cs index 7576c457b..5355a3891 100644 --- a/backend/src/Squidex.Infrastructure/Security/Permission.cs +++ b/backend/src/Squidex.Infrastructure/Security/Permission.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Diagnostics.CodeAnalysis; namespace Squidex.Infrastructure.Security { @@ -112,7 +113,7 @@ namespace Squidex.Infrastructure.Security return id; } - public int CompareTo(Permission other) + public int CompareTo([AllowNull] Permission other) { return other == null ? -1 : string.Compare(id, other.id, StringComparison.Ordinal); } From 9aa909550c5a6ecf62efcadd942b6c87c759b370 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 24 May 2020 12:45:12 +0200 Subject: [PATCH 02/14] More nullable fixes. --- .../src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs index 7a800ecb6..3440b2403 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/IdContentData.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Core.Contents From eab3a45a634e12faf5cb10127e7d575863b79ba0 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 24 May 2020 14:18:22 +0200 Subject: [PATCH 03/14] Deprecated packages updated. --- .../Squidex.Domain.Apps.Entities.csproj | 4 ++-- .../Assets/AzureBlobAssetStore.cs | 4 ++-- .../Squidex.Infrastructure.Azure.csproj | 2 +- .../Squidex.Infrastructure.GetEventStore.csproj | 2 +- .../Squidex.Infrastructure/Squidex.Infrastructure.csproj | 6 +++--- backend/src/Squidex/Squidex.csproj | 9 ++++----- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj index 250038c36..4c2d03c8a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj +++ b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj @@ -29,11 +29,11 @@ - + all runtime; build; native; contentfiles; analyzers - + diff --git a/backend/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs b/backend/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs index 8712cb642..63194eacb 100644 --- a/backend/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs +++ b/backend/src/Squidex.Infrastructure.Azure/Assets/AzureBlobAssetStore.cs @@ -9,8 +9,8 @@ using System; using System.IO; using System.Threading; using System.Threading.Tasks; -using Microsoft.WindowsAzure.Storage; -using Microsoft.WindowsAzure.Storage.Blob; +using Microsoft.Azure.Storage; +using Microsoft.Azure.Storage.Blob; namespace Squidex.Infrastructure.Assets { diff --git a/backend/src/Squidex.Infrastructure.Azure/Squidex.Infrastructure.Azure.csproj b/backend/src/Squidex.Infrastructure.Azure/Squidex.Infrastructure.Azure.csproj index ff938312c..36cabaf5d 100644 --- a/backend/src/Squidex.Infrastructure.Azure/Squidex.Infrastructure.Azure.csproj +++ b/backend/src/Squidex.Infrastructure.Azure/Squidex.Infrastructure.Azure.csproj @@ -10,7 +10,7 @@ - + diff --git a/backend/src/Squidex.Infrastructure.GetEventStore/Squidex.Infrastructure.GetEventStore.csproj b/backend/src/Squidex.Infrastructure.GetEventStore/Squidex.Infrastructure.GetEventStore.csproj index c3bbf6d47..3de2c7e28 100644 --- a/backend/src/Squidex.Infrastructure.GetEventStore/Squidex.Infrastructure.GetEventStore.csproj +++ b/backend/src/Squidex.Infrastructure.GetEventStore/Squidex.Infrastructure.GetEventStore.csproj @@ -10,7 +10,7 @@ True - + diff --git a/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj b/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj index 1f4639195..dc4be4a36 100644 --- a/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj +++ b/backend/src/Squidex.Infrastructure/Squidex.Infrastructure.csproj @@ -21,12 +21,12 @@ - + all runtime; build; native; contentfiles; analyzers - - + + diff --git a/backend/src/Squidex/Squidex.csproj b/backend/src/Squidex/Squidex.csproj index 00beeee54..4ffd6bd44 100644 --- a/backend/src/Squidex/Squidex.csproj +++ b/backend/src/Squidex/Squidex.csproj @@ -35,7 +35,6 @@ - @@ -46,9 +45,9 @@ - - - + + + @@ -56,7 +55,7 @@ - + From e41a59305b6b19ac5650f19933c9480fb7b44584 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 24 May 2020 15:52:42 +0200 Subject: [PATCH 04/14] Get total bytes. --- .../Apps/Plans/UsageGate.cs | 2 +- .../Rules/UsageTracking/UsageTrackerGrain.cs | 2 +- .../UsageTracking/ApiUsageTracker.cs | 11 +++++++++- .../UsageTracking/IApiUsageTracker.cs | 4 +++- .../Apps/Plans/UsageGateTests.cs | 2 +- .../UsageTracking/ApiUsageTrackerTests.cs | 20 +++++++++++++++++-- 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/UsageGate.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/UsageGate.cs index fd2f7ce42..3418bf0f6 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/UsageGate.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/UsageGate.cs @@ -50,7 +50,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans { var appId = app.Id; - var usage = await apiUsageTracker.GetMonthCostsAsync(appId.ToString(), today); + var usage = await apiUsageTracker.GetMonthCallsAsync(appId.ToString(), today); if (IsAboutToBeLocked(today, plan.MaxApiCalls, usage) && !HasNotifiedBefore(app.Id)) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Rules/UsageTracking/UsageTrackerGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Rules/UsageTracking/UsageTrackerGrain.cs index 8edaf0882..dd63d0d45 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Rules/UsageTracking/UsageTrackerGrain.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Rules/UsageTracking/UsageTrackerGrain.cs @@ -83,7 +83,7 @@ namespace Squidex.Domain.Apps.Entities.Rules.UsageTracking if (!target.Triggered.HasValue || target.Triggered < from) { - var costs = await usageTracker.GetMonthCostsAsync(target.AppId.Id.ToString(), today); + var costs = await usageTracker.GetMonthCallsAsync(target.AppId.Id.ToString(), today); var limit = target.Limits; diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/ApiUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/ApiUsageTracker.cs index 727655a6a..d475fa3c0 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/ApiUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/ApiUsageTracker.cs @@ -23,7 +23,7 @@ namespace Squidex.Infrastructure.UsageTracking this.usageTracker = usageTracker; } - public async Task GetMonthCostsAsync(string key, DateTime date) + public async Task GetMonthCallsAsync(string key, DateTime date) { var apiKey = GetKey(key); @@ -32,6 +32,15 @@ namespace Squidex.Infrastructure.UsageTracking return counters.GetInt64(CounterTotalCalls); } + public async Task GetMonthBytesAsync(string key, DateTime date) + { + var apiKey = GetKey(key); + + var counters = await usageTracker.GetForMonthAsync(apiKey, date); + + return counters.GetInt64(CounterTotalBytes); + } + public Task TrackAsync(DateTime date, string key, string? category, double weight, long elapsedMs, long bytes) { var apiKey = GetKey(key); diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/IApiUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/IApiUsageTracker.cs index 6c31bd099..0d960eb1f 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/IApiUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/IApiUsageTracker.cs @@ -15,7 +15,9 @@ namespace Squidex.Infrastructure.UsageTracking { Task TrackAsync(DateTime date, string key, string? category, double weight, long elapsedMs, long bytes); - Task GetMonthCostsAsync(string key, DateTime date); + Task GetMonthCallsAsync(string key, DateTime date); + + Task GetMonthBytesAsync(string key, DateTime date); Task<(ApiStatsSummary, Dictionary> Details)> QueryAsync(string key, DateTime fromDate, DateTime toDate); } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/UsageGateTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/UsageGateTests.cs index 968dc7e5e..34fc2c65f 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/UsageGateTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/UsageGateTests.cs @@ -51,7 +51,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans A.CallTo(() => appPlan.BlockingApiCalls) .ReturnsLazily(x => apiCallsBlocking); - A.CallTo(() => usageTracker.GetMonthCostsAsync(appId.Id.ToString(), today)) + A.CallTo(() => usageTracker.GetMonthCallsAsync(appId.Id.ToString(), today)) .ReturnsLazily(x => Task.FromResult(apiCallsCurrent)); sut = new UsageGate(appPlansProvider, usageTracker, grainFactory); diff --git a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/ApiUsageTrackerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/ApiUsageTrackerTests.cs index 932816305..7e33e731f 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/ApiUsageTrackerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/ApiUsageTrackerTests.cs @@ -48,7 +48,7 @@ namespace Squidex.Infrastructure.UsageTracking } [Fact] - public async Task Should_query_from_tracker() + public async Task Should_query_calls_from_tracker() { var counters = new Counters { @@ -58,11 +58,27 @@ namespace Squidex.Infrastructure.UsageTracking A.CallTo(() => usageTracker.GetForMonthAsync($"{key}_API", date)) .Returns(counters); - var result = await sut.GetMonthCostsAsync(key, date); + var result = await sut.GetMonthCallsAsync(key, date); Assert.Equal(4, result); } + [Fact] + public async Task Should_query_bytes_from_tracker() + { + var counters = new Counters + { + [ApiUsageTracker.CounterTotalBytes] = 14 + }; + + A.CallTo(() => usageTracker.GetForMonthAsync($"{key}_API", date)) + .Returns(counters); + + var result = await sut.GetMonthBytesAsync(key, date); + + Assert.Equal(14, result); + } + [Fact] public async Task Should_query_stats_from_tracker() { From 0aa661defbca93c3c2868b004f7795ec9b64b55f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Tue, 26 May 2020 09:28:34 +0200 Subject: [PATCH 05/14] Tests fixed --- .../Orleans/JsonExternalSerializerTests.cs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/tests/Squidex.Infrastructure.Tests/Orleans/JsonExternalSerializerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Orleans/JsonExternalSerializerTests.cs index 078e36e6a..e0b21bb40 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Orleans/JsonExternalSerializerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Orleans/JsonExternalSerializerTests.cs @@ -78,7 +78,7 @@ namespace Squidex.Infrastructure.Orleans } } - private static DeserializationContext CreateReader(MemoryStream buffer) + private static IDeserializationContext CreateReader(MemoryStream buffer) { var reader = A.Fake(); @@ -89,10 +89,15 @@ namespace Squidex.Infrastructure.Orleans A.CallTo(() => reader.Length) .ReturnsLazily(x => (int)buffer.Length); - return new DeserializationContext(null) { StreamReader = reader }; + var context = A.Fake(); + + A.CallTo(() => context.StreamReader) + .Returns(reader); + + return context; } - private static SerializationContext CreateWriter(MemoryStream buffer) + private static ISerializationContext CreateWriter(MemoryStream buffer) { var writer = A.Fake(); @@ -101,7 +106,12 @@ namespace Squidex.Infrastructure.Orleans A.CallTo(() => writer.CurrentOffset) .ReturnsLazily(x => (int)buffer.Position); - return new SerializationContext(null) { StreamWriter = writer }; + var context = A.Fake(); + + A.CallTo(() => context.StreamWriter) + .Returns(writer); + + return context; } private static List ArrayOfLength(int length) From 0814b00c849f309e19a9a6829e95441792a2128c Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Tue, 26 May 2020 19:04:08 +0200 Subject: [PATCH 06/14] Rule formatter transformations. (#525) --- .../HandleRules/RuleEventFormatter.cs | 105 +++++++++++++++--- .../HandleRules/RuleEventFormatterTests.cs | 48 ++++++++ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index c8824317e..6b9de787e 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -13,6 +13,7 @@ using System.Reflection; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Newtonsoft.Json; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Scripting; @@ -27,9 +28,9 @@ namespace Squidex.Domain.Apps.Core.HandleRules public class RuleEventFormatter { private const string Fallback = "null"; - private static readonly Regex RegexPatternOld = new Regex(@"^(?[^_]*)_(?[^\s]*)", RegexOptions.Compiled); - private static readonly Regex RegexPatternNew = new Regex(@"^\{(?[^_]*)_(?[^\s]*)\}", RegexOptions.Compiled); - private readonly List<(char[] Pattern, Func Replacer)> patterns = new List<(char[] Pattern, Func Replacer)>(); + private static readonly Regex RegexPatternOld = new Regex(@"^(?(?[^_]*)_(?[^\s]*))", RegexOptions.Compiled); + private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[^_]*)_(?[^\s]*))([\s]*\|[\s]*(?.*)){0,1}\}", RegexOptions.Compiled); + private readonly List<(string Pattern, Func Replacer)> patterns = new List<(string Pattern, Func Replacer)>(); private readonly IJsonSerializer jsonSerializer; private readonly IUrlGenerator urlGenerator; private readonly IScriptEngine scriptEngine; @@ -63,7 +64,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules private void AddPattern(string placeholder, Func generator) { - patterns.Add((placeholder.ToCharArray(), generator)); + patterns.Add((placeholder, generator)); } public virtual string ToPayload(T @event) @@ -107,7 +108,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules { parts.Add((currentOffset, i - currentOffset, default)); - var (replacement, length) = GetReplacement(span.Slice(i + 1), @event); + var (replacement, length) = GetReplacement(span.Slice(i + 1).ToString(), @event); if (length > 0) { @@ -139,35 +140,61 @@ namespace Squidex.Domain.Apps.Core.HandleRules return sb.ToString(); } - private (string Result, int Length) GetReplacement(ReadOnlySpan test, EnrichedEvent @event) + private (string Result, int Length) GetReplacement(string test, EnrichedEvent @event) { - for (var j = 0; j < patterns.Count; j++) + var (isNewRegex, match) = Match(test); + + if (match.Success) { - var (pattern, replacer) = patterns[j]; + var (length, text) = ResolveOldPatterns(match, isNewRegex, @event); - if (test.StartsWith(pattern, StringComparison.OrdinalIgnoreCase)) + if (length == 0) { - return (replacer(@event) ?? Fallback, pattern.Length); + (length, text) = ResolveFromPath(match, @event); } + + return (TransformText(text, match.Groups["Transform"]?.Value), length); } - var currentString = test.ToString(); + return (Fallback, 0); + } - var match = RegexPatternNew.Match(currentString); + private (bool IsNew, Match) Match(string test) + { + var match = RegexPatternNew.Match(test); - if (!match.Success) + if (match.Success) { - match = RegexPatternOld.Match(currentString); + return (true, match); } - if (match.Success) + return (false, RegexPatternOld.Match(test)); + } + + private (int Length, string? Result) ResolveOldPatterns(Match match, bool isNewRegex, EnrichedEvent @event) + { + var fullPath = match.Groups["FullPath"].Value; + + for (var j = 0; j < patterns.Count; j++) { - var path = match.Groups["Path"].Value.Split('.', StringSplitOptions.RemoveEmptyEntries); + var (pattern, replacer) = patterns[j]; + + if (fullPath.StartsWith(pattern, StringComparison.OrdinalIgnoreCase)) + { + var result = replacer(@event); - return (CalculateData(@event, path) ?? Fallback, match.Length); + if (isNewRegex) + { + return (match.Length, result); + } + else + { + return (pattern.Length, result); + } + } } - return (Fallback, 0); + return default; } private static string TimestampDate(EnrichedEvent @event) @@ -300,6 +327,48 @@ namespace Squidex.Domain.Apps.Core.HandleRules return null; } + private static string TransformText(string? text, string? transform) + { + if (text != null && !string.IsNullOrWhiteSpace(transform)) + { + var transformations = transform.Split("|", StringSplitOptions.RemoveEmptyEntries); + + foreach (var transformation in transformations) + { + switch (transformation.Trim().ToLowerInvariant()) + { + case "lower": + text = text.ToLowerInvariant(); + break; + case "upper": + text = text.ToUpperInvariant(); + break; + case "escape": + text = JsonConvert.ToString(text); + text = text[1..^1]; + break; + case "slugify": + text = text.Slugify(); + break; + case "trim": + text = text.Trim(); + break; + } + } + } + + return text ?? Fallback; + } + + private (int Length, string? Result) ResolveFromPath(Match match, EnrichedEvent @event) + { + var path = match.Groups["Path"].Value.Split('.', StringSplitOptions.RemoveEmptyEntries); + + var result = CalculateData(@event, path); + + return (match.Length, result); + } + private static string? CalculateData(object @event, string[] path) { object? current = @event; diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index 3021e7dc3..0c478d6df 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -528,5 +528,53 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules Assert.Equal("Created", result); } + + [Theory] + [InlineData("Found in ${ASSET_FILENAME | Upper}.docx", "Found in DONALD DUCK.docx")] + [InlineData("Found in ${ASSET_FILENAME| Upper }.docx", "Found in DONALD DUCK.docx")] + [InlineData("Found in ${ASSET_FILENAME|Upper }.docx", "Found in DONALD DUCK.docx")] + public void Should_transform_replacements_and_igore_whitepsaces(string script, string expect) + { + var @event = new EnrichedAssetEvent { FileName = "Donald Duck" }; + + var result = sut.Format(script, @event); + + Assert.Equal(expect, result); + } + + [Theory] + [InlineData("Found in ${ASSET_FILENAME | Escape | Upper}.docx", "Found in DONALD\\\"DUCK .docx")] + [InlineData("Found in ${ASSET_FILENAME | Escape}.docx", "Found in Donald\\\"Duck .docx")] + [InlineData("Found in ${ASSET_FILENAME | Upper}.docx", "Found in DONALD\"DUCK .docx")] + [InlineData("Found in ${ASSET_FILENAME | Lower}.docx", "Found in donald\"duck .docx")] + [InlineData("Found in ${ASSET_FILENAME | Slugify}.docx", "Found in donald-duck.docx")] + [InlineData("Found in ${ASSET_FILENAME | Trim}.docx", "Found in Donald\"Duck.docx")] + public void Should_transform_replacements(string script, string expect) + { + var @event = new EnrichedAssetEvent { FileName = "Donald\"Duck " }; + + var result = sut.Format(script, @event); + + Assert.Equal(expect, result); + } + + [Theory] + [InlineData("From ${USER_NAME | Escape | Upper}", "From DONALD\\\"DUCK ")] + [InlineData("From ${USER_NAME | Escape}", "From Donald\\\"Duck ")] + [InlineData("From ${USER_NAME | Upper}", "From DONALD\"DUCK ")] + [InlineData("From ${USER_NAME | Lower}", "From donald\"duck ")] + [InlineData("From ${USER_NAME | Slugify}", "From donald-duck")] + [InlineData("From ${USER_NAME | Trim}", "From Donald\"Duck")] + public void Should_transform_replacements_with_simple_pattern(string script, string expect) + { + var @event = new EnrichedContentEvent { User = user }; + + A.CallTo(() => user.Claims) + .Returns(new List { new Claim(SquidexClaimTypes.DisplayName, "Donald\"Duck ") }); + + var result = sut.Format(script, @event); + + Assert.Equal(expect, result); + } } } From 98c785d96445c234d9b1539432520202454c052b Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 27 May 2020 11:13:22 +0200 Subject: [PATCH 07/14] Fallback for old status --- .../Scripting/ScriptContext.cs | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs index dba6b2db5..e6ae1fe5f 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs @@ -49,8 +49,8 @@ namespace Squidex.Domain.Apps.Core.Scripting public NamedContentData? DataOld { - get => GetValue("oldData"); - set => SetValue(value, "oldData"); + get => GetValue(); + set => SetValue(value); } public Status Status @@ -77,7 +77,19 @@ namespace Squidex.Domain.Apps.Core.Scripting set => SetValue(value); } - public void SetValue(object? value, [CallerMemberNameAttribute] string? key = null) + [Obsolete] + public Status OldStatus + { + get => GetValue("statusOld"); + } + + [Obsolete] + public NamedContentData? OldData + { + get => GetValue("dataOld"); + } + + public void SetValue(object? value, [CallerMemberName] string? key = null) { if (key != null) { @@ -85,7 +97,7 @@ namespace Squidex.Domain.Apps.Core.Scripting } } - public T GetValue([CallerMemberNameAttribute] string? key = null) + public T GetValue([CallerMemberName] string? key = null) { if (key != null && TryGetValue(key, out var temp) && temp is T result) { From a69255e575b40b8551793d7e56775b2244283e64 Mon Sep 17 00:00:00 2001 From: Mittul Madaan Date: Wed, 27 May 2020 17:11:50 +0100 Subject: [PATCH 08/14] Kafka rule extension. Added payload, key and headers for JSON. (#526) Co-authored-by: Mittul Madaan --- .../Actions/Kafka/KafkaAction.cs | 15 ++++ .../Actions/Kafka/KafkaActionHandler.cs | 73 ++++++++++++++++++- .../Actions/Kafka/KafkaProducer.cs | 4 +- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs index e1866bd99..1bf220cad 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs @@ -25,5 +25,20 @@ namespace Squidex.Extensions.Actions.Kafka [DataType(DataType.Text)] [Formattable] public string TopicName { get; set; } + + [Display(Name = "Payload (Optional)", Description = "Leave it empty to use the full event as body.")] + [DataType(DataType.MultilineText)] + [Formattable] + public string Payload { get; set; } + + [Display(Name = "Key", Description = "The message key, commonly used for partitioning.")] + [DataType(DataType.Text)] + [Formattable] + public string Key { get; set; } + + [Display(Name = "Headers (Optional)", Description = "The message headers in the format '[Key]=[Value]', one entry per line.")] + [DataType(DataType.MultilineText)] + [Formattable] + public string Headers { get; set; } } } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs index 28d01719a..e7e248334 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs @@ -6,8 +6,11 @@ // ========================================================================== using System; +using System.Collections.Generic; +using System.Text; using System.Threading; using System.Threading.Tasks; +using Confluent.Kafka; using Squidex.Domain.Apps.Core.HandleRules; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; @@ -26,21 +29,83 @@ namespace Squidex.Extensions.Actions.Kafka protected override (string Description, KafkaJob Data) CreateJob(EnrichedEvent @event, KafkaAction action) { + string value, key; + + if (!string.IsNullOrEmpty(action.Payload)) + { + value = Format(action.Payload, @event); + } + else + { + value = ToEnvelopeJson(@event); + } + + if (!string.IsNullOrEmpty(action.Key)) + { + key = Format(action.Key, @event); + } + else + { + key = @event.Name; + } + var ruleJob = new KafkaJob { TopicName = action.TopicName, - MessageKey = @event.Name, - MessageValue = ToEnvelopeJson(@event) + MessageKey = key, + MessageValue = value, + Headers = ParseHeaders(action.Headers, @event) }; return (Description, ruleJob); } + private Dictionary ParseHeaders(string headers, EnrichedEvent @event) + { + if (string.IsNullOrWhiteSpace(headers)) + { + return null; + } + + var headersDictionary = new Dictionary(); + + var lines = headers.Split('\n'); + + foreach (var line in lines) + { + var indexEqual = line.IndexOf('='); + + if (indexEqual > 0 && indexEqual < line.Length - 1) + { + var key = line.Substring(0, indexEqual); + var val = line.Substring(indexEqual + 1); + + val = Format(val, @event); + + headersDictionary[key] = val; + } + } + + return headersDictionary; + } + protected override async Task ExecuteJobAsync(KafkaJob job, CancellationToken ct = default) { try { - await kafkaProducer.Send(job.TopicName, job.MessageKey, job.MessageValue); + var message = new Message { Key = job.MessageKey, Value = job.MessageValue }; + + if (job.Headers?.Count > 0) + { + message.Headers = new Headers(); + + foreach (var header in job.Headers) + { + message.Headers.Add(header.Key, Encoding.UTF8.GetBytes(header.Value)); + } + } + + await kafkaProducer.Send(job.TopicName, message); return Result.Success($"Event pushed to {job.TopicName} kafka topic."); } @@ -58,5 +123,7 @@ namespace Squidex.Extensions.Actions.Kafka public string MessageKey { get; set; } public string MessageValue { get; set; } + + public Dictionary Headers { get; set; } } } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs index cc74e2b90..7f34fd2e3 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs @@ -77,10 +77,8 @@ namespace Squidex.Extensions.Actions.Kafka .WriteProperty("reason", error.Reason)); } - public async Task> Send(string topicName, string key, string value) + public async Task> Send(string topicName, Message message) { - var message = new Message { Key = key, Value = value }; - return await producer.ProduceAsync(topicName, message); } From d3b3f76dcb155038d9a89336150ac195502c8cec Mon Sep 17 00:00:00 2001 From: Mittul Madaan Date: Thu, 28 May 2020 17:26:37 +0100 Subject: [PATCH 09/14] Kafka avro support (#527) * Kafka rule extension. Added payload, key and headers for JSON. * Kafka avro support. Co-authored-by: Mittul Madaan --- .../Actions/Kafka/KafkaAction.cs | 4 + .../Actions/Kafka/KafkaActionHandler.cs | 9 +- .../Actions/Kafka/KafkaProducer.cs | 110 +++++++++++++++++- .../Actions/Kafka/KafkaProducerOptions.cs | 11 ++ .../Squidex.Extensions.csproj | 2 + 5 files changed, 127 insertions(+), 9 deletions(-) diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs index 1bf220cad..50aae0259 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaAction.cs @@ -40,5 +40,9 @@ namespace Squidex.Extensions.Actions.Kafka [DataType(DataType.MultilineText)] [Formattable] public string Headers { get; set; } + + [Display(Name = "Schema (Optional)", Description = "Define a specific AVRO schema in JSON format.")] + [DataType(DataType.MultilineText)] + public string Schema { get; set; } } } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs index e7e248334..dd5964e83 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs @@ -54,7 +54,8 @@ namespace Squidex.Extensions.Actions.Kafka TopicName = action.TopicName, MessageKey = key, MessageValue = value, - Headers = ParseHeaders(action.Headers, @event) + Headers = ParseHeaders(action.Headers, @event), + Schema = action.Schema }; return (Description, ruleJob); @@ -105,13 +106,13 @@ namespace Squidex.Extensions.Actions.Kafka } } - await kafkaProducer.Send(job.TopicName, message); + await kafkaProducer.Send(job.TopicName, message, job.Schema); return Result.Success($"Event pushed to {job.TopicName} kafka topic."); } catch (Exception ex) { - return Result.Failed(ex, "Push to Kafka failed."); + return Result.Failed(ex, $"Push to Kafka failed: {ex}"); } } } @@ -125,5 +126,7 @@ namespace Squidex.Extensions.Actions.Kafka public string MessageValue { get; set; } public Dictionary Headers { get; set; } + + public string Schema { get; set; } } } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs index 7f34fd2e3..a8f2ecfaa 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs @@ -5,20 +5,34 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using System.Collections.Generic; using System.Threading.Tasks; +using Avro; +using Avro.Generic; using Confluent.Kafka; +using Confluent.SchemaRegistry; +using Confluent.SchemaRegistry.Serdes; +using GraphQL.Types; using Microsoft.Extensions.Options; +using Squidex.Infrastructure.Json; +using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.Log; namespace Squidex.Extensions.Actions.Kafka { public sealed class KafkaProducer { - private readonly IProducer producer; + private readonly IProducer textProducer; + private readonly IProducer avroProducer; + private readonly ISchemaRegistryClient schemaRegistry; + private readonly IJsonSerializer jsonSerializer; - public KafkaProducer(IOptions options, ISemanticLog log) + public KafkaProducer(IOptions options, ISemanticLog log, IJsonSerializer jsonSerializer) { - producer = new ProducerBuilder(options.Value) + this.jsonSerializer = jsonSerializer; + + textProducer = new ProducerBuilder(options.Value) .SetErrorHandler((p, error) => { LogError(log, error); @@ -30,6 +44,24 @@ namespace Squidex.Extensions.Actions.Kafka .SetKeySerializer(Serializers.Utf8) .SetValueSerializer(Serializers.Utf8) .Build(); + + if (options.Value.IsSchemaRegistryConfigured()) + { + schemaRegistry = new CachedSchemaRegistryClient(options.Value.SchemaRegistry); + + avroProducer = new ProducerBuilder(options.Value) + .SetErrorHandler((p, error) => + { + LogError(log, error); + }) + .SetLogHandler((p, message) => + { + LogMessage(log, message); + }) + .SetKeySerializer(Serializers.Utf8) + .SetValueSerializer(new AvroSerializer(schemaRegistry, options.Value.AvroSerializer)) + .Build(); + } } private static void LogMessage(ISemanticLog log, LogMessage message) @@ -77,14 +109,80 @@ namespace Squidex.Extensions.Actions.Kafka .WriteProperty("reason", error.Reason)); } - public async Task> Send(string topicName, Message message) + public async Task> Send(string topicName, Message message, string schema) { - return await producer.ProduceAsync(topicName, message); + if (!string.IsNullOrWhiteSpace(schema)) + { + var value = CreateAvroRecord(message.Value, schema); + + var avroMessage = new Message { Key = message.Key, Headers = message.Headers, Value = value }; + + await avroProducer.ProduceAsync(topicName, avroMessage); + } + + return await textProducer.ProduceAsync(topicName, message); + } + + private GenericRecord CreateAvroRecord(string json, string avroSchema) + { + var schema = (RecordSchema)Avro.Schema.Parse(avroSchema); + + var jsonObject = jsonSerializer.Deserialize(json); + + var result = (GenericRecord)GetValue(jsonObject, schema); + + return result; } public void Dispose() { - producer?.Dispose(); + textProducer?.Dispose(); + avroProducer?.Dispose(); + } + + private object GetValue(IJsonValue value, Avro.Schema schema) + { + switch (value) + { + case JsonString s: + return s.Value; + case JsonNumber n: + return n.Value; + case JsonBoolean b: + return b.Value; + case JsonObject o: + { + var recordSchema = (RecordSchema)schema; + + var result = new GenericRecord(recordSchema); + + foreach (var (key, childValue) in o) + { + if (recordSchema.TryGetField(key, out var field)) + { + result.Add(key, GetValue(childValue, field.Schema)); + } + } + + return result; + } + + case JsonArray a: + { + var arraySchema = (ArraySchema)schema; + + var result = new List(); + + foreach (var item in a) + { + result.Add(GetValue(item, arraySchema.ItemSchema)); + } + + return result.ToArray(); + } + } + + return null; } } } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducerOptions.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducerOptions.cs index e1fd7b7fc..6a85799d2 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducerOptions.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducerOptions.cs @@ -6,14 +6,25 @@ // ========================================================================== using Confluent.Kafka; +using Confluent.SchemaRegistry; +using Confluent.SchemaRegistry.Serdes; namespace Squidex.Extensions.Actions.Kafka { public class KafkaProducerOptions : ProducerConfig { + public SchemaRegistryConfig SchemaRegistry { get; set; } + + public AvroSerializerConfig AvroSerializer { get; set; } + public bool IsProducerConfigured() { return !string.IsNullOrWhiteSpace(BootstrapServers); } + + public bool IsSchemaRegistryConfigured() + { + return !string.IsNullOrWhiteSpace(SchemaRegistry?.Url); + } } } diff --git a/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj b/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj index 1f57f09ce..69b9b94df 100644 --- a/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj +++ b/backend/extensions/Squidex.Extensions/Squidex.Extensions.csproj @@ -9,7 +9,9 @@ + + From 3534fe76089eb15eebbfcacba060dd0124fb2798 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 28 May 2020 18:26:58 +0200 Subject: [PATCH 10/14] Feature/content sorting (#523) * Improve content sorting using two stage aggregation framework. * Sorting improvements. --- .../Assets/MongoAssetRepository.cs | 11 +-- .../Operations/QueryContentsByQuery.cs | 72 +++++++++++++++---- .../MongoDb/Queries/LimitExtensions.cs | 25 +++++++ .../Queries/CompareFilter.cs | 7 ++ .../Queries/FilterNode.cs | 4 ++ .../Queries/LogicalFilter.cs | 8 +++ .../Queries/NegateFilter.cs | 7 ++ .../Squidex.Infrastructure/Queries/Query.cs | 17 +++++ .../Contents/MongoDb/ContentsQueryFixture.cs | 10 +-- .../Contents/MongoDb/ContentsQueryTests.cs | 50 ++++++++++--- .../Squidex.Domain.Apps.Entities.Tests.csproj | 1 + .../Queries/QueryTests.cs | 64 +++++++++++++++++ 12 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index 06c44509f..82578a00c 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -85,16 +85,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets return ResultList.Create(assetCount.Result, assetItems.Result); } - catch (MongoQueryException ex) + catch (MongoQueryException ex) when (ex.Message.Contains("17406")) { - if (ex.Message.Contains("17406")) - { - throw new DomainException("Result set is too large to be retrieved. Use $top parameter to reduce the number of items."); - } - else - { - throw; - } + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs index fcf3dec34..03e07343b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs @@ -7,8 +7,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Contents; @@ -22,9 +25,21 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations { internal sealed class QueryContentsByQuery : OperationBase { + private static readonly PropertyPath DefaultOrderField = "mt"; private readonly DataConverter converter; private readonly ITextIndex indexer; + [BsonIgnoreExtraElements] + internal sealed class IdOnly + { + [BsonId] + [BsonElement("_id")] + [BsonRepresentation(BsonType.String)] + public Guid Id { get; set; } + + public MongoContentEntity[] Joined { get; set; } + } + public QueryContentsByQuery(DataConverter converter, ITextIndex indexer) { this.converter = converter; @@ -71,12 +86,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations var filter = CreateFilter(schema.Id, fullTextIds, query); var contentCount = Collection.Find(filter).CountDocumentsAsync(); - var contentItems = - Collection.Find(filter) - .QueryLimit(query) - .QuerySkip(query) - .QuerySort(query) - .ToListAsync(); + var contentItems = FindContentsAsync(query, filter); await Task.WhenAll(contentItems, contentCount); @@ -87,17 +97,53 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations return ResultList.Create(contentCount.Result, contentItems.Result); } - catch (MongoQueryException ex) + catch (MongoCommandException ex) when (ex.Code == 96) { - if (ex.Message.Contains("17406")) - { - throw new DomainException("Result set is too large to be retrieved. Use $top parameter to reduce the number of items."); - } - else + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); + } + catch (MongoQueryException ex) when (ex.Message.Contains("17406")) + { + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); + } + } + + private async Task> FindContentsAsync(ClrQuery query, FilterDefinition filter) + { + if (query.Skip > 0 && !IsSatisfiedByIndex(query)) + { + var projection = Projection.Include("_id"); + + foreach (var field in query.GetAllFields()) { - throw; + projection = Projection.Include(field); } + + var joined = + await Collection.Aggregate() + .Match(filter) + .Project(projection) + .QuerySort(query) + .QuerySkip(query) + .QueryLimit(query) + .Lookup(Collection, x => x.Id, x => x.Id, x => x.Joined) + .ToListAsync(); + + return joined.Select(x => x.Joined[0]).ToList(); } + + var result = + Collection.Find(filter) + .QuerySort(query) + .QueryLimit(query) + .QuerySkip(query) + .ToListAsync(); + + return await result; + } + + private static bool IsSatisfiedByIndex(ClrQuery query) + { + return query.Sort?.Any(x => x.Path == DefaultOrderField && x.Order == SortOrder.Descending) == true; } private static FilterDefinition CreateFilter(Guid schemaId, ICollection? ids, ClrQuery? query) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs index 355138c98..4cc07fe2d 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs @@ -12,6 +12,16 @@ namespace Squidex.Infrastructure.MongoDb.Queries { public static class LimitExtensions { + public static IAggregateFluent QueryLimit(this IAggregateFluent cursor, ClrQuery query) + { + if (query.Take < long.MaxValue) + { + cursor = cursor.Limit((int)query.Take); + } + + return cursor; + } + public static IFindFluent QueryLimit(this IFindFluent cursor, ClrQuery query) { if (query.Take < long.MaxValue) @@ -22,6 +32,16 @@ namespace Squidex.Infrastructure.MongoDb.Queries return cursor; } + public static IAggregateFluent QuerySkip(this IAggregateFluent cursor, ClrQuery query) + { + if (query.Skip > 0) + { + cursor = cursor.Skip((int)query.Skip); + } + + return cursor; + } + public static IFindFluent QuerySkip(this IFindFluent cursor, ClrQuery query) { if (query.Skip > 0) @@ -36,5 +56,10 @@ namespace Squidex.Infrastructure.MongoDb.Queries { return cursor.Sort(query.BuildSort()); } + + public static IAggregateFluent QuerySort(this IAggregateFluent cursor, ClrQuery query) + { + return cursor.Sort(query.BuildSort()); + } } } diff --git a/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs b/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs index 5b522e6c9..c33d039f2 100644 --- a/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public sealed class CompareFilter : FilterNode @@ -28,6 +30,11 @@ namespace Squidex.Infrastructure.Queries Value = value; } + public override void AddFields(HashSet fields) + { + fields.Add(Path.ToString()); + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs b/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs index 125787f11..0ca210b6e 100644 --- a/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs +++ b/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs @@ -5,12 +5,16 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public abstract class FilterNode { public abstract T Accept(FilterNodeVisitor visitor); + public abstract void AddFields(HashSet fields); + public abstract override string ToString(); } } diff --git a/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs b/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs index 1fc2a1416..f2a49a75e 100644 --- a/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs @@ -25,6 +25,14 @@ namespace Squidex.Infrastructure.Queries Type = type; } + public override void AddFields(HashSet fields) + { + foreach (var filter in Filters) + { + filter.AddFields(fields); + } + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs b/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs index 09583a0f8..83f04331a 100644 --- a/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public sealed class NegateFilter : FilterNode @@ -18,6 +20,11 @@ namespace Squidex.Infrastructure.Queries Filter = filter; } + public override void AddFields(HashSet fields) + { + Filter.AddFields(fields); + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/Query.cs b/backend/src/Squidex.Infrastructure/Queries/Query.cs index d9e814d10..136999e10 100644 --- a/backend/src/Squidex.Infrastructure/Queries/Query.cs +++ b/backend/src/Squidex.Infrastructure/Queries/Query.cs @@ -26,6 +26,23 @@ namespace Squidex.Infrastructure.Queries public List Sort { get; set; } = new List(); + public HashSet GetAllFields() + { + var result = new HashSet(); + + if (Sort != null) + { + foreach (var sorting in Sort) + { + result.Add(sorting.Path.ToString()); + } + } + + Filter?.AddFields(result); + + return result; + } + public override string ToString() { var parts = new List(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs index 6d58727a5..fc85d18db 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using FakeItEasy; +using LoremNET; using MongoDB.Bson; using MongoDB.Driver; using Newtonsoft.Json; @@ -102,13 +103,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { for (var i = 0; i < numValues; i++) { - var value = i.ToString(); - var data = new IdContentData() .AddField(1, new ContentFieldData() - .AddJsonValue(JsonValue.Create(value))); + .AddJsonValue(JsonValue.Create(i))) + .AddField(2, + new ContentFieldData() + .AddJsonValue(JsonValue.Create(Lorem.Paragraph(200, 20)))); var content = new MongoContentEntity { @@ -198,7 +200,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { var schemaDef = new Schema("my-schema") - .AddField(Fields.String(1, "value", Partitioning.Invariant)); + .AddField(Fields.Number(1, "value", Partitioning.Invariant)); var schema = Mocks.Schema( diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs index 48bdb6981..abb9142a3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs @@ -62,11 +62,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb [Fact] public async Task Should_query_contents_by_filter() { - var filter = F.Eq("data.value.iv", _.RandomValue()); + var filter = F.Eq("data.value.iv", 12); var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter); - Assert.NotNull(contents); + Assert.NotEmpty(contents); } [Fact] @@ -84,7 +84,23 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb var contents = await QueryAsync(query); - Assert.NotNull(contents); + Assert.NotEmpty(contents); + } + + [Fact] + public async Task Should_query_contents_with_large_skip() + { + var query = new ClrQuery + { + Sort = new List + { + new SortNode("data.value.iv", SortOrder.Ascending) + } + }; + + var contents = await QueryAsync(query, 1000, 9000); + + Assert.NotEmpty(contents); } [Fact] @@ -105,19 +121,33 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { var query = new ClrQuery { - Filter = F.Eq("data.value.iv", _.RandomValue()) + Filter = F.Eq("data.value.iv", 200) }; - var contents = await QueryAsync(query); + var contents = await QueryAsync(query, 1000, 0); - Assert.NotNull(contents); + Assert.NotEmpty(contents); } - private async Task> QueryAsync(ClrQuery clrQuery) + private async Task> QueryAsync(ClrQuery clrQuery, int take = 1000, int skip = 100) { - clrQuery.Top = 1000; - clrQuery.Skip = 100; - clrQuery.Sort = new List { new SortNode("LastModified", SortOrder.Descending) }; + if (clrQuery.Take == long.MaxValue) + { + clrQuery.Take = take; + } + + if (clrQuery.Skip == 0) + { + clrQuery.Skip = skip; + } + + if (clrQuery.Sort.Count == 0) + { + clrQuery.Sort = new List + { + new SortNode("LastModified", SortOrder.Descending) + }; + } var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), clrQuery, SearchScope.All); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj index 0d756ab9b..c60353b65 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj @@ -20,6 +20,7 @@ + diff --git a/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs new file mode 100644 index 000000000..fa1e635a6 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs @@ -0,0 +1,64 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using Xunit; + +namespace Squidex.Infrastructure.Queries +{ + public class QueryTests + { + [Fact] + public void Should_add_fields_from_sorting() + { + var query = new ClrQuery + { + Sort = new List + { + new SortNode("field1", SortOrder.Ascending), + new SortNode("field1", SortOrder.Ascending), + new SortNode("field2", SortOrder.Ascending) + } + }; + + var fields = query.GetAllFields(); + + var expected = new HashSet + { + "field1", + "field2" + }; + + Assert.Equal(expected, fields); + } + + [Fact] + public void Should_add_fields_from_filters() + { + var query = new ClrQuery + { + Filter = + ClrFilter.And( + ClrFilter.Not( + ClrFilter.Eq("field1", 1)), + ClrFilter.Or( + ClrFilter.Eq("field2", 2), + ClrFilter.Eq("field2", 4))) + }; + + var fields = query.GetAllFields(); + + var expected = new HashSet + { + "field1", + "field2" + }; + + Assert.Equal(expected, fields); + } + } +} From 568037f8a2f01981f1b6ac8f2c29048a713654b5 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 28 May 2020 19:07:18 +0200 Subject: [PATCH 11/14] Fixes for rule event formatter. --- .../HandleRules/RuleEventFormatter.cs | 2 +- .../HandleRules/RuleEventFormatterTests.cs | 46 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index 6b9de787e..bd4ab1808 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -29,7 +29,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules { private const string Fallback = "null"; private static readonly Regex RegexPatternOld = new Regex(@"^(?(?[^_]*)_(?[^\s]*))", RegexOptions.Compiled); - private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[^_]*)_(?[^\s]*))([\s]*\|[\s]*(?.*)){0,1}\}", RegexOptions.Compiled); + private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[\w]*)_(?[\w\.\-]*))[\s]*(\|[\s]*(?[^\}]*)){0,1}\}", RegexOptions.Compiled); private readonly List<(string Pattern, Func Replacer)> patterns = new List<(string Pattern, Func Replacer)>(); private readonly IJsonSerializer jsonSerializer; private readonly IUrlGenerator urlGenerator; diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index 0c478d6df..a80910d2a 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -97,7 +97,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Name $APP_NAME has id $APP_ID")] - [InlineData("Name ${$EVENT_APPID.NAME} has id ${EVENT_APPID.ID}")] + [InlineData("Name ${EVENT_APPID.NAME} has id ${EVENT_APPID.ID}")] [InlineData("Script(`Name ${event.appId.name} has id ${event.appId.id}`)")] public void Should_format_app_information_from_event(string script) { @@ -543,15 +543,15 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules } [Theory] - [InlineData("Found in ${ASSET_FILENAME | Escape | Upper}.docx", "Found in DONALD\\\"DUCK .docx")] - [InlineData("Found in ${ASSET_FILENAME | Escape}.docx", "Found in Donald\\\"Duck .docx")] - [InlineData("Found in ${ASSET_FILENAME | Upper}.docx", "Found in DONALD\"DUCK .docx")] - [InlineData("Found in ${ASSET_FILENAME | Lower}.docx", "Found in donald\"duck .docx")] - [InlineData("Found in ${ASSET_FILENAME | Slugify}.docx", "Found in donald-duck.docx")] - [InlineData("Found in ${ASSET_FILENAME | Trim}.docx", "Found in Donald\"Duck.docx")] - public void Should_transform_replacements(string script, string expect) + [InlineData("Found in ${ASSET_FILENAME | Escape | Upper}.docx", "Found in DONALD\\\"DUCK.docx", "Donald\"Duck")] + [InlineData("Found in ${ASSET_FILENAME | Escape}.docx", "Found in Donald\\\"Duck.docx", "Donald\"Duck")] + [InlineData("Found in ${ASSET_FILENAME | Upper}.docx", "Found in DONALD DUCK.docx", "Donald Duck")] + [InlineData("Found in ${ASSET_FILENAME | Lower}.docx", "Found in donald duck.docx", "Donald Duck")] + [InlineData("Found in ${ASSET_FILENAME | Slugify}.docx", "Found in donald-duck.docx", "Donald Duck")] + [InlineData("Found in ${ASSET_FILENAME | Trim}.docx", "Found in Donald Duck.docx", "Donald Duck ")] + public void Should_transform_replacements(string script, string expect, string name) { - var @event = new EnrichedAssetEvent { FileName = "Donald\"Duck " }; + var @event = new EnrichedAssetEvent { FileName = name }; var result = sut.Format(script, @event); @@ -559,18 +559,30 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules } [Theory] - [InlineData("From ${USER_NAME | Escape | Upper}", "From DONALD\\\"DUCK ")] - [InlineData("From ${USER_NAME | Escape}", "From Donald\\\"Duck ")] - [InlineData("From ${USER_NAME | Upper}", "From DONALD\"DUCK ")] - [InlineData("From ${USER_NAME | Lower}", "From donald\"duck ")] - [InlineData("From ${USER_NAME | Slugify}", "From donald-duck")] - [InlineData("From ${USER_NAME | Trim}", "From Donald\"Duck")] - public void Should_transform_replacements_with_simple_pattern(string script, string expect) + [InlineData("From ${USER_NAME | Escape | Upper}", "From DONALD\\\"DUCK", "Donald\"Duck")] + [InlineData("From ${USER_NAME | Escape}", "From Donald\\\"Duck", "Donald\"Duck")] + [InlineData("From ${USER_NAME | Upper}", "From DONALD DUCK", "Donald Duck")] + [InlineData("From ${USER_NAME | Lower}", "From donald duck", "Donald Duck")] + [InlineData("From ${USER_NAME | Slugify}", "From donald-duck", "Donald Duck")] + [InlineData("From ${USER_NAME | Trim}", "From Donald Duck", "Donald Duck ")] + public void Should_transform_replacements_with_simple_pattern(string script, string expect, string name) { var @event = new EnrichedContentEvent { User = user }; A.CallTo(() => user.Claims) - .Returns(new List { new Claim(SquidexClaimTypes.DisplayName, "Donald\"Duck ") }); + .Returns(new List { new Claim(SquidexClaimTypes.DisplayName, name) }); + + var result = sut.Format(script, @event); + + Assert.Equal(expect, result); + } + + [Theory] + [InlineData("{'Key':'${ASSET_FILENAME | Upper}'}", "{'Key':'DONALD DUCK'}")] + [InlineData("{'Key':'${ASSET_FILENAME}'}", "{'Key':'Donald Duck'}")] + public void Should_transform_json_examples(string script, string expect) + { + var @event = new EnrichedAssetEvent { FileName = "Donald Duck" }; var result = sut.Format(script, @event); From 66e1af4f37430574e11da704b1c4313f26eef6d7 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 28 May 2020 21:39:09 +0200 Subject: [PATCH 12/14] Optional values in rule formatter. --- .../HandleRules/RuleEventFormatter.cs | 20 ++++- .../HandleRules/RuleEventFormatterTests.cs | 82 ++++++++++++------- 2 files changed, 68 insertions(+), 34 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index bd4ab1808..4a84a20fa 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -29,7 +29,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules { private const string Fallback = "null"; private static readonly Regex RegexPatternOld = new Regex(@"^(?(?[^_]*)_(?[^\s]*))", RegexOptions.Compiled); - private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[\w]*)_(?[\w\.\-]*))[\s]*(\|[\s]*(?[^\}]*)){0,1}\}", RegexOptions.Compiled); + private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[\w]+)_(?[\w\.\-]+))[\s]*(\|[\s]*(?[^\?}]+))?(\?[\s]*(?[^\}\s]+))?[\s]*\}", RegexOptions.Compiled); private readonly List<(string Pattern, Func Replacer)> patterns = new List<(string Pattern, Func Replacer)>(); private readonly IJsonSerializer jsonSerializer; private readonly IUrlGenerator urlGenerator; @@ -153,7 +153,19 @@ namespace Squidex.Domain.Apps.Core.HandleRules (length, text) = ResolveFromPath(match, @event); } - return (TransformText(text, match.Groups["Transform"]?.Value), length); + var result = TransformText(text, match.Groups["Transform"]?.Value); + + if (result == null) + { + result = match.Groups["Fallback"]?.Value; + } + + if (string.IsNullOrEmpty(result)) + { + result = Fallback; + } + + return (result, length); } return (Fallback, 0); @@ -327,7 +339,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules return null; } - private static string TransformText(string? text, string? transform) + private static string? TransformText(string? text, string? transform) { if (text != null && !string.IsNullOrWhiteSpace(transform)) { @@ -357,7 +369,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules } } - return text ?? Fallback; + return text; } private (int Length, string? Result) ResolveFromPath(Match match, EnrichedEvent @event) diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index a80910d2a..e941a6c27 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -489,44 +489,26 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules Assert.Equal("From client:android", result); } - [Fact] - public void Should_format_json() - { - var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "android") }; - - var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); - - Assert.Equal("{\"actor\":\"client:android\"}", result); - } - - [Fact] - public void Should_format_json_with_special_characters() - { - var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "mobile\"android") }; - - var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); - - Assert.Equal("{\"actor\":\"client:mobile\\\"android\"}", result); - } - - [Fact] - public void Should_evaluate_script_if_starting_with_whitespace() + [Theory] + [InlineData("${EVENT_INVALID ? file}", "file")] + public void Should_provide_fallback_if_path_is_invalid(string script, string expect) { - var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; + var @event = new EnrichedAssetEvent { FileName = null! }; - var result = sut.Format(" Script(`${event.type}`)", @event); + var result = sut.Format(script, @event); - Assert.Equal("Created", result); + Assert.Equal(expect, result); } - [Fact] - public void Should_evaluate_script_if_ends_with_whitespace() + [Theory] + [InlineData("${ASSET_FILENAME ? file}", "file")] + public void Should_provide_fallback_if_value_is_null(string script, string expect) { - var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; + var @event = new EnrichedAssetEvent { FileName = null! }; - var result = sut.Format("Script(`${event.type}`) ", @event); + var result = sut.Format(script, @event); - Assert.Equal("Created", result); + Assert.Equal(expect, result); } [Theory] @@ -588,5 +570,45 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules Assert.Equal(expect, result); } + + [Fact] + public void Should_format_json() + { + var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "android") }; + + var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); + + Assert.Equal("{\"actor\":\"client:android\"}", result); + } + + [Fact] + public void Should_format_json_with_special_characters() + { + var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "mobile\"android") }; + + var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); + + Assert.Equal("{\"actor\":\"client:mobile\\\"android\"}", result); + } + + [Fact] + public void Should_evaluate_script_if_starting_with_whitespace() + { + var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; + + var result = sut.Format(" Script(`${event.type}`)", @event); + + Assert.Equal("Created", result); + } + + [Fact] + public void Should_evaluate_script_if_ends_with_whitespace() + { + var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; + + var result = sut.Format("Script(`${event.type}`) ", @event); + + Assert.Equal("Created", result); + } } } From 3774f26294585fd79460311a6ce1309dcbface08 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 29 May 2020 09:47:30 +0200 Subject: [PATCH 13/14] Fixed scripting. --- .../Scripting/ScriptContext.cs | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs index e6ae1fe5f..e61338a12 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ScriptContext.cs @@ -47,24 +47,12 @@ namespace Squidex.Domain.Apps.Core.Scripting set => SetValue(value); } - public NamedContentData? DataOld - { - get => GetValue(); - set => SetValue(value); - } - public Status Status { get => GetValue(); set => SetValue(value); } - public Status StatusOld - { - get => GetValue(); - set => SetValue(value); - } - public string? AppName { get => GetValue(); @@ -77,19 +65,27 @@ namespace Squidex.Domain.Apps.Core.Scripting set => SetValue(value); } - [Obsolete] - public Status OldStatus + public NamedContentData? DataOld { - get => GetValue("statusOld"); + get => GetValue(); + set + { + SetValue(value, "oldData"); + SetValue(value); + } } - [Obsolete] - public NamedContentData? OldData + public Status StatusOld { - get => GetValue("dataOld"); + get => GetValue(); + set + { + SetValue(value, "oldStatus"); + SetValue(value); + } } - public void SetValue(object? value, [CallerMemberName] string? key = null) + public void SetValue(object? value, [CallerMemberNameAttribute] string? key = null) { if (key != null) { @@ -97,7 +93,7 @@ namespace Squidex.Domain.Apps.Core.Scripting } } - public T GetValue([CallerMemberName] string? key = null) + public T GetValue([CallerMemberNameAttribute] string? key = null) { if (key != null && TryGetValue(key, out var temp) && temp is T result) { From 80bb0642a8f1743c3caa7d78a8e0be7be793fcaa Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sat, 30 May 2020 08:43:49 +0200 Subject: [PATCH 14/14] Feature/async formatting2 (#528) * Make everything async ready. * Testing the extension interface and extracting predefined rules. * Resolve reference. * Do not check for data. * Timestamp transform. --- .../Actions/Algolia/AlgoliaActionHandler.cs | 7 +- .../AzureQueue/AzureQueueActionHandler.cs | 4 +- .../Actions/Comment/CommentActionHandler.cs | 4 +- .../Discourse/DiscourseActionHandler.cs | 6 +- .../ElasticSearchActionHandler.cs | 4 +- .../Actions/Email/EmailActionHandler.cs | 12 +- .../Actions/Kafka/KafkaActionHandler.cs | 12 +- .../Actions/Kafka/KafkaProducer.cs | 2 - .../Actions/Medium/MediumActionHandler.cs | 14 +- .../Notification/NotificationActionHandler.cs | 4 +- .../Prerender/PrerenderActionHandler.cs | 4 +- .../Actions/Slack/SlackActionHandler.cs | 4 +- .../Actions/Twitter/TweetActionHandler.cs | 4 +- .../Actions/Webhook/WebhookActionHandler.cs | 8 +- .../HandleRules/IRuleEventFormatter.cs | 25 ++ .../PredefinedPatternsFormatter.cs | 197 ++++++++ .../HandleRules/RuleActionHandler.cs | 8 +- .../HandleRules/RuleEventFormatter.cs | 425 ++++++------------ .../HandleRules/RuleVariable.cs | 101 +++++ .../Contents/ContentChangedTriggerHandler.cs | 44 +- .../Rules/RuleEnqueuer.cs | 18 +- .../src/Squidex/Config/Domain/RuleServices.cs | 5 +- .../HandleRules/RuleEventFormatterTests.cs | 211 +++++---- .../ContentChangedTriggerHandlerTests.cs | 106 ++++- .../Rules/RuleEnqueuerTests.cs | 6 + 25 files changed, 798 insertions(+), 437 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/IRuleEventFormatter.cs create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/PredefinedPatternsFormatter.cs create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleVariable.cs diff --git a/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs index 257ab688f..aeaf40fbb 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Algolia/AlgoliaActionHandler.cs @@ -33,7 +33,7 @@ namespace Squidex.Extensions.Actions.Algolia }); } - protected override (string Description, AlgoliaJob Data) CreateJob(EnrichedEvent @event, AlgoliaAction action) + protected override async Task<(string Description, AlgoliaJob Data)> CreateJobAsync(EnrichedEvent @event, AlgoliaAction action) { if (@event is EnrichedContentEvent contentEvent) { @@ -45,7 +45,7 @@ namespace Squidex.Extensions.Actions.Algolia AppId = action.AppId, ApiKey = action.ApiKey, ContentId = contentId, - IndexName = Format(action.IndexName, @event) + IndexName = await FormatAsync(action.IndexName, @event) }; if (contentEvent.Type == EnrichedContentEventType.Deleted || @@ -64,7 +64,8 @@ namespace Squidex.Extensions.Actions.Algolia if (!string.IsNullOrEmpty(action.Document)) { - jsonString = Format(action.Document, @event)?.Trim(); + jsonString = await FormatAsync(action.Document, @event); + jsonString = jsonString?.Trim(); } else { diff --git a/backend/extensions/Squidex.Extensions/Actions/AzureQueue/AzureQueueActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/AzureQueue/AzureQueueActionHandler.cs index 79e953045..8339a74de 100644 --- a/backend/extensions/Squidex.Extensions/Actions/AzureQueue/AzureQueueActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/AzureQueue/AzureQueueActionHandler.cs @@ -32,9 +32,9 @@ namespace Squidex.Extensions.Actions.AzureQueue }); } - protected override (string Description, AzureQueueJob Data) CreateJob(EnrichedEvent @event, AzureQueueAction action) + protected override async Task<(string Description, AzureQueueJob Data)> CreateJobAsync(EnrichedEvent @event, AzureQueueAction action) { - var queueName = Format(action.Queue, @event); + var queueName = await FormatAsync(action.Queue, @event); var ruleDescription = $"Send AzureQueueJob to azure queue '{queueName}'"; var ruleJob = new AzureQueueJob diff --git a/backend/extensions/Squidex.Extensions/Actions/Comment/CommentActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Comment/CommentActionHandler.cs index 25f053d9f..c2b89fb54 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Comment/CommentActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Comment/CommentActionHandler.cs @@ -30,11 +30,11 @@ namespace Squidex.Extensions.Actions.Comment this.commandBus = commandBus; } - protected override (string Description, CommentJob Data) CreateJob(EnrichedEvent @event, CommentAction action) + protected override async Task<(string Description, CommentJob Data)> CreateJobAsync(EnrichedEvent @event, CommentAction action) { if (@event is EnrichedContentEvent contentEvent) { - var text = Format(action.Text, @event); + var text = await FormatAsync(action.Text, @event); var actor = contentEvent.Actor; diff --git a/backend/extensions/Squidex.Extensions/Actions/Discourse/DiscourseActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Discourse/DiscourseActionHandler.cs index 6bec528fe..5d1576ab7 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Discourse/DiscourseActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Discourse/DiscourseActionHandler.cs @@ -28,13 +28,13 @@ namespace Squidex.Extensions.Actions.Discourse this.httpClientFactory = httpClientFactory; } - protected override (string Description, DiscourseJob Data) CreateJob(EnrichedEvent @event, DiscourseAction action) + protected override async Task<(string Description, DiscourseJob Data)> CreateJobAsync(EnrichedEvent @event, DiscourseAction action) { var url = $"{action.Url.ToString().TrimEnd('/')}/posts.json?api_key={action.ApiKey}&api_username={action.ApiUsername}"; var json = new Dictionary { - ["title"] = Format(action.Title, @event) + ["title"] = await FormatAsync(action.Title, @event) }; if (action.Topic.HasValue) @@ -47,7 +47,7 @@ namespace Squidex.Extensions.Actions.Discourse json.Add("category", action.Category.Value); } - json["raw"] = Format(action.Text, @event); + json["raw"] = await FormatAsync(action.Text, @event); var requestBody = ToJson(json); diff --git a/backend/extensions/Squidex.Extensions/Actions/ElasticSearch/ElasticSearchActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/ElasticSearch/ElasticSearchActionHandler.cs index cf0ec87fc..19e367783 100644 --- a/backend/extensions/Squidex.Extensions/Actions/ElasticSearch/ElasticSearchActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/ElasticSearch/ElasticSearchActionHandler.cs @@ -36,7 +36,7 @@ namespace Squidex.Extensions.Actions.ElasticSearch }); } - protected override (string Description, ElasticSearchJob Data) CreateJob(EnrichedEvent @event, ElasticSearchAction action) + protected override async Task<(string Description, ElasticSearchJob Data)> CreateJobAsync(EnrichedEvent @event, ElasticSearchAction action) { if (@event is EnrichedContentEvent contentEvent) { @@ -46,7 +46,7 @@ namespace Squidex.Extensions.Actions.ElasticSearch var ruleJob = new ElasticSearchJob { - IndexName = Format(action.IndexName, @event), + IndexName = await FormatAsync(action.IndexName, @event), ServerHost = action.Host.ToString(), ServerUser = action.Username, ServerPassword = action.Password, diff --git a/backend/extensions/Squidex.Extensions/Actions/Email/EmailActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Email/EmailActionHandler.cs index ea23d1eae..499f2f75d 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Email/EmailActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Email/EmailActionHandler.cs @@ -23,7 +23,7 @@ namespace Squidex.Extensions.Actions.Email { } - protected override (string Description, EmailJob Data) CreateJob(EnrichedEvent @event, EmailAction action) + protected override async Task<(string Description, EmailJob Data)> CreateJobAsync(EnrichedEvent @event, EmailAction action) { var ruleJob = new EmailJob { @@ -31,11 +31,11 @@ namespace Squidex.Extensions.Actions.Email ServerUseSsl = action.ServerUseSsl, ServerPassword = action.ServerPassword, ServerPort = action.ServerPort, - ServerUsername = Format(action.ServerUsername, @event), - MessageFrom = Format(action.MessageFrom, @event), - MessageTo = Format(action.MessageTo, @event), - MessageSubject = Format(action.MessageSubject, @event), - MessageBody = Format(action.MessageBody, @event) + ServerUsername = await FormatAsync(action.ServerUsername, @event), + MessageFrom = await FormatAsync(action.MessageFrom, @event), + MessageTo = await FormatAsync(action.MessageTo, @event), + MessageSubject = await FormatAsync(action.MessageSubject, @event), + MessageBody = await FormatAsync(action.MessageBody, @event) }; var description = $"Send an email to {action.MessageTo}"; diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs index dd5964e83..aa7b060fc 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaActionHandler.cs @@ -27,13 +27,13 @@ namespace Squidex.Extensions.Actions.Kafka this.kafkaProducer = kafkaProducer; } - protected override (string Description, KafkaJob Data) CreateJob(EnrichedEvent @event, KafkaAction action) + protected override async Task<(string Description, KafkaJob Data)> CreateJobAsync(EnrichedEvent @event, KafkaAction action) { string value, key; if (!string.IsNullOrEmpty(action.Payload)) { - value = Format(action.Payload, @event); + value = await FormatAsync(action.Payload, @event); } else { @@ -42,7 +42,7 @@ namespace Squidex.Extensions.Actions.Kafka if (!string.IsNullOrEmpty(action.Key)) { - key = Format(action.Key, @event); + key = await FormatAsync(action.Key, @event); } else { @@ -54,14 +54,14 @@ namespace Squidex.Extensions.Actions.Kafka TopicName = action.TopicName, MessageKey = key, MessageValue = value, - Headers = ParseHeaders(action.Headers, @event), + Headers = await ParseHeadersAsync(action.Headers, @event), Schema = action.Schema }; return (Description, ruleJob); } - private Dictionary ParseHeaders(string headers, EnrichedEvent @event) + private async Task> ParseHeadersAsync(string headers, EnrichedEvent @event) { if (string.IsNullOrWhiteSpace(headers)) { @@ -81,7 +81,7 @@ namespace Squidex.Extensions.Actions.Kafka var key = line.Substring(0, indexEqual); var val = line.Substring(indexEqual + 1); - val = Format(val, @event); + val = await FormatAsync(val, @event); headersDictionary[key] = val; } diff --git a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs index a8f2ecfaa..ec7d8c5b7 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Kafka/KafkaProducer.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; using System.Collections.Generic; using System.Threading.Tasks; using Avro; @@ -13,7 +12,6 @@ using Avro.Generic; using Confluent.Kafka; using Confluent.SchemaRegistry; using Confluent.SchemaRegistry.Serdes; -using GraphQL.Types; using Microsoft.Extensions.Options; using Squidex.Infrastructure.Json; using Squidex.Infrastructure.Json.Objects; diff --git a/backend/extensions/Squidex.Extensions/Actions/Medium/MediumActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Medium/MediumActionHandler.cs index 9af62802b..4ed8564ce 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Medium/MediumActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Medium/MediumActionHandler.cs @@ -42,17 +42,17 @@ namespace Squidex.Extensions.Actions.Medium this.serializer = serializer; } - protected override (string Description, MediumJob Data) CreateJob(EnrichedEvent @event, MediumAction action) + protected override async Task<(string Description, MediumJob Data)> CreateJobAsync(EnrichedEvent @event, MediumAction action) { var ruleJob = new MediumJob { AccessToken = action.AccessToken, PublicationId = action.PublicationId }; var requestBody = new { - title = Format(action.Title, @event), + title = await FormatAsync(action.Title, @event), contentFormat = action.IsHtml ? "html" : "markdown", - content = Format(action.Content, @event), - canonicalUrl = Format(action.CanonicalUrl, @event), - tags = ParseTags(@event, action) + content = await FormatAsync(action.Content, @event), + canonicalUrl = await FormatAsync(action.CanonicalUrl, @event), + tags = await ParseTagsAsync(@event, action) }; ruleJob.RequestBody = ToJson(requestBody); @@ -60,7 +60,7 @@ namespace Squidex.Extensions.Actions.Medium return (Description, ruleJob); } - private string[] ParseTags(EnrichedEvent @event, MediumAction action) + private async Task ParseTagsAsync(EnrichedEvent @event, MediumAction action) { if (string.IsNullOrWhiteSpace(action.Tags)) { @@ -69,7 +69,7 @@ namespace Squidex.Extensions.Actions.Medium try { - var jsonTags = Format(action.Tags, @event); + var jsonTags = await FormatAsync(action.Tags, @event); return serializer.Deserialize(jsonTags); } diff --git a/backend/extensions/Squidex.Extensions/Actions/Notification/NotificationActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Notification/NotificationActionHandler.cs index 42f8580de..0093b2335 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Notification/NotificationActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Notification/NotificationActionHandler.cs @@ -40,7 +40,7 @@ namespace Squidex.Extensions.Actions.Notification { if (@event is EnrichedUserEventBase userEvent) { - var text = Format(action.Text, @event); + var text = await FormatAsync(action.Text, @event); var actor = userEvent.Actor; @@ -60,7 +60,7 @@ namespace Squidex.Extensions.Actions.Notification if (!string.IsNullOrWhiteSpace(action.Url)) { - var url = Format(action.Url, @event); + var url = await FormatAsync(action.Url, @event); if (Uri.TryCreate(url, UriKind.RelativeOrAbsolute, out var uri)) { diff --git a/backend/extensions/Squidex.Extensions/Actions/Prerender/PrerenderActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Prerender/PrerenderActionHandler.cs index 2b4e83be8..3e35e6a9c 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Prerender/PrerenderActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Prerender/PrerenderActionHandler.cs @@ -24,9 +24,9 @@ namespace Squidex.Extensions.Actions.Prerender this.httpClientFactory = httpClientFactory; } - protected override (string Description, PrerenderJob Data) CreateJob(EnrichedEvent @event, PrerenderAction action) + protected override async Task<(string Description, PrerenderJob Data)> CreateJobAsync(EnrichedEvent @event, PrerenderAction action) { - var url = Format(action.Url, @event); + var url = await FormatAsync(action.Url, @event); var request = new { prerenderToken = action.Token, url }; var requestBody = ToJson(request); diff --git a/backend/extensions/Squidex.Extensions/Actions/Slack/SlackActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Slack/SlackActionHandler.cs index 39779a5b9..0143e52e6 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Slack/SlackActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Slack/SlackActionHandler.cs @@ -30,9 +30,9 @@ namespace Squidex.Extensions.Actions.Slack this.httpClientFactory = httpClientFactory; } - protected override (string Description, SlackJob Data) CreateJob(EnrichedEvent @event, SlackAction action) + protected override async Task<(string Description, SlackJob Data)> CreateJobAsync(EnrichedEvent @event, SlackAction action) { - var body = new { text = Format(action.Text, @event) }; + var body = new { text = await FormatAsync(action.Text, @event) }; var ruleJob = new SlackJob { diff --git a/backend/extensions/Squidex.Extensions/Actions/Twitter/TweetActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Twitter/TweetActionHandler.cs index 83c95e790..a7cfebe09 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Twitter/TweetActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Twitter/TweetActionHandler.cs @@ -30,11 +30,11 @@ namespace Squidex.Extensions.Actions.Twitter this.twitterOptions = twitterOptions.Value; } - protected override (string Description, TweetJob Data) CreateJob(EnrichedEvent @event, TweetAction action) + protected override async Task<(string Description, TweetJob Data)> CreateJobAsync(EnrichedEvent @event, TweetAction action) { var ruleJob = new TweetJob { - Text = Format(action.Text, @event), + Text = await FormatAsync(action.Text, @event), AccessToken = action.AccessToken, AccessSecret = action.AccessSecret }; diff --git a/backend/extensions/Squidex.Extensions/Actions/Webhook/WebhookActionHandler.cs b/backend/extensions/Squidex.Extensions/Actions/Webhook/WebhookActionHandler.cs index 16cb53a36..6cd86eb5d 100644 --- a/backend/extensions/Squidex.Extensions/Actions/Webhook/WebhookActionHandler.cs +++ b/backend/extensions/Squidex.Extensions/Actions/Webhook/WebhookActionHandler.cs @@ -27,25 +27,25 @@ namespace Squidex.Extensions.Actions.Webhook this.httpClientFactory = httpClientFactory; } - protected override (string Description, WebhookJob Data) CreateJob(EnrichedEvent @event, WebhookAction action) + protected override async Task<(string Description, WebhookJob Data)> CreateJobAsync(EnrichedEvent @event, WebhookAction action) { string requestBody; if (!string.IsNullOrEmpty(action.Payload)) { - requestBody = Format(action.Payload, @event); + requestBody = await FormatAsync(action.Payload, @event); } else { requestBody = ToEnvelopeJson(@event); } - var requestUrl = Format(action.Url, @event); + var requestUrl = await FormatAsync(action.Url, @event); var ruleDescription = $"Send event to webhook '{requestUrl}'"; var ruleJob = new WebhookJob { - RequestUrl = Format(action.Url.ToString(), @event), + RequestUrl = await FormatAsync(action.Url.ToString(), @event), RequestSignature = $"{requestBody}{action.SharedSecret}".Sha256Base64(), RequestBody = requestBody }; diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/IRuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/IRuleEventFormatter.cs new file mode 100644 index 000000000..8fc4762de --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/IRuleEventFormatter.cs @@ -0,0 +1,25 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; + +namespace Squidex.Domain.Apps.Core.HandleRules +{ + public interface IRuleEventFormatter + { + (bool Match, string?, int ReplacedLength) Format(EnrichedEvent @event, string text) + { + return default; + } + + (bool Match, ValueTask) Format(EnrichedEvent @event, object value, string[] path) + { + return default; + } + } +} \ No newline at end of file diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/PredefinedPatternsFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/PredefinedPatternsFormatter.cs new file mode 100644 index 000000000..f2877da5f --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/PredefinedPatternsFormatter.cs @@ -0,0 +1,197 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Globalization; +using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; +using Squidex.Infrastructure; +using Squidex.Shared.Users; + +namespace Squidex.Domain.Apps.Core.HandleRules +{ + public sealed class PredefinedPatternsFormatter : IRuleEventFormatter + { + private readonly List<(string Pattern, Func Replacer)> patterns = new List<(string Pattern, Func Replacer)>(); + private readonly IUrlGenerator urlGenerator; + + public PredefinedPatternsFormatter(IUrlGenerator urlGenerator) + { + Guard.NotNull(urlGenerator, nameof(urlGenerator)); + + this.urlGenerator = urlGenerator; + + AddPattern("APP_ID", AppId); + AddPattern("APP_NAME", AppName); + AddPattern("ASSET_CONTENT_URL", AssetContentUrl); + AddPattern("CONTENT_ACTION", ContentAction); + AddPattern("CONTENT_URL", ContentUrl); + AddPattern("MENTIONED_ID", MentionedId); + AddPattern("MENTIONED_NAME", MentionedName); + AddPattern("MENTIONED_EMAIL", MentionedEmail); + AddPattern("SCHEMA_ID", SchemaId); + AddPattern("SCHEMA_NAME", SchemaName); + AddPattern("TIMESTAMP_DATETIME", TimestampTime); + AddPattern("TIMESTAMP_DATE", TimestampDate); + AddPattern("USER_ID", UserId); + AddPattern("USER_NAME", UserName); + AddPattern("USER_EMAIL", UserEmail); + } + + private void AddPattern(string placeholder, Func generator) + { + patterns.Add((placeholder, generator)); + } + + public (bool Match, string?, int ReplacedLength) Format(EnrichedEvent @event, string text) + { + for (var j = 0; j < patterns.Count; j++) + { + var (pattern, replacer) = patterns[j]; + + if (text.StartsWith(pattern, StringComparison.OrdinalIgnoreCase)) + { + var result = replacer(@event); + + return (true, result, pattern.Length); + } + } + + return default; + } + + private static string TimestampDate(EnrichedEvent @event) + { + return @event.Timestamp.ToDateTimeUtc().ToString("yyy-MM-dd", CultureInfo.InvariantCulture); + } + + private static string TimestampTime(EnrichedEvent @event) + { + return @event.Timestamp.ToDateTimeUtc().ToString("yyy-MM-dd-hh-mm-ss", CultureInfo.InvariantCulture); + } + + private static string AppId(EnrichedEvent @event) + { + return @event.AppId.Id.ToString(); + } + + private static string AppName(EnrichedEvent @event) + { + return @event.AppId.Name; + } + + private static string? SchemaId(EnrichedEvent @event) + { + if (@event is EnrichedSchemaEventBase schemaEvent) + { + return schemaEvent.SchemaId.Id.ToString(); + } + + return null; + } + + private static string? SchemaName(EnrichedEvent @event) + { + if (@event is EnrichedSchemaEventBase schemaEvent) + { + return schemaEvent.SchemaId.Name; + } + + return null; + } + + private static string? ContentAction(EnrichedEvent @event) + { + if (@event is EnrichedContentEvent contentEvent) + { + return contentEvent.Type.ToString(); + } + + return null; + } + + private string? AssetContentUrl(EnrichedEvent @event) + { + if (@event is EnrichedAssetEvent assetEvent) + { + return urlGenerator.AssetContent(assetEvent.Id); + } + + return null; + } + + private string? ContentUrl(EnrichedEvent @event) + { + if (@event is EnrichedContentEvent contentEvent) + { + return urlGenerator.ContentUI(contentEvent.AppId, contentEvent.SchemaId, contentEvent.Id); + } + + return null; + } + + private static string? UserName(EnrichedEvent @event) + { + if (@event is EnrichedUserEventBase userEvent) + { + return userEvent.User?.DisplayName(); + } + + return null; + } + + private static string? UserId(EnrichedEvent @event) + { + if (@event is EnrichedUserEventBase userEvent) + { + return userEvent.User?.Id; + } + + return null; + } + + private static string? UserEmail(EnrichedEvent @event) + { + if (@event is EnrichedUserEventBase userEvent) + { + return userEvent.User?.Email; + } + + return null; + } + + private static string? MentionedName(EnrichedEvent @event) + { + if (@event is EnrichedCommentEvent commentEvent) + { + return commentEvent.MentionedUser.DisplayName(); + } + + return null; + } + + private static string? MentionedId(EnrichedEvent @event) + { + if (@event is EnrichedCommentEvent commentEvent) + { + return commentEvent.MentionedUser.Id; + } + + return null; + } + + private static string? MentionedEmail(EnrichedEvent @event) + { + if (@event is EnrichedCommentEvent commentEvent) + { + return commentEvent.MentionedUser.Email; + } + + return null; + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleActionHandler.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleActionHandler.cs index 92f0b3256..e9b3fb1f1 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleActionHandler.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleActionHandler.cs @@ -47,14 +47,14 @@ namespace Squidex.Domain.Apps.Core.HandleRules return formatter.ToEnvelope(@event); } - protected string? Format(Uri uri, EnrichedEvent @event) + protected ValueTask FormatAsync(Uri uri, EnrichedEvent @event) { - return formatter.Format(uri.ToString(), @event); + return formatter.FormatAsync(uri.ToString(), @event); } - protected string? Format(string text, EnrichedEvent @event) + protected ValueTask FormatAsync(string text, EnrichedEvent @event) { - return formatter.Format(text, @event); + return formatter.FormatAsync(text, @event); } async Task<(string Description, object Data)> IRuleActionHandler.CreateJobAsync(EnrichedEvent @event, RuleAction action) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs index 4a84a20fa..c1b807b75 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleEventFormatter.cs @@ -7,64 +7,72 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.Linq; -using System.Reflection; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using Newtonsoft.Json; -using Squidex.Domain.Apps.Core.Contents; +using NodaTime.Text; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Scripting; using Squidex.Infrastructure; using Squidex.Infrastructure.Json; -using Squidex.Infrastructure.Json.Objects; -using Squidex.Shared.Identity; -using Squidex.Shared.Users; namespace Squidex.Domain.Apps.Core.HandleRules { public class RuleEventFormatter { - private const string Fallback = "null"; + private const string GlobalFallback = "null"; private static readonly Regex RegexPatternOld = new Regex(@"^(?(?[^_]*)_(?[^\s]*))", RegexOptions.Compiled); private static readonly Regex RegexPatternNew = new Regex(@"^\{(?(?[\w]+)_(?[\w\.\-]+))[\s]*(\|[\s]*(?[^\?}]+))?(\?[\s]*(?[^\}\s]+))?[\s]*\}", RegexOptions.Compiled); - private readonly List<(string Pattern, Func Replacer)> patterns = new List<(string Pattern, Func Replacer)>(); private readonly IJsonSerializer jsonSerializer; - private readonly IUrlGenerator urlGenerator; + private readonly IEnumerable formatters; private readonly IScriptEngine scriptEngine; - public RuleEventFormatter(IJsonSerializer jsonSerializer, IUrlGenerator urlGenerator, IScriptEngine scriptEngine) + private struct TextPart + { + public bool IsText; + + public int Length; + + public int Offset; + + public string Fallback; + + public string Transform; + + public ValueTask Replacement; + + public static TextPart Text(int offset, int length) + { + var result = default(TextPart); + result.Offset = offset; + result.Length = length; + result.IsText = true; + + return result; + } + + public static TextPart Variable(ValueTask replacement, string fallback, string transform) + { + var result = default(TextPart); + result.Replacement = replacement; + result.Fallback = fallback; + result.Transform = transform; + + return result; + } + } + + public RuleEventFormatter(IJsonSerializer jsonSerializer, IEnumerable formatters, IScriptEngine scriptEngine) { Guard.NotNull(jsonSerializer, nameof(jsonSerializer)); Guard.NotNull(scriptEngine, nameof(scriptEngine)); - Guard.NotNull(urlGenerator, nameof(urlGenerator)); + Guard.NotNull(formatters, nameof(formatters)); this.jsonSerializer = jsonSerializer; + this.formatters = formatters; this.scriptEngine = scriptEngine; - this.urlGenerator = urlGenerator; - - AddPattern("APP_ID", AppId); - AddPattern("APP_NAME", AppName); - AddPattern("ASSET_CONTENT_URL", AssetContentUrl); - AddPattern("CONTENT_ACTION", ContentAction); - AddPattern("CONTENT_URL", ContentUrl); - AddPattern("MENTIONED_ID", MentionedId); - AddPattern("MENTIONED_NAME", MentionedName); - AddPattern("MENTIONED_EMAIL", MentionedEmail); - AddPattern("SCHEMA_ID", SchemaId); - AddPattern("SCHEMA_NAME", SchemaName); - AddPattern("TIMESTAMP_DATETIME", TimestampTime); - AddPattern("TIMESTAMP_DATE", TimestampDate); - AddPattern("USER_ID", UserId); - AddPattern("USER_NAME", UserName); - AddPattern("USER_EMAIL", UserEmail); - } - - private void AddPattern(string placeholder, Func generator) - { - patterns.Add((placeholder, generator)); } public virtual string ToPayload(T @event) @@ -77,7 +85,7 @@ namespace Squidex.Domain.Apps.Core.HandleRules return jsonSerializer.Serialize(new { type = @event.Name, payload = @event, timestamp = @event.Timestamp }); } - public string? Format(string text, EnrichedEvent @event) + public async ValueTask FormatAsync(string text, EnrichedEvent @event) { if (string.IsNullOrWhiteSpace(text)) { @@ -94,11 +102,55 @@ namespace Squidex.Domain.Apps.Core.HandleRules return scriptEngine.Interpolate(context, script); } + var parts = BuildParts(text, @event); + + await Task.WhenAll(parts.Select(x => x.Replacement.AsTask())); + + return CombineParts(text, parts); + } + + private string CombineParts(string text, List parts) + { var span = text.AsSpan(); - var currentOffset = 0; + var sb = new StringBuilder(); + + foreach (var part in parts) + { + if (!part.IsText) + { + var result = part.Replacement.Result; + + result = TransformText(result, part.Transform); + + if (result == null) + { + result = part.Fallback; + } + + if (string.IsNullOrEmpty(result)) + { + result = GlobalFallback; + } + + sb.Append(result); + } + else + { + sb.Append(span.Slice(part.Offset, part.Length)); + } + } + + return sb.ToString(); + } + + private List BuildParts(string text, EnrichedEvent @event) + { + var parts = new List(); - var parts = new List<(int Offset, int Length, ValueTask Task)>(); + var span = text.AsSpan(); + + var currentOffset = 0; for (var i = 0; i < text.Length; i++) { @@ -106,13 +158,13 @@ namespace Squidex.Domain.Apps.Core.HandleRules if (c == '$') { - parts.Add((currentOffset, i - currentOffset, default)); + parts.Add(TextPart.Text(currentOffset, i - currentOffset)); - var (replacement, length) = GetReplacement(span.Slice(i + 1).ToString(), @event); + var (length, part) = GetReplacement(span.Slice(i + 1).ToString(), @event); if (length > 0) { - parts.Add((0, 0, new ValueTask(replacement))); + parts.Add(part); i += length + 1; } @@ -121,54 +173,28 @@ namespace Squidex.Domain.Apps.Core.HandleRules } } - parts.Add((currentOffset, text.Length - currentOffset, default)); - - var sb = new StringBuilder(); - - foreach (var (offset, length, task) in parts) - { - if (task.Result != null) - { - sb.Append(task.Result); - } - else - { - sb.Append(span.Slice(offset, length)); - } - } + parts.Add(TextPart.Text(currentOffset, text.Length - currentOffset)); - return sb.ToString(); + return parts; } - private (string Result, int Length) GetReplacement(string test, EnrichedEvent @event) + private (int Length, TextPart Part) GetReplacement(string test, EnrichedEvent @event) { var (isNewRegex, match) = Match(test); if (match.Success) { - var (length, text) = ResolveOldPatterns(match, isNewRegex, @event); + var (length, replacement) = ResolveOldPatterns(match, isNewRegex, @event); if (length == 0) { - (length, text) = ResolveFromPath(match, @event); - } - - var result = TransformText(text, match.Groups["Transform"]?.Value); - - if (result == null) - { - result = match.Groups["Fallback"]?.Value; + (length, replacement) = ResolveFromPath(match, @event); } - if (string.IsNullOrEmpty(result)) - { - result = Fallback; - } - - return (result, length); + return (length, TextPart.Variable(replacement, match.Groups["Fallback"].Value, match.Groups["Transform"].Value)); } - return (Fallback, 0); + return default; } private (bool IsNew, Match) Match(string test) @@ -183,162 +209,28 @@ namespace Squidex.Domain.Apps.Core.HandleRules return (false, RegexPatternOld.Match(test)); } - private (int Length, string? Result) ResolveOldPatterns(Match match, bool isNewRegex, EnrichedEvent @event) + private (int Length, ValueTask Result) ResolveOldPatterns(Match match, bool isNewRegex, EnrichedEvent @event) { var fullPath = match.Groups["FullPath"].Value; - for (var j = 0; j < patterns.Count; j++) + foreach (var formatter in formatters) { - var (pattern, replacer) = patterns[j]; + var (replaced, result, replacedLength) = formatter.Format(@event, fullPath); - if (fullPath.StartsWith(pattern, StringComparison.OrdinalIgnoreCase)) + if (replaced) { - var result = replacer(@event); - if (isNewRegex) { - return (match.Length, result); - } - else - { - return (pattern.Length, result); + replacedLength = match.Length; } + + return (replacedLength, new ValueTask(result)); } } return default; } - private static string TimestampDate(EnrichedEvent @event) - { - return @event.Timestamp.ToDateTimeUtc().ToString("yyy-MM-dd", CultureInfo.InvariantCulture); - } - - private static string TimestampTime(EnrichedEvent @event) - { - return @event.Timestamp.ToDateTimeUtc().ToString("yyy-MM-dd-hh-mm-ss", CultureInfo.InvariantCulture); - } - - private static string AppId(EnrichedEvent @event) - { - return @event.AppId.Id.ToString(); - } - - private static string AppName(EnrichedEvent @event) - { - return @event.AppId.Name; - } - - private static string? SchemaId(EnrichedEvent @event) - { - if (@event is EnrichedSchemaEventBase schemaEvent) - { - return schemaEvent.SchemaId.Id.ToString(); - } - - return null; - } - - private static string? SchemaName(EnrichedEvent @event) - { - if (@event is EnrichedSchemaEventBase schemaEvent) - { - return schemaEvent.SchemaId.Name; - } - - return null; - } - - private static string? ContentAction(EnrichedEvent @event) - { - if (@event is EnrichedContentEvent contentEvent) - { - return contentEvent.Type.ToString(); - } - - return null; - } - - private string? AssetContentUrl(EnrichedEvent @event) - { - if (@event is EnrichedAssetEvent assetEvent) - { - return urlGenerator.AssetContent(assetEvent.Id); - } - - return null; - } - - private string? ContentUrl(EnrichedEvent @event) - { - if (@event is EnrichedContentEvent contentEvent) - { - return urlGenerator.ContentUI(contentEvent.AppId, contentEvent.SchemaId, contentEvent.Id); - } - - return null; - } - - private static string? UserName(EnrichedEvent @event) - { - if (@event is EnrichedUserEventBase userEvent) - { - return userEvent.User?.DisplayName(); - } - - return null; - } - - private static string? UserId(EnrichedEvent @event) - { - if (@event is EnrichedUserEventBase userEvent) - { - return userEvent.User?.Id; - } - - return null; - } - - private static string? UserEmail(EnrichedEvent @event) - { - if (@event is EnrichedUserEventBase userEvent) - { - return userEvent.User?.Email; - } - - return null; - } - - private static string? MentionedName(EnrichedEvent @event) - { - if (@event is EnrichedCommentEvent commentEvent) - { - return commentEvent.MentionedUser.DisplayName(); - } - - return null; - } - - private static string? MentionedId(EnrichedEvent @event) - { - if (@event is EnrichedCommentEvent commentEvent) - { - return commentEvent.MentionedUser.Id; - } - - return null; - } - - private static string? MentionedEmail(EnrichedEvent @event) - { - if (@event is EnrichedCommentEvent commentEvent) - { - return commentEvent.MentionedUser.Email; - } - - return null; - } - private static string? TransformText(string? text, string? transform) { if (text != null && !string.IsNullOrWhiteSpace(transform)) @@ -365,6 +257,29 @@ namespace Squidex.Domain.Apps.Core.HandleRules case "trim": text = text.Trim(); break; + case "timestamp_ms": + { + var instant = InstantPattern.General.Parse(text); + + if (instant.Success) + { + text = instant.Value.ToUnixTimeMilliseconds().ToString(); + } + + break; + } + + case "timestamp_seconds": + { + var instant = InstantPattern.General.Parse(text); + + if (instant.Success) + { + text = instant.Value.ToUnixTimeSeconds().ToString(); + } + + break; + } } } } @@ -372,90 +287,30 @@ namespace Squidex.Domain.Apps.Core.HandleRules return text; } - private (int Length, string? Result) ResolveFromPath(Match match, EnrichedEvent @event) + private (int Length, ValueTask Result) ResolveFromPath(Match match, EnrichedEvent @event) { var path = match.Groups["Path"].Value.Split('.', StringSplitOptions.RemoveEmptyEntries); - var result = CalculateData(@event, path); - - return (match.Length, result); - } - - private static string? CalculateData(object @event, string[] path) - { - object? current = @event; + var (result, remaining) = RuleVariable.GetValue(@event, path); - foreach (var segment in path) + if (remaining.Length > 0 && result != null) { - if (current is NamedContentData data) + foreach (var formatter in formatters) { - if (!data.TryGetValue(segment, out var temp) || temp == null) - { - return null; - } + var (replaced, result2) = formatter.Format(@event, result, remaining); - current = temp; - } - else if (current is ContentFieldData field) - { - if (!field.TryGetValue(segment, out var temp) || temp == null) + if (replaced) { - return null; + return (match.Length, result2); } - - current = temp; - } - else if (current is IJsonValue json) - { - if (!json.TryGet(segment, out var temp) || temp == null || temp.Type == JsonValueType.Null) - { - return null; - } - - current = temp; - } - else if (current != null) - { - if (current is IUser user) - { - var type = segment; - - if (string.Equals(type, "Name", StringComparison.OrdinalIgnoreCase)) - { - type = SquidexClaimTypes.DisplayName; - } - - var claim = user.Claims.FirstOrDefault(x => string.Equals(x.Type, type, StringComparison.OrdinalIgnoreCase)); - - if (claim != null) - { - current = claim.Value; - continue; - } - } - - const BindingFlags bindingFlags = - BindingFlags.FlattenHierarchy | - BindingFlags.Public | - BindingFlags.Instance; - - var properties = current.GetType().GetProperties(bindingFlags); - var property = properties.FirstOrDefault(x => x.CanRead && string.Equals(x.Name, segment, StringComparison.OrdinalIgnoreCase)); - - if (property == null) - { - return null; - } - - current = property.GetValue(current); - } - else - { - return null; } } + else if (remaining.Length == 0) + { + return (match.Length, new ValueTask(result?.ToString())); + } - return current?.ToString(); + return (match.Length, default); } private static bool TryGetScript(string text, out string script) diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleVariable.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleVariable.cs new file mode 100644 index 000000000..f4a1f7665 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/RuleVariable.cs @@ -0,0 +1,101 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Linq; +using System.Reflection; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Infrastructure.Json.Objects; +using Squidex.Shared.Identity; +using Squidex.Shared.Users; + +namespace Squidex.Domain.Apps.Core.HandleRules +{ + public static class RuleVariable + { + public static (object? Result, string[] Remaining) GetValue(object @event, string[] path) + { + object? current = @event; + + var i = 0; + + for (; i < path.Length; i++) + { + var segment = path[i]; + + if (current is NamedContentData data) + { + if (!data.TryGetValue(segment, out var temp) || temp == null) + { + break; + } + + current = temp; + } + else if (current is ContentFieldData field) + { + if (!field.TryGetValue(segment, out var temp) || temp == null) + { + break; + } + + current = temp; + } + else if (current is IJsonValue json) + { + if (!json.TryGet(segment, out var temp) || temp == null || temp.Type == JsonValueType.Null) + { + break; + } + + current = temp; + } + else if (current != null) + { + if (current is IUser user) + { + var type = segment; + + if (string.Equals(type, "Name", StringComparison.OrdinalIgnoreCase)) + { + type = SquidexClaimTypes.DisplayName; + } + + var claim = user.Claims.FirstOrDefault(x => string.Equals(x.Type, type, StringComparison.OrdinalIgnoreCase)); + + if (claim != null) + { + current = claim.Value; + continue; + } + } + + const BindingFlags bindingFlags = + BindingFlags.FlattenHierarchy | + BindingFlags.Public | + BindingFlags.Instance; + + var properties = current.GetType().GetProperties(bindingFlags); + var property = properties.FirstOrDefault(x => x.CanRead && string.Equals(x.Name, segment, StringComparison.OrdinalIgnoreCase)); + + if (property == null) + { + break; + } + + current = property.GetValue(current); + } + else + { + break; + } + } + + return (current, path.Skip(i).ToArray()); + } + } +} \ No newline at end of file diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentChangedTriggerHandler.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentChangedTriggerHandler.cs index f1bc57915..05401a941 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentChangedTriggerHandler.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentChangedTriggerHandler.cs @@ -14,24 +14,62 @@ using Squidex.Domain.Apps.Core.Rules.Triggers; using Squidex.Domain.Apps.Core.Scripting; using Squidex.Domain.Apps.Events.Contents; using Squidex.Infrastructure; +using Squidex.Infrastructure.Caching; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.Reflection; namespace Squidex.Domain.Apps.Entities.Contents { - public sealed class ContentChangedTriggerHandler : RuleTriggerHandler + public sealed class ContentChangedTriggerHandler : RuleTriggerHandler, IRuleEventFormatter { private readonly IScriptEngine scriptEngine; private readonly IContentLoader contentLoader; + private readonly ILocalCache localCache; - public ContentChangedTriggerHandler(IScriptEngine scriptEngine, IContentLoader contentLoader) + public ContentChangedTriggerHandler(IScriptEngine scriptEngine, IContentLoader contentLoader, ILocalCache localCache) { Guard.NotNull(scriptEngine, nameof(scriptEngine)); Guard.NotNull(contentLoader, nameof(contentLoader)); + Guard.NotNull(localCache, nameof(localCache)); this.scriptEngine = scriptEngine; - this.contentLoader = contentLoader; + this.localCache = localCache; + } + + public (bool Match, ValueTask) Format(EnrichedEvent @event, object value, string[] path) + { + if (value is JsonArray array && + array.Count > 0 && + array[0] is JsonString s && + Guid.TryParse(s.Value, out var referenceId)) + { + return (true, GetReferenceValueAsync(referenceId, path)); + } + + return default; + } + + private async ValueTask GetReferenceValueAsync(Guid referenceId, string[] path) + { + var reference = await GetContentFromCacheAsync(referenceId); + + var (result, remaining) = RuleVariable.GetValue(reference, path); + + if (remaining.Length == 0) + { + return result?.ToString(); + } + + return default; + } + + private Task GetContentFromCacheAsync(Guid referenceId) + { + var cacheKey = $"FORMAT_REFERENCE_{referenceId}"; + + return localCache.GetOrCreate(cacheKey, () => contentLoader.GetAsync(referenceId)); } protected override async Task CreateEnrichedEventAsync(Envelope @event) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs b/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs index f934c86c8..40fa07190 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs @@ -14,6 +14,7 @@ using Squidex.Domain.Apps.Core.Rules; using Squidex.Domain.Apps.Entities.Rules.Repositories; using Squidex.Domain.Apps.Events; using Squidex.Infrastructure; +using Squidex.Infrastructure.Caching; using Squidex.Infrastructure.EventSourcing; namespace Squidex.Domain.Apps.Entities.Rules @@ -24,6 +25,7 @@ namespace Squidex.Domain.Apps.Entities.Rules private readonly IRuleEventRepository ruleEventRepository; private readonly IAppProvider appProvider; private readonly IMemoryCache cache; + private readonly ILocalCache localCache; private readonly RuleService ruleService; public string Name @@ -36,18 +38,19 @@ namespace Squidex.Domain.Apps.Entities.Rules get { return ".*"; } } - public RuleEnqueuer(IAppProvider appProvider, IMemoryCache cache, IRuleEventRepository ruleEventRepository, + public RuleEnqueuer(IAppProvider appProvider, IMemoryCache cache, ILocalCache localCache, IRuleEventRepository ruleEventRepository, RuleService ruleService) { Guard.NotNull(appProvider, nameof(appProvider)); Guard.NotNull(cache, nameof(cache)); + Guard.NotNull(localCache, nameof(localCache)); Guard.NotNull(ruleEventRepository, nameof(ruleEventRepository)); Guard.NotNull(ruleService, nameof(ruleService)); this.appProvider = appProvider; this.cache = cache; - + this.localCache = localCache; this.ruleEventRepository = ruleEventRepository; this.ruleService = ruleService; } @@ -67,11 +70,14 @@ namespace Squidex.Domain.Apps.Entities.Rules Guard.NotNull(rule, nameof(rule)); Guard.NotNull(@event, nameof(@event)); - var jobs = await ruleService.CreateJobsAsync(rule, ruleId, @event); - - foreach (var job in jobs) + using (localCache.StartContext()) { - await ruleEventRepository.EnqueueAsync(job, job.Created); + var jobs = await ruleService.CreateJobsAsync(rule, ruleId, @event); + + foreach (var job in jobs) + { + await ruleEventRepository.EnqueueAsync(job, job.Created); + } } } diff --git a/backend/src/Squidex/Config/Domain/RuleServices.cs b/backend/src/Squidex/Config/Domain/RuleServices.cs index 7b4bab01a..250bc5600 100644 --- a/backend/src/Squidex/Config/Domain/RuleServices.cs +++ b/backend/src/Squidex/Config/Domain/RuleServices.cs @@ -44,7 +44,7 @@ namespace Squidex.Config.Domain .As(); services.AddSingletonAs() - .As(); + .As().As(); services.AddSingletonAs() .As(); @@ -73,6 +73,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); + services.AddSingletonAs() + .As(); + services.AddSingletonAs() .AsSelf(); diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs index e941a6c27..cb469f17f 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleEventFormatterTests.cs @@ -7,7 +7,9 @@ using System; using System.Collections.Generic; +using System.Runtime.InteropServices; using System.Security.Claims; +using System.Threading.Tasks; using FakeItEasy; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; @@ -38,6 +40,26 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules private readonly Guid assetId = Guid.NewGuid(); private readonly RuleEventFormatter sut; + private class FakeContentResolver : IRuleEventFormatter + { + public (bool Match, ValueTask) Format(EnrichedEvent @event, object value, string[] path) + { + if (path[0] == "data" && value is JsonArray _) + { + return (true, GetValueAsync()); + } + + return default; + } + + private async ValueTask GetValueAsync() + { + await Task.Delay(5); + + return "Reference"; + } + } + public RuleEventFormatterTests() { A.CallTo(() => urlGenerator.ContentUI(appId, schemaId, contentId)) @@ -64,7 +86,13 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules var cache = new MemoryCache(Options.Create(new MemoryCacheOptions())); - sut = new RuleEventFormatter(TestUtils.DefaultSerializer, urlGenerator, new JintScriptEngine(cache, extensions)); + var formatters = new IRuleEventFormatter[] + { + new PredefinedPatternsFormatter(urlGenerator), + new FakeContentResolver() + }; + + sut = new RuleEventFormatter(TestUtils.DefaultSerializer, formatters, new JintScriptEngine(cache, extensions)); } [Fact] @@ -99,11 +127,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("Name $APP_NAME has id $APP_ID")] [InlineData("Name ${EVENT_APPID.NAME} has id ${EVENT_APPID.ID}")] [InlineData("Script(`Name ${event.appId.name} has id ${event.appId.id}`)")] - public void Should_format_app_information_from_event(string script) + public async Task Should_format_app_information_from_event(string script) { var @event = new EnrichedContentEvent { AppId = appId }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal($"Name my-app has id {appId.Id}", result); } @@ -111,11 +139,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Name $SCHEMA_NAME has id $SCHEMA_ID")] [InlineData("Script(`Name ${event.schemaId.name} has id ${event.schemaId.id}`)")] - public void Should_format_schema_information_from_event(string script) + public async Task Should_format_schema_information_from_event(string script) { var @event = new EnrichedContentEvent { SchemaId = schemaId }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal($"Name my-schema has id {schemaId.Id}", result); } @@ -123,11 +151,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Full: $TIMESTAMP_DATETIME")] [InlineData("Script(`Full: ${formatDate(event.timestamp, 'yyyy-MM-dd-hh-mm-ss')}`)")] - public void Should_format_timestamp_information_from_event(string script) + public async Task Should_format_timestamp_information_from_event(string script) { var @event = new EnrichedContentEvent { Timestamp = now }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal($"Full: {now:yyyy-MM-dd-hh-mm-ss}", result); } @@ -135,11 +163,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Date: $TIMESTAMP_DATE")] [InlineData("Script(`Date: ${formatDate(event.timestamp, 'yyyy-MM-dd')}`)")] - public void Should_format_timestamp_date_information_from_event(string script) + public async Task Should_format_timestamp_date_information_from_event(string script) { var @event = new EnrichedContentEvent { Timestamp = now }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal($"Date: {now:yyyy-MM-dd}", result); } @@ -148,11 +176,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("From $MENTIONED_NAME ($MENTIONED_EMAIL, $MENTIONED_ID)")] [InlineData("From ${COMMENT_MENTIONEDUSER.NAME} (${COMMENT_MENTIONEDUSER.EMAIL}, ${COMMENT_MENTIONEDUSER.ID})")] [InlineData("Script(`From ${event.mentionedUser.name} (${event.mentionedUser.email}, ${event.mentionedUser.id})`)")] - public void Should_format_email_and_display_name_from_mentioned_user(string script) + public async Task Should_format_email_and_display_name_from_mentioned_user(string script) { var @event = new EnrichedCommentEvent { MentionedUser = user }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("From me (me@email.com, user123)", result); } @@ -160,11 +188,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("From $USER_NAME ($USER_EMAIL, $USER_ID)")] [InlineData("Script(`From ${event.user.name} (${event.user.email}, ${event.user.id})`)")] - public void Should_format_email_and_display_name_from_user(string script) + public async Task Should_format_email_and_display_name_from_user(string script) { var @event = new EnrichedContentEvent { User = user }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("From me (me@email.com, user123)", result); } @@ -172,11 +200,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("From $USER_NAME ($USER_EMAIL, $USER_ID)")] [InlineData("Script(`From ${event.user.name} (${event.user.email}, ${event.user.id})`)")] - public void Should_return_null_if_user_is_not_found(string script) + public async Task Should_return_null_if_user_is_not_found(string script) { var @event = new EnrichedContentEvent(); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("From null (null, null)", result); } @@ -184,11 +212,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("From $USER_NAME ($USER_EMAIL, $USER_ID)")] [InlineData("Script(`From ${event.user.name} (${event.user.email}, ${event.user.id})`)")] - public void Should_format_email_and_display_name_from_client(string script) + public async Task Should_format_email_and_display_name_from_client(string script) { var @event = new EnrichedContentEvent { User = new ClientUser(new RefToken(RefTokenType.Client, "android")) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("From client:android (client:android, android)", result); } @@ -196,11 +224,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Version: $ASSET_VERSION")] [InlineData("Script(`Version: ${event.version}`)")] - public void Should_format_base_property(string script) + public async Task Should_format_base_property(string script) { var @event = new EnrichedAssetEvent { Version = 13 }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Version: 13", result); } @@ -208,11 +236,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("File: $ASSET_FILENAME")] [InlineData("Script(`File: ${event.fileName}`)")] - public void Should_format_asset_file_name_from_event(string script) + public async Task Should_format_asset_file_name_from_event(string script) { var @event = new EnrichedAssetEvent { FileName = "my-file.png" }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("File: my-file.png", result); } @@ -220,11 +248,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Type: $ASSET_ASSETTYPE")] [InlineData("Script(`Type: ${event.assetType}`)")] - public void Should_format_asset_asset_type_from_event(string script) + public async Task Should_format_asset_asset_type_from_event(string script) { var @event = new EnrichedAssetEvent { AssetType = AssetType.Audio }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Type: Audio", result); } @@ -232,11 +260,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Download at $ASSET_CONTENT_URL")] [InlineData("Script(`Download at ${assetContentUrl()}`)")] - public void Should_format_asset_content_url_from_event(string script) + public async Task Should_format_asset_content_url_from_event(string script) { var @event = new EnrichedAssetEvent { Id = assetId }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Download at asset-content-url", result); } @@ -244,11 +272,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Download at $ASSET_CONTENT_URL")] [InlineData("Script(`Download at ${assetContentUrl()}`)")] - public void Should_return_null_when_asset_content_url_not_found(string script) + public async Task Should_return_null_when_asset_content_url_not_found(string script) { var @event = new EnrichedContentEvent(); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Download at null", result); } @@ -256,11 +284,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Go to $CONTENT_URL")] [InlineData("Script(`Go to ${contentUrl()}`)")] - public void Should_format_content_url_from_event(string script) + public async Task Should_format_content_url_from_event(string script) { var @event = new EnrichedContentEvent { AppId = appId, Id = contentId, SchemaId = schemaId }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Go to content-url", result); } @@ -268,11 +296,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("Go to $CONTENT_URL")] [InlineData("Script(`Go to ${contentUrl()}`)")] - public void Should_return_null_when_content_url_when_not_found(string script) + public async Task Should_return_null_when_content_url_when_not_found(string script) { var @event = new EnrichedAssetEvent(); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Go to null", result); } @@ -281,11 +309,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("$CONTENT_STATUS")] [InlineData("Script(contentAction())")] [InlineData("Script(`${event.status}`)")] - public void Should_format_content_status_when_found(string script) + public async Task Should_format_content_status_when_found(string script) { var @event = new EnrichedContentEvent { Status = Status.Published }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Published", result); } @@ -293,11 +321,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_ACTION")] [InlineData("Script(contentAction())")] - public void Should_return_null_when_content_status_not_found(string script) + public async Task Should_return_null_when_content_status_not_found(string script) { var @event = new EnrichedAssetEvent(); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -305,11 +333,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_ACTION")] [InlineData("Script(`${event.type}`)")] - public void Should_format_content_actions_when_found(string script) + public async Task Should_format_content_actions_when_found(string script) { var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Created", result); } @@ -317,11 +345,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_ACTION")] [InlineData("Script(contentAction())")] - public void Should_return_null_when_content_action_not_found(string script) + public async Task Should_return_null_when_content_action_not_found(string script) { var @event = new EnrichedAssetEvent(); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -329,7 +357,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.country.iv")] [InlineData("Script(`${event.data.country.iv}`)")] - public void Should_return_null_when_field_not_found(string script) + public async Task Should_return_null_when_field_not_found(string script) { var @event = new EnrichedContentEvent { @@ -340,7 +368,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddValue("iv", "Berlin")) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -348,7 +376,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.de")] [InlineData("Script(`${event.data.country.iv}`)")] - public void Should_return_null_when_partition_not_found(string script) + public async Task Should_return_null_when_partition_not_found(string script) { var @event = new EnrichedContentEvent { @@ -359,7 +387,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddValue("iv", "Berlin")) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -367,7 +395,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.iv.10")] [InlineData("Script(`${event.data.country.de[10]}`)")] - public void Should_return_null_when_array_item_not_found(string script) + public async Task Should_return_null_when_array_item_not_found(string script) { var @event = new EnrichedContentEvent { @@ -378,7 +406,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddJsonValue(JsonValue.Array())) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -386,7 +414,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.de.Name")] [InlineData("Script(`${event.data.city.de.Location}`)")] - public void Should_return_null_when_property_not_found(string script) + public async Task Should_return_null_when_property_not_found(string script) { var @event = new EnrichedContentEvent { @@ -397,7 +425,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddJsonValue(JsonValue.Object().Add("name", "Berlin"))) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("null", result); } @@ -405,7 +433,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.iv")] [InlineData("Script(`${event.data.city.iv}`)")] - public void Should_return_plain_value_when_found(string script) + public async Task Should_return_plain_value_when_found(string script) { var @event = new EnrichedContentEvent { @@ -416,7 +444,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddValue("iv", "Berlin")) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Berlin", result); } @@ -424,7 +452,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.iv.0")] [InlineData("Script(`${event.data.city.iv[0]}`)")] - public void Should_return_plain_value_from_array_when_found(string script) + public async Task Should_return_plain_value_from_array_when_found(string script) { var @event = new EnrichedContentEvent { @@ -435,7 +463,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddJsonValue(JsonValue.Array("Berlin"))) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Berlin", result); } @@ -443,7 +471,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.iv.name")] [InlineData("Script(`${event.data.city.iv.name}`)")] - public void Should_return_plain_value_from_object_when_found(string script) + public async Task Should_return_plain_value_from_object_when_found(string script) { var @event = new EnrichedContentEvent { @@ -454,7 +482,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddJsonValue(JsonValue.Object().Add("name", "Berlin"))) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("Berlin", result); } @@ -462,7 +490,7 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("$CONTENT_DATA.city.iv")] [InlineData("Script(`${JSON.stringify(event.data.city.iv)}`)")] - public void Should_return_json_string_when_object(string script) + public async Task Should_return_json_string_when_object(string script) { var @event = new EnrichedContentEvent { @@ -473,40 +501,57 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules .AddJsonValue(JsonValue.Object().Add("name", "Berlin"))) }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("{\"name\":\"Berlin\"}", result); } + [Fact] + public async Task Should_resolve_reference() + { + var @event = new EnrichedContentEvent + { + Data = + new NamedContentData() + .AddField("city", + new ContentFieldData() + .AddJsonValue(JsonValue.Array())) + }; + + var result = await sut.FormatAsync("${CONTENT_DATA.city.iv.data.name}", @event); + + Assert.Equal("Reference", result); + } + [Theory] [InlineData("Script(`From ${event.actor}`)")] - public void Should_format_actor(string script) + public async Task Should_format_actor(string script) { var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "android") }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal("From client:android", result); } [Theory] [InlineData("${EVENT_INVALID ? file}", "file")] - public void Should_provide_fallback_if_path_is_invalid(string script, string expect) + public async Task Should_provide_fallback_if_path_is_invalid(string script, string expect) { var @event = new EnrichedAssetEvent { FileName = null! }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } [Theory] [InlineData("${ASSET_FILENAME ? file}", "file")] - public void Should_provide_fallback_if_value_is_null(string script, string expect) + public async Task Should_provide_fallback_if_value_is_null(string script, string expect) { var @event = new EnrichedAssetEvent { FileName = null! }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } @@ -515,11 +560,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("Found in ${ASSET_FILENAME | Upper}.docx", "Found in DONALD DUCK.docx")] [InlineData("Found in ${ASSET_FILENAME| Upper }.docx", "Found in DONALD DUCK.docx")] [InlineData("Found in ${ASSET_FILENAME|Upper }.docx", "Found in DONALD DUCK.docx")] - public void Should_transform_replacements_and_igore_whitepsaces(string script, string expect) + public async Task Should_transform_replacements_and_igore_whitepsaces(string script, string expect) { var @event = new EnrichedAssetEvent { FileName = "Donald Duck" }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } @@ -531,11 +576,11 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("Found in ${ASSET_FILENAME | Lower}.docx", "Found in donald duck.docx", "Donald Duck")] [InlineData("Found in ${ASSET_FILENAME | Slugify}.docx", "Found in donald-duck.docx", "Donald Duck")] [InlineData("Found in ${ASSET_FILENAME | Trim}.docx", "Found in Donald Duck.docx", "Donald Duck ")] - public void Should_transform_replacements(string script, string expect, string name) + public async Task Should_transform_replacements(string script, string expect, string name) { var @event = new EnrichedAssetEvent { FileName = name }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } @@ -547,14 +592,14 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [InlineData("From ${USER_NAME | Lower}", "From donald duck", "Donald Duck")] [InlineData("From ${USER_NAME | Slugify}", "From donald-duck", "Donald Duck")] [InlineData("From ${USER_NAME | Trim}", "From Donald Duck", "Donald Duck ")] - public void Should_transform_replacements_with_simple_pattern(string script, string expect, string name) + public async Task Should_transform_replacements_with_simple_pattern(string script, string expect, string name) { var @event = new EnrichedContentEvent { User = user }; A.CallTo(() => user.Claims) .Returns(new List { new Claim(SquidexClaimTypes.DisplayName, name) }); - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } @@ -562,51 +607,63 @@ namespace Squidex.Domain.Apps.Core.Operations.HandleRules [Theory] [InlineData("{'Key':'${ASSET_FILENAME | Upper}'}", "{'Key':'DONALD DUCK'}")] [InlineData("{'Key':'${ASSET_FILENAME}'}", "{'Key':'Donald Duck'}")] - public void Should_transform_json_examples(string script, string expect) + public async Task Should_transform_json_examples(string script, string expect) { var @event = new EnrichedAssetEvent { FileName = "Donald Duck" }; - var result = sut.Format(script, @event); + var result = await sut.FormatAsync(script, @event); + + Assert.Equal(expect, result); + } + + [Theory] + [InlineData("${ASSET_LASTMODIFIED | timestamp_seconds}", "1590769584")] + [InlineData("${ASSET_LASTMODIFIED | timestamp_ms}", "1590769584000")] + public async Task Should_transform_timestamp(string script, string expect) + { + var @event = new EnrichedAssetEvent { LastModified = Instant.FromUnixTimeSeconds(1590769584) }; + + var result = await sut.FormatAsync(script, @event); Assert.Equal(expect, result); } [Fact] - public void Should_format_json() + public async Task Should_format_json() { var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "android") }; - var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); + var result = await sut.FormatAsync("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); Assert.Equal("{\"actor\":\"client:android\"}", result); } [Fact] - public void Should_format_json_with_special_characters() + public async Task Should_format_json_with_special_characters() { var @event = new EnrichedContentEvent { Actor = new RefToken(RefTokenType.Client, "mobile\"android") }; - var result = sut.Format("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); + var result = await sut.FormatAsync("Script(JSON.stringify({ actor: event.actor.toString() }))", @event); Assert.Equal("{\"actor\":\"client:mobile\\\"android\"}", result); } [Fact] - public void Should_evaluate_script_if_starting_with_whitespace() + public async Task Should_evaluate_script_if_starting_with_whitespace() { var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; - var result = sut.Format(" Script(`${event.type}`)", @event); + var result = await sut.FormatAsync(" Script(`${event.type}`)", @event); Assert.Equal("Created", result); } [Fact] - public void Should_evaluate_script_if_ends_with_whitespace() + public async Task Should_evaluate_script_if_ends_with_whitespace() { var @event = new EnrichedContentEvent { Type = EnrichedContentEventType.Created }; - var result = sut.Format("Script(`${event.type}`) ", @event); + var result = await sut.FormatAsync("Script(`${event.type}`) ", @event); Assert.Equal("Created", result); } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentChangedTriggerHandlerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentChangedTriggerHandlerTests.cs index 29a41aaa0..376aa73f7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentChangedTriggerHandlerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentChangedTriggerHandlerTests.cs @@ -20,7 +20,9 @@ using Squidex.Domain.Apps.Events; using Squidex.Domain.Apps.Events.Assets; using Squidex.Domain.Apps.Events.Contents; using Squidex.Infrastructure; +using Squidex.Infrastructure.Caching; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.Json.Objects; using Xunit; #pragma warning disable SA1401 // Fields must be private @@ -31,8 +33,10 @@ namespace Squidex.Domain.Apps.Entities.Contents public class ContentChangedTriggerHandlerTests { private readonly IScriptEngine scriptEngine = A.Fake(); + private readonly ILocalCache localCache = new AsyncLocalCache(); private readonly IContentLoader contentLoader = A.Fake(); - private readonly IRuleTriggerHandler sut; + private readonly ContentChangedTriggerHandler sut; + private readonly IRuleTriggerHandler handler; private readonly Guid ruleId = Guid.NewGuid(); private static readonly NamedId SchemaMatch = NamedId.Of(Guid.NewGuid(), "my-schema1"); private static readonly NamedId SchemaNonMatch = NamedId.Of(Guid.NewGuid(), "my-schema2"); @@ -45,7 +49,9 @@ namespace Squidex.Domain.Apps.Entities.Contents A.CallTo(() => scriptEngine.Evaluate(A._, "false")) .Returns(false); - sut = new ContentChangedTriggerHandler(scriptEngine, contentLoader); + sut = new ContentChangedTriggerHandler(scriptEngine, contentLoader, localCache); + + handler = sut; } public static IEnumerable TestEvents() @@ -58,6 +64,58 @@ namespace Squidex.Domain.Apps.Entities.Contents yield return new object[] { new ContentStatusChanged { Change = StatusChange.Unpublished }, EnrichedContentEventType.Unpublished }; } + [Fact] + public async Task Should_resolve_reference_if_value_from_content_loader() + { + var referenceId = Guid.NewGuid(); + var referenceValue = JsonValue.Array(referenceId); + + SetupReference(referenceId); + + var (handled, result) = sut.Format(null!, referenceValue, new[] { "data", "field1", "iv" }); + + Assert.True(handled); + Assert.Equal("Hello", await result); + } + + [Fact] + public async Task Should_resolve_reference_only_once() + { + using (localCache.StartContext()) + { + var referenceId = Guid.NewGuid(); + var referenceValue = JsonValue.Array(referenceId); + + SetupReference(referenceId); + + var (handled1, result1) = sut.Format(null!, referenceValue, new[] { "data", "field1", "iv" }); + var (handled2, result2) = sut.Format(null!, referenceValue, new[] { "data", "field2", "iv" }); + + Assert.True(handled1); + Assert.Equal("Hello", await result1); + + Assert.True(handled2); + Assert.Equal("World", await result2); + + A.CallTo(() => contentLoader.GetAsync(A._, A._)) + .MustHaveHappenedOnceExactly(); + } + } + + [Fact] + public async Task Should_not_return_value_if_path_not_found_in_reference() + { + var referenceId = Guid.NewGuid(); + var referenceValue = JsonValue.Array(referenceId); + + SetupReference(referenceId); + + var (handled, result) = sut.Format(null!, referenceValue, new[] { "data", "invalid", "iv" }); + + Assert.True(handled); + Assert.Null(await result); + } + [Theory] [MemberData(nameof(TestEvents))] public async Task Should_create_enriched_events(ContentEvent @event, EnrichedContentEventType type) @@ -67,7 +125,7 @@ namespace Squidex.Domain.Apps.Entities.Contents A.CallTo(() => contentLoader.GetAsync(@event.ContentId, 12)) .Returns(new ContentEntity { SchemaId = SchemaMatch }); - var result = await sut.CreateEnrichedEventsAsync(envelope); + var result = await handler.CreateEnrichedEventsAsync(envelope); var enrichedEvent = result.Single() as EnrichedContentEvent; @@ -90,7 +148,7 @@ namespace Squidex.Domain.Apps.Entities.Contents A.CallTo(() => contentLoader.GetAsync(@event.ContentId, 11)) .Returns(new ContentEntity { SchemaId = SchemaMatch, Version = 11, Data = dataOld }); - var result = await sut.CreateEnrichedEventsAsync(envelope); + var result = await handler.CreateEnrichedEventsAsync(envelope); var enrichedEvent = result.Single() as EnrichedContentEvent; @@ -103,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: true, schemaId: null, condition: null, action: trigger => { - var result = sut.Trigger(new AssetCreated(), trigger, ruleId); + var result = handler.Trigger(new AssetCreated(), trigger, ruleId); Assert.False(result); }); @@ -114,7 +172,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: null, condition: null, action: trigger => { - var result = sut.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); + var result = handler.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); Assert.False(result); }); @@ -125,7 +183,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: true, schemaId: SchemaMatch, condition: null, action: trigger => { - var result = sut.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); + var result = handler.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); Assert.True(result); }); @@ -136,7 +194,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaMatch, condition: string.Empty, action: trigger => { - var result = sut.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); + var result = handler.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); Assert.True(result); }); @@ -147,7 +205,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaNonMatch, condition: null, action: trigger => { - var result = sut.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); + var result = handler.Trigger(new ContentCreated { SchemaId = SchemaMatch }, trigger, ruleId); Assert.False(result); }); @@ -158,7 +216,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: true, schemaId: null, condition: null, action: trigger => { - var result = sut.Trigger(new EnrichedAssetEvent(), trigger); + var result = handler.Trigger(new EnrichedAssetEvent(), trigger); Assert.False(result); }); @@ -169,7 +227,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: null, condition: null, action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.False(result); }); @@ -180,7 +238,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: true, schemaId: SchemaMatch, condition: null, action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.True(result); }); @@ -191,7 +249,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaMatch, condition: string.Empty, action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.True(result); }); @@ -202,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaMatch, condition: "true", action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.True(result); }); @@ -213,7 +271,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaNonMatch, condition: null, action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.False(result); }); @@ -224,7 +282,7 @@ namespace Squidex.Domain.Apps.Entities.Contents { TestForTrigger(handleAll: false, schemaId: SchemaMatch, condition: "false", action: trigger => { - var result = sut.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); + var result = handler.Trigger(new EnrichedContentEvent { SchemaId = SchemaMatch }, trigger); Assert.False(result); }); @@ -258,5 +316,21 @@ namespace Squidex.Domain.Apps.Entities.Contents .MustHaveHappened(); } } + + private void SetupReference(Guid referenceId) + { + A.CallTo(() => contentLoader.GetAsync(referenceId, EtagVersion.Any)) + .Returns(new ContentEntity + { + Data = + new NamedContentData() + .AddField("field1", + new ContentFieldData() + .AddJsonValue(JsonValue.Create("Hello"))) + .AddField("field2", + new ContentFieldData() + .AddJsonValue(JsonValue.Create("World"))) + }); + } } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs index c008c9650..36f1195c3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs @@ -18,6 +18,7 @@ using Squidex.Domain.Apps.Core.Rules.Triggers; using Squidex.Domain.Apps.Entities.Rules.Repositories; using Squidex.Domain.Apps.Events.Contents; using Squidex.Infrastructure; +using Squidex.Infrastructure.Caching; using Squidex.Infrastructure.EventSourcing; using Xunit; @@ -27,6 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Rules { private readonly IAppProvider appProvider = A.Fake(); private readonly IMemoryCache cache = new MemoryCache(Options.Create(new MemoryCacheOptions())); + private readonly ILocalCache localCache = A.Fake(); private readonly IRuleEventRepository ruleEventRepository = A.Fake(); private readonly Instant now = SystemClock.Instance.GetCurrentInstant(); private readonly NamedId appId = NamedId.Of(Guid.NewGuid(), "my-app"); @@ -43,6 +45,7 @@ namespace Squidex.Domain.Apps.Entities.Rules sut = new RuleEnqueuer( appProvider, cache, + localCache, ruleEventRepository, ruleService); } @@ -81,6 +84,9 @@ namespace Squidex.Domain.Apps.Entities.Rules A.CallTo(() => ruleEventRepository.EnqueueAsync(job, now, default)) .MustHaveHappened(); + + A.CallTo(() => localCache.StartContext()) + .MustHaveHappened(); } [Fact]