From fb3a9d87fb21b7e0a0f347a37f89237190df6112 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Wed, 7 Dec 2022 17:03:27 +0100 Subject: [PATCH] Eliminate stop words before search. (#949) * Eliminate stop words before search. * Improve tokenization. * Fix tests * Update fluid * Cleanup. --- .../Extensions/EventFluidExtensions.cs | 18 +-- ...Squidex.Domain.Apps.Core.Operations.csproj | 2 +- .../Templates/CustomFluidParser.cs | 19 +++ .../Extensions/ContentFluidExtension.cs | 74 ++++----- .../Extensions/DateTimeFluidExtension.cs | 37 +++-- .../Extensions/JsonArrayFluidValue.cs | 14 +- .../Extensions/StringFluidExtension.cs | 30 ++-- .../Extensions/StringWordsFluidExtension.cs | 10 +- .../Extensions/UserFluidExtension.cs | 17 ++- .../Templates/FluidTemplateEngine.cs | 76 ++++----- .../Templates/IFluidExtension.cs | 6 +- .../Templates/TemplateParseException.cs | 31 +--- .../Text/MongoTextIndex.cs | 5 +- .../Text/MongoTextIndexBase.cs | 60 +++++--- .../Text/Tokenizer.cs | 144 ++++++++++++++++++ .../Assets/AssetsFluidExtension.cs | 86 ++++------- .../Contents/ReferencesFluidExtension.cs | 77 ++++------ .../Assets/AssetsFluidExtensionTests.cs | 29 ++-- .../Contents/MongoDb/TokenizerTests.cs | 63 ++++++++ .../Queries/ContentQueryServiceTests.cs | 2 +- .../Contents/ReferencesFluidExtensionTests.cs | 63 +++----- 21 files changed, 513 insertions(+), 350 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/Templates/CustomFluidParser.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventFluidExtensions.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventFluidExtensions.cs index a9fa812dd..421b7e95b 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventFluidExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventFluidExtensions.cs @@ -23,15 +23,15 @@ public sealed class EventFluidExtensions : IFluidExtension this.urlGenerator = urlGenerator; } - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - TemplateContext.GlobalFilters.AddFilter("contentUrl", ContentUrl); - TemplateContext.GlobalFilters.AddFilter("assetContentUrl", AssetContentUrl); - TemplateContext.GlobalFilters.AddFilter("assetContentAppUrl", AssetContentAppUrl); - TemplateContext.GlobalFilters.AddFilter("assetContentSlugUrl", AssetContentSlugUrl); + options.Filters.AddFilter("contentUrl", ContentUrl); + options.Filters.AddFilter("assetContentUrl", AssetContentUrl); + options.Filters.AddFilter("assetContentAppUrl", AssetContentAppUrl); + options.Filters.AddFilter("assetContentSlugUrl", AssetContentSlugUrl); } - private FluidValue ContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context) + private ValueTask ContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context) { var value = input.ToObjectValue(); @@ -60,7 +60,7 @@ public sealed class EventFluidExtensions : IFluidExtension return NilValue.Empty; } - private FluidValue AssetContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context) + private ValueTask AssetContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context) { var value = input.ToObjectValue(); @@ -89,7 +89,7 @@ public sealed class EventFluidExtensions : IFluidExtension return NilValue.Empty; } - private FluidValue AssetContentAppUrl(FluidValue input, FilterArguments arguments, TemplateContext context) + private ValueTask AssetContentAppUrl(FluidValue input, FilterArguments arguments, TemplateContext context) { var value = input.ToObjectValue(); @@ -118,7 +118,7 @@ public sealed class EventFluidExtensions : IFluidExtension return NilValue.Empty; } - private FluidValue AssetContentSlugUrl(FluidValue input, FilterArguments arguments, TemplateContext context) + private ValueTask AssetContentSlugUrl(FluidValue input, FilterArguments arguments, TemplateContext context) { var value = input.ToObjectValue(); diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj b/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj index b95ea946c..b8a7b5d1e 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj @@ -18,7 +18,7 @@ - + diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/CustomFluidParser.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/CustomFluidParser.cs new file mode 100644 index 000000000..e21d86069 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/CustomFluidParser.cs @@ -0,0 +1,19 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Fluid; +using Fluid.Ast; +using Parlot.Fluent; + +namespace Squidex.Domain.Apps.Core.Templates; + +public sealed class CustomFluidParser : FluidParser +{ + public Deferred PrimaryParser => Primary; + + public Parser CommaParser => Comma; +} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/ContentFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/ContentFluidExtension.cs index 6985da240..99afa3d22 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/ContentFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/ContentFluidExtension.cs @@ -6,6 +6,7 @@ // ========================================================================== using Fluid; +using Fluid.Accessors; using Fluid.Values; using Squidex.Domain.Apps.Core.Contents; using Squidex.Infrastructure; @@ -15,52 +16,57 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions; public sealed class ContentFluidExtension : IFluidExtension { - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - FluidValue.SetTypeMapping(x => new ObjectValue(x)); - FluidValue.SetTypeMapping(x => new ObjectValue(x)); - FluidValue.SetTypeMapping(x => new ObjectValue(x)); - FluidValue.SetTypeMapping(x => new JsonArrayFluidValue(x)); - - FluidValue.SetTypeMapping(source => + options.ValueConverters.Add(source => { - switch (source.Value) + switch (source) { - case null: - return FluidValue.Create(null); - case bool b: - return FluidValue.Create(b); - case double n: - return FluidValue.Create(n); - case string s: - return FluidValue.Create(s); + case ContentData d: + return new ObjectValue(d); + case ContentFieldData f: + return new ObjectValue(f); + case JsonArray a: + return new JsonArrayFluidValue(a, options); case JsonObject o: return new ObjectValue(o); - case JsonArray a: - return new JsonArrayFluidValue(a); - } - - ThrowHelper.InvalidOperationException(); - return default!; - }); + case JsonValue v: + switch (v.Value) + { + case null: + return NilValue.Instance; + case bool b: + return BooleanValue.Create(b); + case double n: + return NumberValue.Create((decimal)n); + case string s: + return StringValue.Create(s); + case JsonArray a: + return new JsonArrayFluidValue(a, options); + case JsonObject o: + return new ObjectValue(o); + } - memberAccessStrategy.Register((value, name) => - { - if (value.Value is JsonObject o) - { - return o.GetValueOrDefault(name); + ThrowHelper.InvalidOperationException(); + break; } return null; }); - memberAccessStrategy.Register( - (value, name) => value.GetValueOrDefault(name)); + options.MemberAccessStrategy.Register("*", new DelegateAccessor((source, name, context) => + { + return source.GetValueOrDefault(name); + })); - memberAccessStrategy.Register( - (value, name) => value.GetValueOrDefault(name).Value); + options.MemberAccessStrategy.Register("*", new DelegateAccessor((source, name, context) => + { + return source.GetValueOrDefault(name); + })); - memberAccessStrategy.Register( - (value, name) => value.GetValueOrDefault(name).Value); + options.MemberAccessStrategy.Register("*", new DelegateAccessor((source, name, context) => + { + return source.GetValueOrDefault(name); + })); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/DateTimeFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/DateTimeFluidExtension.cs index 6c04dd29f..cc7a3769a 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/DateTimeFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/DateTimeFluidExtension.cs @@ -15,35 +15,34 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions; public class DateTimeFluidExtension : IFluidExtension { - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + private static readonly FilterDelegate FormatDate = (input, arguments, context) => { - TemplateContext.GlobalFilters.AddFilter("format_date", FormatDate); + if (arguments.Count == 1) + { + return FormatDateCore(input, x => Format(arguments, x)); + } - TemplateContext.GlobalFilters.AddFilter("timestamp", FormatTimestamp); - TemplateContext.GlobalFilters.AddFilter("timestamp_sec", FormatTimestampSec); - } + return input; + }; - public static FluidValue FormatTimestamp(FluidValue input, FilterArguments arguments, TemplateContext context) + private static readonly FilterDelegate FormatTimestamp = (input, arguments, context) => { - return FormatDate(input, x => FluidValue.Create(x.ToUnixTimeMilliseconds())); - } + return FormatDateCore(input, x => NumberValue.Create(x.ToUnixTimeMilliseconds())); + }; - public static FluidValue FormatTimestampSec(FluidValue input, FilterArguments arguments, TemplateContext context) + private static readonly FilterDelegate FormatTimestampSec = (input, arguments, context) => { - return FormatDate(input, x => FluidValue.Create(x.ToUnixTimeMilliseconds() / 1000)); - } + return FormatDateCore(input, x => NumberValue.Create(x.ToUnixTimeMilliseconds() / 1000)); + }; - public static FluidValue FormatDate(FluidValue input, FilterArguments arguments, TemplateContext context) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - if (arguments.Count == 1) - { - return FormatDate(input, x => Format(arguments, x)); - } - - return input; + options.Filters.AddFilter("format_date", FormatDate); + options.Filters.AddFilter("timestamp", FormatTimestamp); + options.Filters.AddFilter("timestamp_sec", FormatTimestampSec); } - private static FluidValue FormatDate(FluidValue input, Func formatter) + private static ValueTask FormatDateCore(FluidValue input, Func formatter) { switch (input) { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/JsonArrayFluidValue.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/JsonArrayFluidValue.cs index d58b3dabe..eab565608 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/JsonArrayFluidValue.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/JsonArrayFluidValue.cs @@ -16,12 +16,14 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions; public sealed class JsonArrayFluidValue : FluidValue { private readonly JsonArray value; + private readonly TemplateOptions options; public override FluidValues Type { get; } = FluidValues.Array; - public JsonArrayFluidValue(JsonArray value) + public JsonArrayFluidValue(JsonArray value, TemplateOptions options) { this.value = value; + this.options = options; } public override bool Equals(FluidValue other) @@ -59,7 +61,7 @@ public sealed class JsonArrayFluidValue : FluidValue case "first": if (value.Count > 0) { - return Create(value[0]); + return Create(value[0], options); } break; @@ -67,7 +69,7 @@ public sealed class JsonArrayFluidValue : FluidValue case "last": if (value.Count > 0) { - return Create(value[^1]); + return Create(value[^1], options); } break; @@ -82,17 +84,17 @@ public sealed class JsonArrayFluidValue : FluidValue if (i >= 0 && i < value.Count) { - return Create(value[i]); + return Create(value[i], options); } return NilValue.Instance; } - public override IEnumerable Enumerate() + public override IEnumerable Enumerate(TemplateContext context) { foreach (var item in value) { - yield return Create(item); + yield return Create(item, options); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs index 3193aab0b..612857638 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs @@ -20,7 +20,7 @@ public sealed class StringFluidExtension : IFluidExtension { var result = value.ToStringValue().Slugify(); - return FluidValue.Create(result); + return StringValue.Create(result); } return input; @@ -28,42 +28,42 @@ public sealed class StringFluidExtension : IFluidExtension private static readonly FilterDelegate Escape = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().JsonEscape()); + return StringValue.Create(input.ToStringValue().JsonEscape()); }; private static readonly FilterDelegate Markdown2Text = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().Markdown2Text()); + return StringValue.Create(input.ToStringValue().Markdown2Text()); }; private static readonly FilterDelegate Html2Text = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().Html2Text()); + return StringValue.Create(input.ToStringValue().Html2Text()); }; private static readonly FilterDelegate Trim = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().Trim()); + return StringValue.Create(input.ToStringValue().Trim()); }; private static readonly FilterDelegate MD5 = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().ToMD5()); + return StringValue.Create(input.ToStringValue().ToMD5()); }; private static readonly FilterDelegate Sha256 = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().ToSha256()); + return StringValue.Create(input.ToStringValue().ToSha256()); }; - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - TemplateContext.GlobalFilters.AddFilter("escape", Escape); - TemplateContext.GlobalFilters.AddFilter("html2text", Html2Text); - TemplateContext.GlobalFilters.AddFilter("markdown2text", Markdown2Text); - TemplateContext.GlobalFilters.AddFilter("md5", MD5); - TemplateContext.GlobalFilters.AddFilter("sha256", Sha256); - TemplateContext.GlobalFilters.AddFilter("slugify", Slugify); - TemplateContext.GlobalFilters.AddFilter("trim", Trim); + options.Filters.AddFilter("escape", Escape); + options.Filters.AddFilter("html2text", Html2Text); + options.Filters.AddFilter("markdown2text", Markdown2Text); + options.Filters.AddFilter("md5", MD5); + options.Filters.AddFilter("sha256", Sha256); + options.Filters.AddFilter("slugify", Slugify); + options.Filters.AddFilter("trim", Trim); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs index 6a3f6c048..dbe38963c 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs @@ -15,17 +15,17 @@ public class StringWordsFluidExtension : IFluidExtension { private static readonly FilterDelegate WordCount = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().WordCount()); + return NumberValue.Create(input.ToStringValue().WordCount()); }; private static readonly FilterDelegate CharacterCount = (input, arguments, context) => { - return FluidValue.Create(input.ToStringValue().CharacterCount()); + return NumberValue.Create(input.ToStringValue().CharacterCount()); }; - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - TemplateContext.GlobalFilters.AddFilter("word_count", WordCount); - TemplateContext.GlobalFilters.AddFilter("character_count", CharacterCount); + options.Filters.AddFilter("word_count", WordCount); + options.Filters.AddFilter("character_count", CharacterCount); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/UserFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/UserFluidExtension.cs index edf7eb7af..00841419b 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/UserFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/UserFluidExtension.cs @@ -6,6 +6,7 @@ // ========================================================================== using Fluid; +using Fluid.Accessors; using Fluid.Values; using Squidex.Shared.Identity; using Squidex.Shared.Users; @@ -14,30 +15,30 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions; public sealed class UserFluidExtension : IFluidExtension { - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { - memberAccessStrategy.Register((user, name) => + options.MemberAccessStrategy.Register(new DelegateAccessor((source, name, context) => { switch (name) { case "id": - return new StringValue(user.Id); + return StringValue.Create(source.Id); case "email": - return new StringValue(user.Email); + return StringValue.Create(source.Email); case "name": - return new StringValue(user.Claims.DisplayName()); + return StringValue.Create(source.Claims.DisplayName()); default: { - var claim = user.Claims.FirstOrDefault(x => string.Equals(name, x.Type, StringComparison.OrdinalIgnoreCase)); + var claim = source.Claims.FirstOrDefault(x => string.Equals(name, x.Type, StringComparison.OrdinalIgnoreCase)); if (claim != null) { - return new StringValue(claim.Value); + return StringValue.Create(claim.Value); } return NilValue.Instance; } } - }); + })); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/FluidTemplateEngine.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/FluidTemplateEngine.cs index b5951b4ac..c79dc35e5 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/FluidTemplateEngine.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/FluidTemplateEngine.cs @@ -13,77 +13,55 @@ namespace Squidex.Domain.Apps.Core.Templates; public sealed class FluidTemplateEngine : ITemplateEngine { - private readonly IEnumerable extensions; + private readonly TemplateOptions options = new TemplateOptions(); + private readonly CustomFluidParser parser = new CustomFluidParser(); - private sealed class SquidexTemplate : BaseFluidTemplate + public FluidTemplateEngine(IEnumerable extensions) { - public static void Setup(IEnumerable extensions) + options.MemberAccessStrategy = new UnsafeMemberAccessStrategy { - foreach (var extension in extensions) - { - extension.RegisterLanguageExtensions(Factory); - } - } + MemberNameStrategy = MemberNameStrategies.CamelCase + }; - public static void SetupTypes(IEnumerable extensions) + foreach (var extension in extensions) { - var globalTypes = TemplateContext.GlobalMemberAccessStrategy; - - globalTypes.MemberNameStrategy = MemberNameStrategies.CamelCase; + extension.RegisterLanguageExtensions(parser, options); + } - foreach (var extension in extensions) + options.ValueConverters.Add(value => + { + if (value is RefTokenType tokenType) { - extension.RegisterGlobalTypes(globalTypes); + return StringValue.Create(tokenType.ToString().ToLowerInvariant()); } - foreach (var type in SquidexCoreModel.Assembly.GetTypes().Where(x => x.IsEnum)) + if (value?.GetType().IsEnum == true) { - FluidValue.SetTypeMapping(type, x => new StringValue(x.ToString())); + return new StringValue(value.ToString()); } - FluidValue.SetTypeMapping(x => new StringValue(x.ToString().ToLowerInvariant())); - - globalTypes.Register>(); - globalTypes.Register>(); - globalTypes.Register>(); - globalTypes.Register>(); - globalTypes.Register(); - } - } - - public FluidTemplateEngine(IEnumerable extensions) - { - this.extensions = extensions; - - SquidexTemplate.Setup(extensions); - SquidexTemplate.SetupTypes(extensions); + return null; + }); } public async Task RenderAsync(string template, TemplateVars variables) { Guard.NotNull(variables); - if (SquidexTemplate.TryParse(template, out var parsed, out var errors)) + if (!parser.TryParse(template, out var parsed, out var error)) { - var context = new TemplateContext(); - - foreach (var extension in extensions) - { - extension.BeforeRun(context); - } - - foreach (var (key, value) in variables) - { - context.MemberAccessStrategy.Register(value.GetType()); - - context.SetValue(key, value); - } + throw new TemplateParseException(template, error); + } - var result = await parsed.RenderAsync(context); + var context = new TemplateContext(options); - return result; + foreach (var (key, value) in variables) + { + context.SetValue(key, value); } - throw new TemplateParseException(template, errors); + var result = await parsed.RenderAsync(context); + + return result; } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/IFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/IFluidExtension.cs index 7811a1346..137bec755 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/IFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/IFluidExtension.cs @@ -11,11 +11,7 @@ namespace Squidex.Domain.Apps.Core.Templates; public interface IFluidExtension { - void RegisterLanguageExtensions(FluidParserFactory factory) - { - } - - void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) + void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) { } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/TemplateParseException.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/TemplateParseException.cs index a6a147a05..39147f584 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/TemplateParseException.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Templates/TemplateParseException.cs @@ -6,48 +6,33 @@ // ========================================================================== using System.Runtime.Serialization; -using System.Text; namespace Squidex.Domain.Apps.Core.Templates; [Serializable] public class TemplateParseException : Exception { - public IReadOnlyList Errors { get; } + public string Error { get; set; } - public TemplateParseException(string template, IEnumerable errors, Exception? inner = null) - : base(BuildErrorMessage(errors, template), inner) + public TemplateParseException(string template, string error, Exception? inner = null) + : base(BuildErrorMessage(error, template), inner) { - Errors = errors.ToList(); + Error = error; } protected TemplateParseException(SerializationInfo info, StreamingContext context) : base(info, context) { - Errors = (info.GetValue(nameof(Errors), typeof(List)) as List) ?? new List(); + Error = info.GetString(nameof(Error)) ?? string.Empty; } public override void GetObjectData(SerializationInfo info, StreamingContext context) { - info.AddValue(nameof(Errors), Errors.ToList()); + info.AddValue(nameof(Error), Error); } - private static string BuildErrorMessage(IEnumerable errors, string template) + private static string BuildErrorMessage(string error, string template) { - var sb = new StringBuilder(); - - sb.AppendLine("Failed to parse template"); - - foreach (var error in errors) - { - sb.Append(" * "); - sb.AppendLine(error); - } - - sb.AppendLine(); - sb.AppendLine("Template:"); - sb.AppendLine(template); - - return sb.ToString(); + return $"Failed to parse template with <{error}>, template: {template}."; } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs index 2594b7ded..5cd40a069 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs @@ -34,6 +34,9 @@ public sealed class MongoTextIndex : MongoTextIndexBase BuildTexts(Dictionary source) { - return source.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList(); + return source.Select(x => new MongoTextIndexEntityText + { + Text = Tokenizer.TokenizerTerms(x.Value, x.Key) + }).ToList(); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs index 80edd1070..5b80f12a8 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs @@ -35,6 +35,17 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase Results { get; } = new List<(DomainId Id, double Score)>(); + + public string SearchTerms { get; set; } + + public IAppEntity App { get; set; } + + public SearchScope SearchScope { get; set; } + } + protected MongoTextIndexBase(IMongoDatabase database) : base(database) { @@ -131,71 +142,74 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase(); + var search = new SearchOperation + { + App = app, + SearchTerms = Tokenizer.TokenizeQuery(query.Text), + SearchScope = scope + }; if (query.RequiredSchemaIds?.Count > 0) { - await SearchBySchemaAsync(result, search, app, query.RequiredSchemaIds, scope, take, 1, ct); + await SearchBySchemaAsync(search, query.RequiredSchemaIds, query.Take, 1, ct); } else if (query.PreferredSchemaId == null) { - await SearchByAppAsync(result, search, app, scope, take, 1, ct); + await SearchByAppAsync(search, query.Take, 1, ct); } else { - var halfBucket = take / 2; + var halfBucket = query.Take / 2; var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); - await SearchBySchemaAsync(result, search, app, schemaIds, scope, halfBucket, 1.1, ct); - await SearchByAppAsync(result, search, app, scope, halfBucket, 1, ct); + await SearchBySchemaAsync(search, schemaIds, halfBucket, 1.1, ct); + await SearchByAppAsync(search, halfBucket, 1, ct); } - return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); + return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); } - private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IAppEntity app, IEnumerable schemaIds, SearchScope scope, int take, double factor, + private Task SearchBySchemaAsync(SearchOperation search, IEnumerable schemaIds, int take, double factor, CancellationToken ct = default) { var filter = Filter.And( - Filter.Eq(x => x.AppId, app.Id), + Filter.Eq(x => x.AppId, search.App.Id), Filter.In(x => x.SchemaId, schemaIds), - Filter_ByScope(scope), - Filter.Text(text, "none")); + Filter_ByScope(search.SearchScope), + Filter.Text(search.SearchTerms, "none")); - return SearchAsync(result, filter, scope, take, factor, ct); + return SearchAsync(search, filter, take, factor, ct); } - private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor, + private Task SearchByAppAsync(SearchOperation search, int take, double factor, CancellationToken ct = default) { var filter = Filter.And( - Filter.Eq(x => x.AppId, app.Id), + Filter.Eq(x => x.AppId, search.App.Id), Filter.Exists(x => x.SchemaId), - Filter_ByScope(scope), - Filter.Text(text, "none")); + Filter_ByScope(search.SearchScope), + Filter.Text(search.SearchTerms, "none")); - return SearchAsync(result, filter, scope, take, factor, ct); + return SearchAsync(search, filter, take, factor, ct); } - private async Task SearchAsync(List<(DomainId, double)> result, FilterDefinition> filter, SearchScope scope, int take, double factor, + private async Task SearchAsync(SearchOperation search, FilterDefinition> filter, int take, double factor, CancellationToken ct = default) { var byText = - await GetCollection(scope).Find(filter).Limit(take) + await GetCollection(search.SearchScope).Find(filter).Limit(take) .Project(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score")) .ToListAsync(ct); - result.AddRange(byText.Select(x => (x.ContentId, x.Score * factor))); + search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor))); } private static FilterDefinition> Filter_ByScope(SearchScope scope) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs new file mode 100644 index 000000000..4a23b9666 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs @@ -0,0 +1,144 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Reflection; +using Lucene.Net.Analysis; +using Lucene.Net.Analysis.Core; +using Lucene.Net.Analysis.Standard; +using Lucene.Net.Analysis.TokenAttributes; +using Lucene.Net.Analysis.Util; +using Lucene.Net.Util; +using Squidex.Domain.Apps.Core; +using Squidex.Infrastructure; +using Squidex.Infrastructure.ObjectPool; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text; + +public static class Tokenizer +{ + private const LuceneVersion Version = LuceneVersion.LUCENE_48; + private static readonly Dictionary StopWords = new Dictionary(StringComparer.OrdinalIgnoreCase); + + static Tokenizer() + { + foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) + { + if (!typeof(Analyzer).IsAssignableFrom(type)) + { + continue; + } + + var language = type.Namespace!.Split('.')[^1]; + + if (language.Length != 2) + { + continue; + } + + try + { + var stopWordMethod = + type.GetProperties(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(x => x.PropertyType == typeof(CharArraySet)); + + if (stopWordMethod != null) + { + var stopWords = (CharArraySet)stopWordMethod.GetValue(null)!; + + StopWords[language] = stopWords; + } + } + catch (MissingMethodException) + { + continue; + } + } + } + + public static string TokenizeQuery(string query) + { + query = query.Trim(); + + var textLanguage = Language.EN.Iso2Code; + var textReader = new StringReader(query); + + if (query.Length > 2 && query[2] == ':') + { + textLanguage = query.Substring(0, 2); + textReader.Read(); + textReader.Read(); + } + + return TokenizeWord(textReader, textLanguage); + } + + public static string TokenizerTerms(string query, string language) + { + var stopWords = + string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ? + null : + StopWords.GetValueOrDefault(language) ?? + StopWords["en"]; + + return Tokenize(new StringReader(query), stopWords); + } + + private static string TokenizeWord(TextReader reader, string language) + { + var stopWords = + string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ? + null : + StopWords.GetValueOrDefault(language) ?? + StopWords["en"]; + + return Tokenize(reader, stopWords); + } + + private static string Tokenize(TextReader reader, CharArraySet? stopWords) + { + var tokenizer = new StandardTokenizer(Version, reader); + var tokenStream = (TokenStream)new StandardFilter(Version, tokenizer); + + // Stop words are case sensitive, therefore we have to lowercase it first. + tokenStream = new LowerCaseFilter(Version, tokenStream); + + if (stopWords != null) + { + tokenStream = new StopFilter(Version, tokenStream, stopWords); + } + + var sb = DefaultPools.StringBuilder.Get(); + try + { + // Actually not idea what this is doing, but it seems to work. + var attribute = tokenStream.AddAttribute(); + + tokenStream.Reset(); + + using (tokenStream) + { + while (tokenStream.IncrementToken()) + { + var text = attribute.ToString(); + + if (sb.Length > 0) + { + sb.Append(' '); + } + + sb.Append(text); + } + } + + return sb.ToString(); + } + finally + { + DefaultPools.StringBuilder.Return(sb); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs index 7290ccd55..1a4c16056 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs @@ -8,79 +8,62 @@ using System.Text.Encodings.Web; using Fluid; using Fluid.Ast; -using Fluid.Tags; using Fluid.Values; using Microsoft.Extensions.DependencyInjection; using Squidex.Assets; using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Templates; -using Squidex.Domain.Apps.Core.ValidateContent; using Squidex.Infrastructure; +using static Parlot.Fluent.Parsers; namespace Squidex.Domain.Apps.Entities.Assets; public sealed class AssetsFluidExtension : IFluidExtension { - private static readonly FluidValue ErrorNullAsset = FluidValue.Create(null); private static readonly FluidValue ErrorNoAsset = new StringValue("NoAsset"); private static readonly FluidValue ErrorNoImage = new StringValue("NoImage"); private static readonly FluidValue ErrorTooBig = new StringValue("ErrorTooBig"); private readonly IServiceProvider serviceProvider; - private sealed class AssetTag : ArgumentsTag + public AssetsFluidExtension(IServiceProvider serviceProvider) { - private readonly IServiceProvider serviceProvider; + this.serviceProvider = serviceProvider; + } - public AssetTag(IServiceProvider serviceProvider) - { - this.serviceProvider = serviceProvider; - } + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) + { + AddAssetFilter(options); + AddAssetTextFilter(options); - public override async ValueTask WriteToAsync(TextWriter writer, - TextEncoder encoder, TemplateContext context, FilterArgument[] arguments) + parser.RegisterParserTag("asset", + parser.PrimaryParser.AndSkip(ZeroOrOne(parser.CommaParser)).And(parser.PrimaryParser), + ResolveAsset); + } + + private async ValueTask ResolveAsset(ValueTuple arguments, TextWriter writer, TextEncoder encoder, TemplateContext context) + { + if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) { - if (arguments.Length == 2 && context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) - { - var id = await arguments[1].Expression.EvaluateAsync(context); + var (nameArg, idArg) = arguments; - var asset = await ResolveAssetAsync(serviceProvider, enrichedEvent.AppId.Id, id); + var assetId = await idArg.EvaluateAsync(context); + var asset = await ResolveAssetAsync(serviceProvider, enrichedEvent.AppId.Id, assetId); - if (asset != null) - { - var name = (await arguments[0].Expression.EvaluateAsync(context)).ToStringValue(); + if (asset != null) + { + var name = (await nameArg.EvaluateAsync(context)).ToStringValue(); - context.SetValue(name, asset); - } + context.SetValue(name, asset); } - - return Completion.Normal; } - } - - public AssetsFluidExtension(IServiceProvider serviceProvider) - { - this.serviceProvider = serviceProvider; - } - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) - { - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register>(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - - AddAssetFilter(); - AddAssetTextFilter(); + return Completion.Normal; } - private void AddAssetFilter() + private void AddAssetFilter(TemplateOptions options) { - TemplateContext.GlobalFilters.AddAsyncFilter("asset", async (input, arguments, context) => + options.Filters.AddFilter("asset", async (input, arguments, context) => { if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) { @@ -88,19 +71,19 @@ public sealed class AssetsFluidExtension : IFluidExtension if (asset == null) { - return ErrorNullAsset; + return NilValue.Instance; } - return FluidValue.Create(asset); + return FluidValue.Create(asset, options); } - return ErrorNullAsset; + return NilValue.Instance; }); } - private void AddAssetTextFilter() + private void AddAssetTextFilter(TemplateOptions options) { - TemplateContext.GlobalFilters.AddAsyncFilter("assetText", async (input, arguments, context) => + options.Filters.AddFilter("assetText", async (input, arguments, context) => { if (input is not ObjectValue objectValue) { @@ -134,7 +117,7 @@ public sealed class AssetsFluidExtension : IFluidExtension return ErrorNoAsset; }); - TemplateContext.GlobalFilters.AddAsyncFilter("assetBlurHash", async (input, arguments, context) => + options.Filters.AddFilter("assetBlurHash", async (input, arguments, context) => { if (input is not ObjectValue objectValue) { @@ -189,11 +172,6 @@ public sealed class AssetsFluidExtension : IFluidExtension }); } - public void RegisterLanguageExtensions(FluidParserFactory factory) - { - factory.RegisterTag("asset", new AssetTag(serviceProvider)); - } - private static async Task ResolveAssetAsync(IServiceProvider serviceProvider, DomainId appId, FluidValue id) { var appProvider = serviceProvider.GetRequiredService(); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ReferencesFluidExtension.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ReferencesFluidExtension.cs index 6120c27b8..bf4e01cce 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/ReferencesFluidExtension.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/ReferencesFluidExtension.cs @@ -8,12 +8,13 @@ using System.Text.Encodings.Web; using Fluid; using Fluid.Ast; -using Fluid.Tags; using Fluid.Values; using Microsoft.Extensions.DependencyInjection; using Squidex.Domain.Apps.Core.Rules.EnrichedEvents; using Squidex.Domain.Apps.Core.Templates; +using Squidex.Domain.Apps.Entities.Contents.Queries.Steps; using Squidex.Infrastructure; +using static Parlot.Fluent.Parsers; #pragma warning disable CA1826 // Do not use Enumerable methods on indexable collections @@ -21,60 +22,45 @@ namespace Squidex.Domain.Apps.Entities.Contents; public sealed class ReferencesFluidExtension : IFluidExtension { - private static readonly FluidValue ErrorNullReference = FluidValue.Create(null); private readonly IServiceProvider serviceProvider; - private sealed class ReferenceTag : ArgumentsTag + public ReferencesFluidExtension(IServiceProvider serviceProvider) { - private readonly IServiceProvider serviceProvider; + this.serviceProvider = serviceProvider; + } - public ReferenceTag(IServiceProvider serviceProvider) - { - this.serviceProvider = serviceProvider; - } + public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options) + { + AddReferenceFilter(options); - public override async ValueTask WriteToAsync(TextWriter writer, - TextEncoder encoder, TemplateContext context, FilterArgument[] arguments) + parser.RegisterParserTag("reference", + parser.PrimaryParser.AndSkip(ZeroOrOne(parser.CommaParser)).And(parser.PrimaryParser), + ResolveReference); + } + + private async ValueTask ResolveReference(ValueTuple arguments, TextWriter writer, TextEncoder encoder, TemplateContext context) + { + if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) { - if (arguments.Length == 2 && context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) - { - var id = await arguments[1].Expression.EvaluateAsync(context); + var (nameArg, idArg) = arguments; - var content = await ResolveContentAsync(serviceProvider, enrichedEvent.AppId.Id, id); + var contentId = await idArg.EvaluateAsync(context); + var content = await ResolveContentAsync(serviceProvider, enrichedEvent.AppId.Id, contentId); - if (content != null) - { - var name = (await arguments[0].Expression.EvaluateAsync(context)).ToStringValue(); + if (content != null) + { + var name = (await nameArg.EvaluateAsync(context)).ToStringValue(); - context.SetValue(name, content); - } + context.SetValue(name, content); } - - return Completion.Normal; } - } - - public ReferencesFluidExtension(IServiceProvider serviceProvider) - { - this.serviceProvider = serviceProvider; - } - public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy) - { - memberAccessStrategy.Register(); - memberAccessStrategy.Register>(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - memberAccessStrategy.Register(); - - AddReferenceFilter(); + return Completion.Normal; } - private void AddReferenceFilter() + private void AddReferenceFilter(TemplateOptions options) { - TemplateContext.GlobalFilters.AddAsyncFilter("reference", async (input, arguments, context) => + options.Filters.AddFilter("reference", async (input, arguments, context) => { if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent) { @@ -82,21 +68,16 @@ public sealed class ReferencesFluidExtension : IFluidExtension if (content == null) { - return ErrorNullReference; + return NilValue.Instance; } - return FluidValue.Create(content); + return FluidValue.Create(content, options); } - return ErrorNullReference; + return NilValue.Instance; }); } - public void RegisterLanguageExtensions(FluidParserFactory factory) - { - factory.RegisterTag("reference", new ReferenceTag(serviceProvider)); - } - private static async Task ResolveContentAsync(IServiceProvider serviceProvider, DomainId appId, FluidValue id) { var appProvider = serviceProvider.GetRequiredService(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs index e902793b6..c5a08b725 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs @@ -76,8 +76,6 @@ public class AssetsFluidExtensionTests [Fact] public async Task Should_resolve_assets_in_loop() { - var (vars, assets) = SetupAssetsVars(); - var template = @" {% for id in event.data.assets.iv %} {% asset 'ref', id %} @@ -85,21 +83,25 @@ public class AssetsFluidExtensionTests {% endfor %} "; - var expected = $@" - Text: {assets[0].FileName} {assets[0].Id} - Text: {assets[1].FileName} {assets[1].Id} - "; + await ResolveAssetsAsync(template); + } - var actual = await sut.RenderAsync(template, vars); + [Fact] + public async Task Should_resolve_assets_in_loop_without_commata() + { + var template = @" + {% for id in event.data.assets.iv %} + {% asset 'ref' id %} + Text: {{ ref.fileName }} {{ ref.id }} + {% endfor %} + "; - Assert.Equal(Cleanup(expected), Cleanup(actual)); + await ResolveAssetsAsync(template); } [Fact] public async Task Should_resolve_assets_in_loop_with_filter() { - var (vars, assets) = SetupAssetsVars(); - var template = @" {% for id in event.data.assets.iv %} {% assign ref = id | asset %} @@ -107,6 +109,13 @@ public class AssetsFluidExtensionTests {% endfor %} "; + await ResolveAssetsAsync(template); + } + + private async Task ResolveAssetsAsync(string template) + { + var (vars, assets) = SetupAssetsVars(); + var expected = $@" Text: {assets[0].FileName} {assets[0].Id} Text: {assets[1].FileName} {assets[1].Id} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs new file mode 100644 index 000000000..7da20e7e8 --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs @@ -0,0 +1,63 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Entities.MongoDb.Text; + +namespace Squidex.Domain.Apps.Entities.Contents.MongoDb; + +public class TokenizerTests +{ + [Fact] + public void Should_eliminate_stop_words_in_implicit_english() + { + var source = "The only thing that matters, is time."; + + var parsed = Tokenizer.TokenizeQuery(source); + + Assert.Equal("only thing matters time", parsed); + } + + [Fact] + public void Should_eliminate_stop_words_in_explicit_english() + { + var source = "en:The only thing that matters, is time."; + + var parsed = Tokenizer.TokenizeQuery(source); + + Assert.Equal("only thing matters time", parsed); + } + + [Fact] + public void Should_eliminate_stop_words_in_explicit_english2() + { + var source = "en:when i do this it is pretty slow"; + + var parsed = Tokenizer.TokenizeQuery(source); + + Assert.Equal("when i do pretty slow", parsed); + } + + [Fact] + public void Should_not_eliminate_stop_words_for_iv_language() + { + var source = "iv:The only thing that matters, is time."; + + var parsed = Tokenizer.TokenizeQuery(source); + + Assert.Equal("the only thing that matters is time", parsed); + } + + [Fact] + public void Should_eliminate_stop_words_in_explicit_german() + { + var source = "de:Nur die Zeit spielt eine Rolle"; + + var parsed = Tokenizer.TokenizeQuery(source); + + Assert.Equal("zeit spielt rolle", parsed); + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryServiceTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryServiceTests.cs index b26d7dcde..5aa81e6eb 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryServiceTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryServiceTests.cs @@ -54,7 +54,7 @@ public class ContentQueryServiceTests A.CallTo(() => appProvider.GetSchemasAsync(appId.Id, ct)) .Returns(new List { schema }); - A.CallTo(() => queryParser.ParseAsync(A._, A._, A._)) + A.CallTo(() => queryParser.ParseAsync(A._, A._, A._, ct)) .ReturnsLazily(c => Task.FromResult(c.GetArgument(1)!)); var options = Options.Create(new ContentOptions()); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ReferencesFluidExtensionTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ReferencesFluidExtensionTests.cs index 342b0da91..058948fc1 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ReferencesFluidExtensionTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ReferencesFluidExtensionTests.cs @@ -46,32 +46,6 @@ public class ReferencesFluidExtensionTests [Fact] public async Task Should_resolve_references_in_loop() { - var referenceId1 = DomainId.NewGuid(); - var reference1 = CreateReference(referenceId1, 1); - var referenceId2 = DomainId.NewGuid(); - var reference2 = CreateReference(referenceId2, 2); - - var @event = new EnrichedContentEvent - { - Data = - new ContentData() - .AddField("references", - new ContentFieldData() - .AddInvariant(JsonValue.Array(referenceId1, referenceId2))), - AppId = appId - }; - - A.CallTo(() => contentQuery.QueryAsync(A._, A.That.HasIds(referenceId1), A._)) - .Returns(ResultList.CreateFrom(1, reference1)); - - A.CallTo(() => contentQuery.QueryAsync(A._, A.That.HasIds(referenceId2), A._)) - .Returns(ResultList.CreateFrom(1, reference2)); - - var vars = new TemplateVars - { - ["event"] = @event - }; - var template = @" {% for id in event.data.references.iv %} {% reference 'ref', id %} @@ -79,18 +53,36 @@ public class ReferencesFluidExtensionTests {% endfor %} "; - var expected = $@" - Text: Hello 1 World 1 {referenceId1} - Text: Hello 2 World 2 {referenceId2} - "; + await ResolveReferencesAsync(template); + } - var actual = await sut.RenderAsync(template, vars); + [Fact] + public async Task Should_resolve_references_in_loop_without_commata() + { + var template = @" + {% for id in event.data.references.iv %} + {% reference 'ref' id %} + Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }} + {% endfor %} + "; - Assert.Equal(Cleanup(expected), Cleanup(actual)); + await ResolveReferencesAsync(template); } [Fact] public async Task Should_resolve_references_in_loop_with_filter() + { + var template = @" + {% for id in event.data.references.iv %} + {% assign ref = id | reference %} + Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }} + {% endfor %} + "; + + await ResolveReferencesAsync(template); + } + + private async Task ResolveReferencesAsync(string template) { var referenceId1 = DomainId.NewGuid(); var reference1 = CreateReference(referenceId1, 1); @@ -118,13 +110,6 @@ public class ReferencesFluidExtensionTests ["event"] = @event }; - var template = @" - {% for id in event.data.references.iv %} - {% assign ref = id | reference %} - Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }} - {% endfor %} - "; - var expected = $@" Text: Hello 1 World 1 {referenceId1} Text: Hello 2 World 2 {referenceId2}