Eliminate stop words before search. (#949)

* Eliminate stop words before search.

* Improve tokenization.

* Fix tests

* Update fluid

* Cleanup.

backend/src/Squidex.Domain.Apps.Core.Operations/HandleRules/Extensions/EventFluidExtensions.cs

@@ -23,15 +23,15 @@ public sealed class EventFluidExtensions : IFluidExtension
         this.urlGenerator = urlGenerator;
     }
 
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddFilter("contentUrl", ContentUrl);
+        options.Filters.AddFilter("contentUrl", ContentUrl);
-        TemplateContext.GlobalFilters.AddFilter("assetContentUrl", AssetContentUrl);
+        options.Filters.AddFilter("assetContentUrl", AssetContentUrl);
-        TemplateContext.GlobalFilters.AddFilter("assetContentAppUrl", AssetContentAppUrl);
+        options.Filters.AddFilter("assetContentAppUrl", AssetContentAppUrl);
-        TemplateContext.GlobalFilters.AddFilter("assetContentSlugUrl", AssetContentSlugUrl);
+        options.Filters.AddFilter("assetContentSlugUrl", AssetContentSlugUrl);
     }
 
-    private FluidValue ContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private ValueTask<FluidValue> ContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
     {
         var value = input.ToObjectValue();
 
@@ -60,7 +60,7 @@ public sealed class EventFluidExtensions : IFluidExtension
         return NilValue.Empty;
     }
 
-    private FluidValue AssetContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private ValueTask<FluidValue> AssetContentUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
     {
         var value = input.ToObjectValue();
 
@@ -89,7 +89,7 @@ public sealed class EventFluidExtensions : IFluidExtension
         return NilValue.Empty;
     }
 
-    private FluidValue AssetContentAppUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private ValueTask<FluidValue> AssetContentAppUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
     {
         var value = input.ToObjectValue();
 
@@ -118,7 +118,7 @@ public sealed class EventFluidExtensions : IFluidExtension
         return NilValue.Empty;
     }
 
-    private FluidValue AssetContentSlugUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private ValueTask<FluidValue> AssetContentSlugUrl(FluidValue input, FilterArguments arguments, TemplateContext context)
     {
         var value = input.ToObjectValue();
 

backend/src/Squidex.Domain.Apps.Core.Operations/Squidex.Domain.Apps.Core.Operations.csproj

@@ -18,7 +18,7 @@
     <ProjectReference Include="..\Squidex.Shared\Squidex.Shared.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Fluid.Core.Squidex" Version="1.0.0-beta" />
+    <PackageReference Include="Fluid.Core" Version="2.3.0" />
 	<PackageReference Include="GeoJSON.Net" Version="1.2.19" />
 	<PackageReference Include="Jint" Version="3.0.0-beta-2036" />
 	<PackageReference Include="Meziantou.Analyzer" Version="1.0.746">

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/CustomFluidParser.cs

@@ -0,0 +1,19 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Fluid;
+using Fluid.Ast;
+using Parlot.Fluent;
+
+namespace Squidex.Domain.Apps.Core.Templates;
+
+public sealed class CustomFluidParser : FluidParser
+{
+    public Deferred<Expression> PrimaryParser => Primary;
+
+    public Parser<char> CommaParser => Comma;
+}

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/ContentFluidExtension.cs

@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using Fluid;
+using Fluid.Accessors;
 using Fluid.Values;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Infrastructure;
@@ -15,52 +16,57 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions;
 
 public sealed class ContentFluidExtension : IFluidExtension
 {
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        FluidValue.SetTypeMapping<ContentData>(x => new ObjectValue(x));
+        options.ValueConverters.Add(source =>
-        FluidValue.SetTypeMapping<ContentFieldData>(x => new ObjectValue(x));
-        FluidValue.SetTypeMapping<JsonObject>(x => new ObjectValue(x));
-        FluidValue.SetTypeMapping<JsonArray>(x => new JsonArrayFluidValue(x));
-
-        FluidValue.SetTypeMapping<JsonValue>(source =>
         {
-            switch (source.Value)
+            switch (source)
             {
-                case null:
+                case ContentData d:
-                    return FluidValue.Create(null);
+                    return new ObjectValue(d);
-                case bool b:
+                case ContentFieldData f:
-                    return FluidValue.Create(b);
+                    return new ObjectValue(f);
-                case double n:
+                case JsonArray a:
-                    return FluidValue.Create(n);
+                    return new JsonArrayFluidValue(a, options);
-                case string s:
-                    return FluidValue.Create(s);
                 case JsonObject o:
                     return new ObjectValue(o);
-                case JsonArray a:
+                case JsonValue v:
-                    return new JsonArrayFluidValue(a);
+                    switch (v.Value)
-            }
+                    {
-
+                        case null:
-            ThrowHelper.InvalidOperationException();
+                            return NilValue.Instance;
-            return default!;
+                        case bool b:
-        });
+                            return BooleanValue.Create(b);
+                        case double n:
+                            return NumberValue.Create((decimal)n);
+                        case string s:
+                            return StringValue.Create(s);
+                        case JsonArray a:
+                            return new JsonArrayFluidValue(a, options);
+                        case JsonObject o:
+                            return new ObjectValue(o);
+                    }
 
-        memberAccessStrategy.Register<JsonValue, object?>((value, name) =>
+                    ThrowHelper.InvalidOperationException();
-        {
+                    break;
-            if (value.Value is JsonObject o)
-            {
-                return o.GetValueOrDefault(name);
             }
 
             return null;
         });
 
-        memberAccessStrategy.Register<ContentData, object?>(
+        options.MemberAccessStrategy.Register<ContentData>("*", new DelegateAccessor<ContentData, object?>((source, name, context) =>
-            (value, name) => value.GetValueOrDefault(name));
+        {
+            return source.GetValueOrDefault(name);
+        }));
 
-        memberAccessStrategy.Register<ContentFieldData, object?>(
+        options.MemberAccessStrategy.Register<ContentFieldData>("*", new DelegateAccessor<ContentFieldData, object?>((source, name, context) =>
-            (value, name) => value.GetValueOrDefault(name).Value);
+        {
+            return source.GetValueOrDefault(name);
+        }));
 
-        memberAccessStrategy.Register<JsonObject, object?>(
+        options.MemberAccessStrategy.Register<JsonObject>("*", new DelegateAccessor<JsonObject, object?>((source, name, context) =>
-            (value, name) => value.GetValueOrDefault(name).Value);
+        {
+            return source.GetValueOrDefault(name);
+        }));
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/DateTimeFluidExtension.cs

@@ -15,35 +15,34 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions;
 
 public class DateTimeFluidExtension : IFluidExtension
 {
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    private static readonly FilterDelegate FormatDate = (input, arguments, context) =>
     {
-        TemplateContext.GlobalFilters.AddFilter("format_date", FormatDate);
+        if (arguments.Count == 1)
+        {
+            return FormatDateCore(input, x => Format(arguments, x));
+        }
 
-        TemplateContext.GlobalFilters.AddFilter("timestamp", FormatTimestamp);
+        return input;
-        TemplateContext.GlobalFilters.AddFilter("timestamp_sec", FormatTimestampSec);
+    };
-    }
 
-    public static FluidValue FormatTimestamp(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private static readonly FilterDelegate FormatTimestamp = (input, arguments, context) =>
     {
-        return FormatDate(input, x => FluidValue.Create(x.ToUnixTimeMilliseconds()));
+        return FormatDateCore(input, x => NumberValue.Create(x.ToUnixTimeMilliseconds()));
-    }
+    };
 
-    public static FluidValue FormatTimestampSec(FluidValue input, FilterArguments arguments, TemplateContext context)
+    private static readonly FilterDelegate FormatTimestampSec = (input, arguments, context) =>
     {
-        return FormatDate(input, x => FluidValue.Create(x.ToUnixTimeMilliseconds() / 1000));
+        return FormatDateCore(input, x => NumberValue.Create(x.ToUnixTimeMilliseconds() / 1000));
-    }
+    };
 
-    public static FluidValue FormatDate(FluidValue input, FilterArguments arguments, TemplateContext context)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        if (arguments.Count == 1)
+        options.Filters.AddFilter("format_date", FormatDate);
-        {
+        options.Filters.AddFilter("timestamp", FormatTimestamp);
-            return FormatDate(input, x => Format(arguments, x));
+        options.Filters.AddFilter("timestamp_sec", FormatTimestampSec);
-        }
-
-        return input;
     }
 
-    private static FluidValue FormatDate(FluidValue input, Func<DateTimeOffset, FluidValue> formatter)
+    private static ValueTask<FluidValue> FormatDateCore(FluidValue input, Func<DateTimeOffset, FluidValue> formatter)
     {
         switch (input)
         {

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/JsonArrayFluidValue.cs

@@ -16,12 +16,14 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions;
 public sealed class JsonArrayFluidValue : FluidValue
 {
     private readonly JsonArray value;
+    private readonly TemplateOptions options;
 
     public override FluidValues Type { get; } = FluidValues.Array;
 
-    public JsonArrayFluidValue(JsonArray value)
+    public JsonArrayFluidValue(JsonArray value, TemplateOptions options)
     {
         this.value = value;
+        this.options = options;
     }
 
     public override bool Equals(FluidValue other)
@@ -59,7 +61,7 @@ public sealed class JsonArrayFluidValue : FluidValue
             case "first":
                 if (value.Count > 0)
                 {
-                    return Create(value[0]);
+                    return Create(value[0], options);
                 }
 
                 break;
@@ -67,7 +69,7 @@ public sealed class JsonArrayFluidValue : FluidValue
             case "last":
                 if (value.Count > 0)
                 {
-                    return Create(value[^1]);
+                    return Create(value[^1], options);
                 }
 
                 break;
@@ -82,17 +84,17 @@ public sealed class JsonArrayFluidValue : FluidValue
 
         if (i >= 0 && i < value.Count)
         {
-            return Create(value[i]);
+            return Create(value[i], options);
         }
 
         return NilValue.Instance;
     }
 
-    public override IEnumerable<FluidValue> Enumerate()
+    public override IEnumerable<FluidValue> Enumerate(TemplateContext context)
     {
         foreach (var item in value)
         {
-            yield return Create(item);
+            yield return Create(item, options);
         }
     }
 

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringFluidExtension.cs

@@ -20,7 +20,7 @@ public sealed class StringFluidExtension : IFluidExtension
         {
             var result = value.ToStringValue().Slugify();
 
-            return FluidValue.Create(result);
+            return StringValue.Create(result);
         }
 
         return input;
@@ -28,42 +28,42 @@ public sealed class StringFluidExtension : IFluidExtension
 
     private static readonly FilterDelegate Escape = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().JsonEscape());
+        return StringValue.Create(input.ToStringValue().JsonEscape());
     };
 
     private static readonly FilterDelegate Markdown2Text = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().Markdown2Text());
+        return StringValue.Create(input.ToStringValue().Markdown2Text());
     };
 
     private static readonly FilterDelegate Html2Text = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().Html2Text());
+        return StringValue.Create(input.ToStringValue().Html2Text());
     };
 
     private static readonly FilterDelegate Trim = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().Trim());
+        return StringValue.Create(input.ToStringValue().Trim());
     };
 
     private static readonly FilterDelegate MD5 = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().ToMD5());
+        return StringValue.Create(input.ToStringValue().ToMD5());
     };
 
     private static readonly FilterDelegate Sha256 = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().ToSha256());
+        return StringValue.Create(input.ToStringValue().ToSha256());
     };
 
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddFilter("escape", Escape);
+        options.Filters.AddFilter("escape", Escape);
-        TemplateContext.GlobalFilters.AddFilter("html2text", Html2Text);
+        options.Filters.AddFilter("html2text", Html2Text);
-        TemplateContext.GlobalFilters.AddFilter("markdown2text", Markdown2Text);
+        options.Filters.AddFilter("markdown2text", Markdown2Text);
-        TemplateContext.GlobalFilters.AddFilter("md5", MD5);
+        options.Filters.AddFilter("md5", MD5);
-        TemplateContext.GlobalFilters.AddFilter("sha256", Sha256);
+        options.Filters.AddFilter("sha256", Sha256);
-        TemplateContext.GlobalFilters.AddFilter("slugify", Slugify);
+        options.Filters.AddFilter("slugify", Slugify);
-        TemplateContext.GlobalFilters.AddFilter("trim", Trim);
+        options.Filters.AddFilter("trim", Trim);
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/StringWordsFluidExtension.cs

@@ -15,17 +15,17 @@ public class StringWordsFluidExtension : IFluidExtension
 {
     private static readonly FilterDelegate WordCount = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().WordCount());
+        return NumberValue.Create(input.ToStringValue().WordCount());
     };
 
     private static readonly FilterDelegate CharacterCount = (input, arguments, context) =>
     {
-        return FluidValue.Create(input.ToStringValue().CharacterCount());
+        return NumberValue.Create(input.ToStringValue().CharacterCount());
     };
 
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddFilter("word_count", WordCount);
+        options.Filters.AddFilter("word_count", WordCount);
-        TemplateContext.GlobalFilters.AddFilter("character_count", CharacterCount);
+        options.Filters.AddFilter("character_count", CharacterCount);
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/Extensions/UserFluidExtension.cs

@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using Fluid;
+using Fluid.Accessors;
 using Fluid.Values;
 using Squidex.Shared.Identity;
 using Squidex.Shared.Users;
@@ -14,30 +15,30 @@ namespace Squidex.Domain.Apps.Core.Templates.Extensions;
 
 public sealed class UserFluidExtension : IFluidExtension
 {
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
     {
-        memberAccessStrategy.Register<IUser, FluidValue>((user, name) =>
+        options.MemberAccessStrategy.Register<IUser>(new DelegateAccessor<IUser, FluidValue>((source, name, context) =>
         {
             switch (name)
             {
                 case "id":
-                    return new StringValue(user.Id);
+                    return StringValue.Create(source.Id);
                 case "email":
-                    return new StringValue(user.Email);
+                    return StringValue.Create(source.Email);
                 case "name":
-                    return new StringValue(user.Claims.DisplayName());
+                    return StringValue.Create(source.Claims.DisplayName());
                 default:
                     {
-                        var claim = user.Claims.FirstOrDefault(x => string.Equals(name, x.Type, StringComparison.OrdinalIgnoreCase));
+                        var claim = source.Claims.FirstOrDefault(x => string.Equals(name, x.Type, StringComparison.OrdinalIgnoreCase));
 
                         if (claim != null)
                         {
-                            return new StringValue(claim.Value);
+                            return StringValue.Create(claim.Value);
                         }
 
                         return NilValue.Instance;
                     }
             }
-        });
+        }));
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/FluidTemplateEngine.cs

@@ -13,77 +13,55 @@ namespace Squidex.Domain.Apps.Core.Templates;
 
 public sealed class FluidTemplateEngine : ITemplateEngine
 {
-    private readonly IEnumerable<IFluidExtension> extensions;
+    private readonly TemplateOptions options = new TemplateOptions();
+    private readonly CustomFluidParser parser = new CustomFluidParser();
 
-    private sealed class SquidexTemplate : BaseFluidTemplate<SquidexTemplate>
+    public FluidTemplateEngine(IEnumerable<IFluidExtension> extensions)
     {
-        public static void Setup(IEnumerable<IFluidExtension> extensions)
+        options.MemberAccessStrategy = new UnsafeMemberAccessStrategy
         {
-            foreach (var extension in extensions)
+            MemberNameStrategy = MemberNameStrategies.CamelCase
-            {
+        };
-                extension.RegisterLanguageExtensions(Factory);
-            }
-        }
 
-        public static void SetupTypes(IEnumerable<IFluidExtension> extensions)
+        foreach (var extension in extensions)
         {
-            var globalTypes = TemplateContext.GlobalMemberAccessStrategy;
+            extension.RegisterLanguageExtensions(parser, options);
-
+        }
-            globalTypes.MemberNameStrategy = MemberNameStrategies.CamelCase;
 
-            foreach (var extension in extensions)
+        options.ValueConverters.Add(value =>
+        {
+            if (value is RefTokenType tokenType)
             {
-                extension.RegisterGlobalTypes(globalTypes);
+                return StringValue.Create(tokenType.ToString().ToLowerInvariant());
             }
 
-            foreach (var type in SquidexCoreModel.Assembly.GetTypes().Where(x => x.IsEnum))
+            if (value?.GetType().IsEnum == true)
             {
-                FluidValue.SetTypeMapping(type, x => new StringValue(x.ToString()));
+                return new StringValue(value.ToString());
             }
 
-            FluidValue.SetTypeMapping<RefTokenType>(x => new StringValue(x.ToString().ToLowerInvariant()));
+            return null;
-
+        });
-            globalTypes.Register<NamedId<DomainId>>();
-            globalTypes.Register<NamedId<Guid>>();
-            globalTypes.Register<NamedId<string>>();
-            globalTypes.Register<NamedId<long>>();
-            globalTypes.Register<RefToken>();
-        }
-    }
-
-    public FluidTemplateEngine(IEnumerable<IFluidExtension> extensions)
-    {
-        this.extensions = extensions;
-
-        SquidexTemplate.Setup(extensions);
-        SquidexTemplate.SetupTypes(extensions);
     }
 
     public async Task<string> RenderAsync(string template, TemplateVars variables)
     {
         Guard.NotNull(variables);
 
-        if (SquidexTemplate.TryParse(template, out var parsed, out var errors))
+        if (!parser.TryParse(template, out var parsed, out var error))
         {
-            var context = new TemplateContext();
+            throw new TemplateParseException(template, error);
-
+        }
-            foreach (var extension in extensions)
-            {
-                extension.BeforeRun(context);
-            }
-
-            foreach (var (key, value) in variables)
-            {
-                context.MemberAccessStrategy.Register(value.GetType());
-
-                context.SetValue(key, value);
-            }
 
-            var result = await parsed.RenderAsync(context);
+        var context = new TemplateContext(options);
 
-            return result;
+        foreach (var (key, value) in variables)
+        {
+            context.SetValue(key, value);
         }
 
-        throw new TemplateParseException(template, errors);
+        var result = await parsed.RenderAsync(context);
+
+        return result;
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/IFluidExtension.cs

@@ -11,11 +11,7 @@ namespace Squidex.Domain.Apps.Core.Templates;
 
 public interface IFluidExtension
 {
-    void RegisterLanguageExtensions(FluidParserFactory factory)
+    void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
-    {
-    }
-
-    void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
     {
     }
 

backend/src/Squidex.Domain.Apps.Core.Operations/Templates/TemplateParseException.cs

@@ -6,48 +6,33 @@
 // ==========================================================================
 
 using System.Runtime.Serialization;
-using System.Text;
 
 namespace Squidex.Domain.Apps.Core.Templates;
 
 [Serializable]
 public class TemplateParseException : Exception
 {
-    public IReadOnlyList<string> Errors { get; }
+    public string Error { get; set; }
 
-    public TemplateParseException(string template, IEnumerable<string> errors, Exception? inner = null)
+    public TemplateParseException(string template, string error, Exception? inner = null)
-        : base(BuildErrorMessage(errors, template), inner)
+        : base(BuildErrorMessage(error, template), inner)
     {
-        Errors = errors.ToList();
+        Error = error;
     }
 
     protected TemplateParseException(SerializationInfo info, StreamingContext context)
         : base(info, context)
     {
-        Errors = (info.GetValue(nameof(Errors), typeof(List<string>)) as List<string>) ?? new List<string>();
+        Error = info.GetString(nameof(Error)) ?? string.Empty;
     }
 
     public override void GetObjectData(SerializationInfo info, StreamingContext context)
     {
-        info.AddValue(nameof(Errors), Errors.ToList());
+        info.AddValue(nameof(Error), Error);
     }
 
-    private static string BuildErrorMessage(IEnumerable<string> errors, string template)
+    private static string BuildErrorMessage(string error, string template)
     {
-        var sb = new StringBuilder();
+        return $"Failed to parse template with <{error}>, template: {template}.";
-
-        sb.AppendLine("Failed to parse template");
-
-        foreach (var error in errors)
-        {
-            sb.Append(" * ");
-            sb.AppendLine(error);
-        }
-
-        sb.AppendLine();
-        sb.AppendLine("Template:");
-        sb.AppendLine(template);
-
-        return sb.ToString();
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs

@@ -34,6 +34,9 @@ public sealed class MongoTextIndex : MongoTextIndexBase<List<MongoTextIndexEntit
 
     protected override List<MongoTextIndexEntityText> BuildTexts(Dictionary<string, string> source)
     {
-        return source.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList();
+        return source.Select(x => new MongoTextIndexEntityText
+        {
+            Text = Tokenizer.TokenizerTerms(x.Value, x.Key)
+        }).ToList();
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs

@@ -35,6 +35,17 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         public double Score { get; set; }
     }
 
+    private sealed class SearchOperation
+    {
+        public List<(DomainId Id, double Score)> Results { get; } = new List<(DomainId Id, double Score)>();
+
+        public string SearchTerms { get; set; }
+
+        public IAppEntity App { get; set; }
+
+        public SearchScope SearchScope { get; set; }
+    }
+
     protected MongoTextIndexBase(IMongoDatabase database)
         : base(database)
     {
@@ -131,71 +142,74 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         Guard.NotNull(app);
         Guard.NotNull(query);
 
-        var (search, take) = query;
+        if (string.IsNullOrWhiteSpace(query.Text))
-
-        if (string.IsNullOrWhiteSpace(search))
         {
             return null;
         }
 
-        var result = new List<(DomainId Id, double Score)>();
+        var search = new SearchOperation
+        {
+            App = app,
+            SearchTerms = Tokenizer.TokenizeQuery(query.Text),
+            SearchScope = scope
+        };
 
         if (query.RequiredSchemaIds?.Count > 0)
         {
-            await SearchBySchemaAsync(result, search, app, query.RequiredSchemaIds, scope, take, 1, ct);
+            await SearchBySchemaAsync(search, query.RequiredSchemaIds, query.Take, 1, ct);
         }
         else if (query.PreferredSchemaId == null)
         {
-            await SearchByAppAsync(result, search, app, scope, take, 1, ct);
+            await SearchByAppAsync(search, query.Take, 1, ct);
         }
         else
         {
-            var halfBucket = take / 2;
+            var halfBucket = query.Take / 2;
 
             var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1);
 
-            await SearchBySchemaAsync(result, search, app, schemaIds, scope, halfBucket, 1.1, ct);
+            await SearchBySchemaAsync(search, schemaIds, halfBucket, 1.1, ct);
-            await SearchByAppAsync(result, search, app, scope, halfBucket, 1, ct);
+            await SearchByAppAsync(search, halfBucket, 1, ct);
         }
 
-        return result.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
+        return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
     }
 
-    private Task SearchBySchemaAsync(List<(DomainId, double)> result, string text, IAppEntity app, IEnumerable<DomainId> schemaIds, SearchScope scope, int take, double factor,
+    private Task SearchBySchemaAsync(SearchOperation search, IEnumerable<DomainId> schemaIds, int take, double factor,
         CancellationToken ct = default)
     {
         var filter =
             Filter.And(
-                Filter.Eq(x => x.AppId, app.Id),
+                Filter.Eq(x => x.AppId, search.App.Id),
                 Filter.In(x => x.SchemaId, schemaIds),
-                Filter_ByScope(scope),
+                Filter_ByScope(search.SearchScope),
-                Filter.Text(text, "none"));
+                Filter.Text(search.SearchTerms, "none"));
 
-        return SearchAsync(result, filter, scope, take, factor, ct);
+        return SearchAsync(search, filter, take, factor, ct);
     }
 
-    private Task SearchByAppAsync(List<(DomainId, double)> result, string text, IAppEntity app, SearchScope scope, int take, double factor,
+    private Task SearchByAppAsync(SearchOperation search, int take, double factor,
         CancellationToken ct = default)
     {
         var filter =
             Filter.And(
-                Filter.Eq(x => x.AppId, app.Id),
+                Filter.Eq(x => x.AppId, search.App.Id),
                 Filter.Exists(x => x.SchemaId),
-                Filter_ByScope(scope),
+                Filter_ByScope(search.SearchScope),
-                Filter.Text(text, "none"));
+                Filter.Text(search.SearchTerms, "none"));
 
-        return SearchAsync(result, filter, scope, take, factor, ct);
+        return SearchAsync(search, filter, take, factor, ct);
     }
 
-    private async Task SearchAsync(List<(DomainId, double)> result, FilterDefinition<MongoTextIndexEntity<T>> filter, SearchScope scope, int take, double factor,
+    private async Task SearchAsync(SearchOperation search, FilterDefinition<MongoTextIndexEntity<T>> filter, int take, double factor,
         CancellationToken ct = default)
     {
         var byText =
-            await GetCollection(scope).Find(filter).Limit(take)
+            await GetCollection(search.SearchScope).Find(filter).Limit(take)
                 .Project<MongoTextResult>(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score"))
                 .ToListAsync(ct);
 
-        result.AddRange(byText.Select(x => (x.ContentId, x.Score * factor)));
+        search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor)));
     }
 
     private static FilterDefinition<MongoTextIndexEntity<T>> Filter_ByScope(SearchScope scope)

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs

@@ -0,0 +1,144 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Reflection;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Core;
+using Lucene.Net.Analysis.Standard;
+using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Analysis.Util;
+using Lucene.Net.Util;
+using Squidex.Domain.Apps.Core;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.ObjectPool;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
+
+public static class Tokenizer
+{
+    private const LuceneVersion Version = LuceneVersion.LUCENE_48;
+    private static readonly Dictionary<string, CharArraySet> StopWords = new Dictionary<string, CharArraySet>(StringComparer.OrdinalIgnoreCase);
+
+    static Tokenizer()
+    {
+        foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes())
+        {
+            if (!typeof(Analyzer).IsAssignableFrom(type))
+            {
+                continue;
+            }
+
+            var language = type.Namespace!.Split('.')[^1];
+
+            if (language.Length != 2)
+            {
+                continue;
+            }
+
+            try
+            {
+                var stopWordMethod =
+                    type.GetProperties(BindingFlags.Public | BindingFlags.Static)
+                        .FirstOrDefault(x => x.PropertyType == typeof(CharArraySet));
+
+                if (stopWordMethod != null)
+                {
+                    var stopWords = (CharArraySet)stopWordMethod.GetValue(null)!;
+
+                    StopWords[language] = stopWords;
+                }
+            }
+            catch (MissingMethodException)
+            {
+                continue;
+            }
+        }
+    }
+
+    public static string TokenizeQuery(string query)
+    {
+        query = query.Trim();
+
+        var textLanguage = Language.EN.Iso2Code;
+        var textReader = new StringReader(query);
+
+        if (query.Length > 2 && query[2] == ':')
+        {
+            textLanguage = query.Substring(0, 2);
+            textReader.Read();
+            textReader.Read();
+        }
+
+        return TokenizeWord(textReader, textLanguage);
+    }
+
+    public static string TokenizerTerms(string query, string language)
+    {
+        var stopWords =
+            string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ?
+            null :
+            StopWords.GetValueOrDefault(language) ??
+            StopWords["en"];
+
+        return Tokenize(new StringReader(query), stopWords);
+    }
+
+    private static string TokenizeWord(TextReader reader, string language)
+    {
+        var stopWords =
+            string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ?
+            null :
+            StopWords.GetValueOrDefault(language) ??
+            StopWords["en"];
+
+        return Tokenize(reader, stopWords);
+    }
+
+    private static string Tokenize(TextReader reader, CharArraySet? stopWords)
+    {
+        var tokenizer = new StandardTokenizer(Version, reader);
+        var tokenStream = (TokenStream)new StandardFilter(Version, tokenizer);
+
+        // Stop words are case sensitive, therefore we have to lowercase it first.
+        tokenStream = new LowerCaseFilter(Version, tokenStream);
+
+        if (stopWords != null)
+        {
+            tokenStream = new StopFilter(Version, tokenStream, stopWords);
+        }
+
+        var sb = DefaultPools.StringBuilder.Get();
+        try
+        {
+            // Actually not idea what this is doing, but it seems to work.
+            var attribute = tokenStream.AddAttribute<ICharTermAttribute>();
+
+            tokenStream.Reset();
+
+            using (tokenStream)
+            {
+                while (tokenStream.IncrementToken())
+                {
+                    var text = attribute.ToString();
+
+                    if (sb.Length > 0)
+                    {
+                        sb.Append(' ');
+                    }
+
+                    sb.Append(text);
+                }
+            }
+
+            return sb.ToString();
+        }
+        finally
+        {
+            DefaultPools.StringBuilder.Return(sb);
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities/Assets/AssetsFluidExtension.cs

@@ -8,79 +8,62 @@
 using System.Text.Encodings.Web;
 using Fluid;
 using Fluid.Ast;
-using Fluid.Tags;
 using Fluid.Values;
 using Microsoft.Extensions.DependencyInjection;
 using Squidex.Assets;
 using Squidex.Domain.Apps.Core.Assets;
 using Squidex.Domain.Apps.Core.Rules.EnrichedEvents;
 using Squidex.Domain.Apps.Core.Templates;
-using Squidex.Domain.Apps.Core.ValidateContent;
 using Squidex.Infrastructure;
+using static Parlot.Fluent.Parsers;
 
 namespace Squidex.Domain.Apps.Entities.Assets;
 
 public sealed class AssetsFluidExtension : IFluidExtension
 {
-    private static readonly FluidValue ErrorNullAsset = FluidValue.Create(null);
     private static readonly FluidValue ErrorNoAsset = new StringValue("NoAsset");
     private static readonly FluidValue ErrorNoImage = new StringValue("NoImage");
     private static readonly FluidValue ErrorTooBig = new StringValue("ErrorTooBig");
     private readonly IServiceProvider serviceProvider;
 
-    private sealed class AssetTag : ArgumentsTag
+    public AssetsFluidExtension(IServiceProvider serviceProvider)
     {
-        private readonly IServiceProvider serviceProvider;
+        this.serviceProvider = serviceProvider;
+    }
 
-        public AssetTag(IServiceProvider serviceProvider)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
-        {
+    {
-            this.serviceProvider = serviceProvider;
+        AddAssetFilter(options);
-        }
+        AddAssetTextFilter(options);
 
-        public override async ValueTask<Completion> WriteToAsync(TextWriter writer,
+        parser.RegisterParserTag("asset", 
-            TextEncoder encoder, TemplateContext context, FilterArgument[] arguments)
+            parser.PrimaryParser.AndSkip(ZeroOrOne(parser.CommaParser)).And(parser.PrimaryParser),
+            ResolveAsset);
+    }
+
+    private async ValueTask<Completion> ResolveAsset(ValueTuple<Expression, Expression> arguments, TextWriter writer, TextEncoder encoder, TemplateContext context)
+    {
+        if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
         {
-            if (arguments.Length == 2 && context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
+            var (nameArg, idArg) = arguments;
-            {
-                var id = await arguments[1].Expression.EvaluateAsync(context);
 
-                var asset = await ResolveAssetAsync(serviceProvider, enrichedEvent.AppId.Id, id);
+            var assetId = await idArg.EvaluateAsync(context);
+            var asset = await ResolveAssetAsync(serviceProvider, enrichedEvent.AppId.Id, assetId);
 
-                if (asset != null)
+            if (asset != null)
-                {
+            {
-                    var name = (await arguments[0].Expression.EvaluateAsync(context)).ToStringValue();
+                var name = (await nameArg.EvaluateAsync(context)).ToStringValue();
 
-                    context.SetValue(name, asset);
+                context.SetValue(name, asset);
-                }
             }
-
-            return Completion.Normal;
         }
-    }
-
-    public AssetsFluidExtension(IServiceProvider serviceProvider)
-    {
-        this.serviceProvider = serviceProvider;
-    }
 
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+        return Completion.Normal;
-    {
-        memberAccessStrategy.Register<IAssetEntity>();
-        memberAccessStrategy.Register<IAssetInfo>();
-        memberAccessStrategy.Register<IWithId<DomainId>>();
-        memberAccessStrategy.Register<IEntity>();
-        memberAccessStrategy.Register<IEntityWithCreatedBy>();
-        memberAccessStrategy.Register<IEntityWithLastModifiedBy>();
-        memberAccessStrategy.Register<IEntityWithVersion>();
-        memberAccessStrategy.Register<IEnrichedAssetEntity>();
-
-        AddAssetFilter();
-        AddAssetTextFilter();
     }
 
-    private void AddAssetFilter()
+    private void AddAssetFilter(TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddAsyncFilter("asset", async (input, arguments, context) =>
+        options.Filters.AddFilter("asset", async (input, arguments, context) =>
         {
             if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
             {
@@ -88,19 +71,19 @@ public sealed class AssetsFluidExtension : IFluidExtension
 
                 if (asset == null)
                 {
-                    return ErrorNullAsset;
+                    return NilValue.Instance;
                 }
 
-                return FluidValue.Create(asset);
+                return FluidValue.Create(asset, options);
             }
 
-            return ErrorNullAsset;
+            return NilValue.Instance;
         });
     }
 
-    private void AddAssetTextFilter()
+    private void AddAssetTextFilter(TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddAsyncFilter("assetText", async (input, arguments, context) =>
+        options.Filters.AddFilter("assetText", async (input, arguments, context) =>
         {
             if (input is not ObjectValue objectValue)
             {
@@ -134,7 +117,7 @@ public sealed class AssetsFluidExtension : IFluidExtension
             return ErrorNoAsset;
         });
 
-        TemplateContext.GlobalFilters.AddAsyncFilter("assetBlurHash", async (input, arguments, context) =>
+        options.Filters.AddFilter("assetBlurHash", async (input, arguments, context) =>
         {
             if (input is not ObjectValue objectValue)
             {
@@ -189,11 +172,6 @@ public sealed class AssetsFluidExtension : IFluidExtension
         });
     }
 
-    public void RegisterLanguageExtensions(FluidParserFactory factory)
-    {
-        factory.RegisterTag("asset", new AssetTag(serviceProvider));
-    }
-
     private static async Task<IAssetEntity?> ResolveAssetAsync(IServiceProvider serviceProvider, DomainId appId, FluidValue id)
     {
         var appProvider = serviceProvider.GetRequiredService<IAppProvider>();

backend/src/Squidex.Domain.Apps.Entities/Contents/ReferencesFluidExtension.cs

@@ -8,12 +8,13 @@
 using System.Text.Encodings.Web;
 using Fluid;
 using Fluid.Ast;
-using Fluid.Tags;
 using Fluid.Values;
 using Microsoft.Extensions.DependencyInjection;
 using Squidex.Domain.Apps.Core.Rules.EnrichedEvents;
 using Squidex.Domain.Apps.Core.Templates;
+using Squidex.Domain.Apps.Entities.Contents.Queries.Steps;
 using Squidex.Infrastructure;
+using static Parlot.Fluent.Parsers;
 
 #pragma warning disable CA1826 // Do not use Enumerable methods on indexable collections
 
@@ -21,60 +22,45 @@ namespace Squidex.Domain.Apps.Entities.Contents;
 
 public sealed class ReferencesFluidExtension : IFluidExtension
 {
-    private static readonly FluidValue ErrorNullReference = FluidValue.Create(null);
     private readonly IServiceProvider serviceProvider;
 
-    private sealed class ReferenceTag : ArgumentsTag
+    public ReferencesFluidExtension(IServiceProvider serviceProvider)
     {
-        private readonly IServiceProvider serviceProvider;
+        this.serviceProvider = serviceProvider;
+    }
 
-        public ReferenceTag(IServiceProvider serviceProvider)
+    public void RegisterLanguageExtensions(CustomFluidParser parser, TemplateOptions options)
-        {
+    {
-            this.serviceProvider = serviceProvider;
+        AddReferenceFilter(options);
-        }
 
-        public override async ValueTask<Completion> WriteToAsync(TextWriter writer,
+        parser.RegisterParserTag("reference",
-            TextEncoder encoder, TemplateContext context, FilterArgument[] arguments)
+            parser.PrimaryParser.AndSkip(ZeroOrOne(parser.CommaParser)).And(parser.PrimaryParser), 
+            ResolveReference);
+    }
+
+    private async ValueTask<Completion> ResolveReference(ValueTuple<Expression, Expression> arguments, TextWriter writer, TextEncoder encoder, TemplateContext context)
+    {
+        if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
         {
-            if (arguments.Length == 2 && context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
+            var (nameArg, idArg) = arguments;
-            {
-                var id = await arguments[1].Expression.EvaluateAsync(context);
 
-                var content = await ResolveContentAsync(serviceProvider, enrichedEvent.AppId.Id, id);
+            var contentId = await idArg.EvaluateAsync(context);
+            var content = await ResolveContentAsync(serviceProvider, enrichedEvent.AppId.Id, contentId);
 
-                if (content != null)
+            if (content != null)
-                {
+            {
-                    var name = (await arguments[0].Expression.EvaluateAsync(context)).ToStringValue();
+                var name = (await nameArg.EvaluateAsync(context)).ToStringValue();
 
-                    context.SetValue(name, content);
+                context.SetValue(name, content);
-                }
             }
-
-            return Completion.Normal;
         }
-    }
-
-    public ReferencesFluidExtension(IServiceProvider serviceProvider)
-    {
-        this.serviceProvider = serviceProvider;
-    }
 
-    public void RegisterGlobalTypes(IMemberAccessStrategy memberAccessStrategy)
+        return Completion.Normal;
-    {
-        memberAccessStrategy.Register<IContentEntity>();
-        memberAccessStrategy.Register<IWithId<DomainId>>();
-        memberAccessStrategy.Register<IEntity>();
-        memberAccessStrategy.Register<IEntityWithCreatedBy>();
-        memberAccessStrategy.Register<IEntityWithLastModifiedBy>();
-        memberAccessStrategy.Register<IEntityWithVersion>();
-        memberAccessStrategy.Register<IEnrichedContentEntity>();
-
-        AddReferenceFilter();
     }
 
-    private void AddReferenceFilter()
+    private void AddReferenceFilter(TemplateOptions options)
     {
-        TemplateContext.GlobalFilters.AddAsyncFilter("reference", async (input, arguments, context) =>
+        options.Filters.AddFilter("reference", async (input, arguments, context) =>
         {
             if (context.GetValue("event")?.ToObjectValue() is EnrichedEvent enrichedEvent)
             {
@@ -82,21 +68,16 @@ public sealed class ReferencesFluidExtension : IFluidExtension
 
                 if (content == null)
                 {
-                    return ErrorNullReference;
+                    return NilValue.Instance;
                 }
 
-                return FluidValue.Create(content);
+                return FluidValue.Create(content, options);
             }
 
-            return ErrorNullReference;
+            return NilValue.Instance;
         });
     }
 
-    public void RegisterLanguageExtensions(FluidParserFactory factory)
-    {
-        factory.RegisterTag("reference", new ReferenceTag(serviceProvider));
-    }
-
     private static async Task<IContentEntity?> ResolveContentAsync(IServiceProvider serviceProvider, DomainId appId, FluidValue id)
     {
         var appProvider = serviceProvider.GetRequiredService<IAppProvider>();

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetsFluidExtensionTests.cs

@@ -76,8 +76,6 @@ public class AssetsFluidExtensionTests
     [Fact]
     public async Task Should_resolve_assets_in_loop()
     {
-        var (vars, assets) = SetupAssetsVars();
-
         var template = @"
                 {% for id in event.data.assets.iv %}
                     {% asset 'ref', id %}
@@ -85,21 +83,25 @@ public class AssetsFluidExtensionTests
                 {% endfor %}
             ";
 
-        var expected = $@"
+        await ResolveAssetsAsync(template);
-                Text: {assets[0].FileName} {assets[0].Id}
+    }
-                Text: {assets[1].FileName} {assets[1].Id}
-            ";
 
-        var actual = await sut.RenderAsync(template, vars);
+    [Fact]
+    public async Task Should_resolve_assets_in_loop_without_commata()
+    {
+        var template = @"
+                {% for id in event.data.assets.iv %}
+                    {% asset 'ref' id %}
+                    Text: {{ ref.fileName }} {{ ref.id }}
+                {% endfor %}
+            ";
 
-        Assert.Equal(Cleanup(expected), Cleanup(actual));
+        await ResolveAssetsAsync(template);
     }
 
     [Fact]
     public async Task Should_resolve_assets_in_loop_with_filter()
     {
-        var (vars, assets) = SetupAssetsVars();
-
         var template = @"
                 {% for id in event.data.assets.iv %}
                     {% assign ref = id | asset %}
@@ -107,6 +109,13 @@ public class AssetsFluidExtensionTests
                 {% endfor %}
             ";
 
+        await ResolveAssetsAsync(template);
+    }
+
+    private async Task ResolveAssetsAsync(string template)
+    {
+        var (vars, assets) = SetupAssetsVars();
+
         var expected = $@"
                 Text: {assets[0].FileName} {assets[0].Id}
                 Text: {assets[1].FileName} {assets[1].Id}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs

@@ -0,0 +1,63 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Domain.Apps.Entities.MongoDb.Text;
+
+namespace Squidex.Domain.Apps.Entities.Contents.MongoDb;
+
+public class TokenizerTests
+{
+    [Fact]
+    public void Should_eliminate_stop_words_in_implicit_english()
+    {
+        var source = "The only thing that matters, is time.";
+
+        var parsed = Tokenizer.TokenizeQuery(source);
+
+        Assert.Equal("only thing matters time", parsed);
+    }
+
+    [Fact]
+    public void Should_eliminate_stop_words_in_explicit_english()
+    {
+        var source = "en:The only thing that matters, is time.";
+
+        var parsed = Tokenizer.TokenizeQuery(source);
+
+        Assert.Equal("only thing matters time", parsed);
+    }
+
+    [Fact]
+    public void Should_eliminate_stop_words_in_explicit_english2()
+    {
+        var source = "en:when i do this it is pretty slow";
+
+        var parsed = Tokenizer.TokenizeQuery(source);
+
+        Assert.Equal("when i do pretty slow", parsed);
+    }
+
+    [Fact]
+    public void Should_not_eliminate_stop_words_for_iv_language()
+    {
+        var source = "iv:The only thing that matters, is time.";
+
+        var parsed = Tokenizer.TokenizeQuery(source);
+
+        Assert.Equal("the only thing that matters is time", parsed);
+    }
+
+    [Fact]
+    public void Should_eliminate_stop_words_in_explicit_german()
+    {
+        var source = "de:Nur die Zeit spielt eine Rolle";
+
+        var parsed = Tokenizer.TokenizeQuery(source);
+
+        Assert.Equal("zeit spielt rolle", parsed);
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Queries/ContentQueryServiceTests.cs

@@ -54,7 +54,7 @@ public class ContentQueryServiceTests
         A.CallTo(() => appProvider.GetSchemasAsync(appId.Id, ct))
             .Returns(new List<ISchemaEntity> { schema });
 
-        A.CallTo(() => queryParser.ParseAsync(A<Context>._, A<Q>._, A<ISchemaEntity?>._))
+        A.CallTo(() => queryParser.ParseAsync(A<Context>._, A<Q>._, A<ISchemaEntity?>._, ct))
             .ReturnsLazily(c => Task.FromResult(c.GetArgument<Q>(1)!));
 
         var options = Options.Create(new ContentOptions());

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ReferencesFluidExtensionTests.cs

@@ -46,32 +46,6 @@ public class ReferencesFluidExtensionTests
     [Fact]
     public async Task Should_resolve_references_in_loop()
     {
-        var referenceId1 = DomainId.NewGuid();
-        var reference1 = CreateReference(referenceId1, 1);
-        var referenceId2 = DomainId.NewGuid();
-        var reference2 = CreateReference(referenceId2, 2);
-
-        var @event = new EnrichedContentEvent
-        {
-            Data =
-                new ContentData()
-                    .AddField("references",
-                        new ContentFieldData()
-                            .AddInvariant(JsonValue.Array(referenceId1, referenceId2))),
-            AppId = appId
-        };
-
-        A.CallTo(() => contentQuery.QueryAsync(A<Context>._, A<Q>.That.HasIds(referenceId1), A<CancellationToken>._))
-            .Returns(ResultList.CreateFrom(1, reference1));
-
-        A.CallTo(() => contentQuery.QueryAsync(A<Context>._, A<Q>.That.HasIds(referenceId2), A<CancellationToken>._))
-            .Returns(ResultList.CreateFrom(1, reference2));
-
-        var vars = new TemplateVars
-        {
-            ["event"] = @event
-        };
-
         var template = @"
                 {% for id in event.data.references.iv %}
                     {% reference 'ref', id %}
@@ -79,18 +53,36 @@ public class ReferencesFluidExtensionTests
                 {% endfor %}
             ";
 
-        var expected = $@"
+        await ResolveReferencesAsync(template);
-                Text: Hello 1 World 1 {referenceId1}
+    }
-                Text: Hello 2 World 2 {referenceId2}
-            ";
 
-        var actual = await sut.RenderAsync(template, vars);
+    [Fact]
+    public async Task Should_resolve_references_in_loop_without_commata()
+    {
+        var template = @"
+                {% for id in event.data.references.iv %}
+                    {% reference 'ref' id %}
+                    Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }}
+                {% endfor %}
+            ";
 
-        Assert.Equal(Cleanup(expected), Cleanup(actual));
+        await ResolveReferencesAsync(template);
     }
 
     [Fact]
     public async Task Should_resolve_references_in_loop_with_filter()
+    {
+        var template = @"
+                {% for id in event.data.references.iv %}
+                    {% assign ref = id | reference %}
+                    Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }}
+                {% endfor %}
+            ";
+
+        await ResolveReferencesAsync(template);
+    }
+
+    private async Task ResolveReferencesAsync(string template)
     {
         var referenceId1 = DomainId.NewGuid();
         var reference1 = CreateReference(referenceId1, 1);
@@ -118,13 +110,6 @@ public class ReferencesFluidExtensionTests
             ["event"] = @event
         };
 
-        var template = @"
-                {% for id in event.data.references.iv %}
-                    {% assign ref = id | reference %}
-                    Text: {{ ref.data.field1.iv }} {{ ref.data.field2.iv }} {{ ref.id }}
-                {% endfor %}
-            ";
-
         var expected = $@"
                 Text: Hello 1 World 1 {referenceId1}
                 Text: Hello 2 World 2 {referenceId2}