Collation (#1139)

* Support for collation.

* Add to GraphQL.

backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.Designer.cs

@@ -843,6 +843,15 @@ namespace Squidex.Domain.Apps.Core {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to The collation or language code that should be used to compare strings, e.g. for sorting..
+        /// </summary>
+        public static string QueryCollation {
+            get {
+                return ResourceManager.GetString("QueryCollation", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to Optional OData filter..
         /// </summary>
@@ -879,6 +888,15 @@ namespace Squidex.Domain.Apps.Core {
             }
         }
         
+        /// <summary>
+        ///   Looks up a localized string similar to Picks N random of elements from the schema..
+        /// </summary>
+        public static string QueryRandom {
+            get {
+                return ResourceManager.GetString("QueryRandom", resourceCulture);
+            }
+        }
+        
         /// <summary>
         ///   Looks up a localized string similar to Optional OData full text search..
         /// </summary>

backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.resx

@@ -378,6 +378,9 @@
   <data name="Operation" xml:space="preserve">
     <value>The current operation.</value>
   </data>
+  <data name="QueryCollation" xml:space="preserve">
+    <value>The collation or language code that should be used to compare strings, e.g. for sorting.</value>
+  </data>
   <data name="QueryFilter" xml:space="preserve">
     <value>Optional OData filter.</value>
   </data>
@@ -390,6 +393,9 @@
   <data name="QueryQ" xml:space="preserve">
     <value>JSON query as well formatted json string. Overrides all other query parameters, except 'ids'.</value>
   </data>
+  <data name="QueryRandom" xml:space="preserve">
+    <value>Picks N random of elements from the schema.</value>
+  </data>
   <data name="QuerySearch" xml:space="preserve">
     <value>Optional OData full text search.</value>
   </data>

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/Extensions.cs

@@ -86,6 +86,13 @@ public static class Extensions
     public static async Task<List<MongoContentEntity>> QueryContentsAsync(this IMongoCollection<MongoContentEntity> collection, FilterDefinition<MongoContentEntity> filter, ClrQuery query, Q q,
         CancellationToken ct)
     {
+        Collation? collation = null;
+
+        if (query.Collation != null)
+        {
+            collation = new Collation(query.Collation);
+        }
+
         if (query.Skip > 0 && !query.IsSatisfiedByIndex())
         {
             // If we have to skip over items, we could reach the limit of the sort buffer, therefore get the ids and all filter fields only
@@ -97,10 +104,15 @@ public static class Extensions
                 projection = projection.Include(field);
             }
 
+            var aggregateOptions = new AggregateOptions
+            {
+                Collation = collation,
+            };
+
             if (query.Random > 0)
             {
                 var ids =
-                    await collection.Aggregate()
+                    await collection.Aggregate(aggregateOptions)
                         .Match(filter)
                         .Project<IdOnly>(projection)
                         .QuerySort(query)
@@ -118,7 +130,7 @@ public static class Extensions
             }
 
             var joined =
-                await collection.Aggregate()
+                await collection.Aggregate(aggregateOptions)
                     .Match(filter)
                     .Project<IdOnly>(projection)
                     .QuerySort(query)
@@ -138,8 +150,13 @@ public static class Extensions
             return joined.Select(x => x.Joined[0]).ToList();
         }
 
+        var findOptions = new FindOptions
+        {
+            Collation = collation,
+        };
+
         var result =
-            collection.Find(filter)
+            collection.Find(filter, findOptions)
                 .QuerySort(query)
                 .QuerySkip(query)
                 .QueryLimit(query)

backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/Contents/ContentActions.cs

@@ -185,6 +185,18 @@ internal static class ContentActions
                 Description = FieldDescriptions.QuerySearch,
                 DefaultValue = null
             },
+            new QueryArgument(Scalars.String)
+            {
+                Name = "collation",
+                Description = FieldDescriptions.QueryCollation,
+                DefaultValue = null
+            },
+            new QueryArgument(Scalars.Int)
+            {
+                Name = "random",
+                Description = FieldDescriptions.QueryRandom,
+                DefaultValue = null
+            },
         ];
 
         public static readonly IFieldResolver Query = Resolvers.Async<object, object>(async (_, fieldContext, context) =>

backend/src/Squidex.Infrastructure/Queries/OData/EdmModelExtensions.cs

@@ -70,6 +70,7 @@ public static class EdmModelExtensions
             parser.ParseFilter(query);
             parser.ParseSort(query);
             parser.ParseRandom(query);
+            parser.ParseCollation(query);
         }
 
         return query;

backend/src/Squidex.Infrastructure/Queries/OData/FilterBuilder.cs

@@ -1,4 +1,4 @@
-// ==========================================================================
+// ==========================================================================
 //  Squidex Headless CMS
 // ==========================================================================
 //  Copyright (c) Squidex UG (haftungsbeschraenkt)

backend/src/Squidex.Infrastructure/Queries/OData/LimitExtensions.cs

@@ -47,4 +47,16 @@ public static class LimitExtensions
             result.Random = random;
         }
     }
+
+    public static void ParseCollation(this ODataUriParser query, ClrQuery result)
+    {
+        var customQueries = query.CustomQueryOptions;
+
+        var randomQuery = customQueries.FirstOrDefault(x =>
+            string.Equals(x.Key, "collation", StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(x.Key, "collate", StringComparison.OrdinalIgnoreCase) ||
+            string.Equals(x.Key, "$collation", StringComparison.OrdinalIgnoreCase));
+
+        result.Collation = randomQuery.Value;
+    }
 }

backend/src/Squidex.Infrastructure/Queries/Query.cs

@@ -15,6 +15,8 @@ public class Query<TValue>
 
     public string? FullText { get; set; }
 
+    public string? Collation { get; set; }
+
     public long Skip { get; set; }
 
     public long Take { get; set; } = long.MaxValue;
@@ -61,6 +63,12 @@ public class Query<TValue>
             sb.Append($"FullText: '{FullText.Replace('\'', '\'')}'");
         }
 
+        if (Collation != null)
+        {
+            sb.AppendIfNotEmpty("; ");
+            sb.Append($"Collation: '{Collation}'");
+        }
+
         if (Skip > 0)
         {
             sb.AppendIfNotEmpty("; ");

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLQueriesTests.cs

@@ -51,13 +51,13 @@ public class GraphQLQueriesTests : GraphQLTestBase
     }
 
     [Fact]
-    public async Task Should_query_contents_with_full_text()
+    public async Task Should_query_contents_with_full_text_and_collation()
     {
         var contentId = DomainId.NewGuid();
         var content = TestContent.Create(contentId);
 
         A.CallTo(() => contentQuery.QueryAsync(MatchsContentContext(), content.SchemaId.Id.ToString(),
-                A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$search=\"Hello\"" && x.NoTotal),
+                A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$search=\"Hello\"&$collation=tr" && x.NoTotal),
                 A<CancellationToken>._))
             .Returns(ResultList.CreateFrom(0, content));
 
@@ -65,7 +65,46 @@ public class GraphQLQueriesTests : GraphQLTestBase
         {
             Query = @"
                 query {
-                  queryMySchemaContents(search: 'Hello') {
+                  queryMySchemaContents(search: 'Hello', collation: 'tr') {
+                    {fields}
+                  }
+                }",
+            Args = new
+            {
+                fields = TestContent.AllFlatFields
+            }
+        });
+
+        var expected = new
+        {
+            data = new
+            {
+                queryMySchemaContents = new[]
+                {
+                    TestContent.FlatResponse(content)
+                }
+            }
+        };
+
+        AssertResult(expected, actual);
+    }
+
+    [Fact]
+    public async Task Should_query_contents_with_random()
+    {
+        var contentId = DomainId.NewGuid();
+        var content = TestContent.Create(contentId);
+
+        A.CallTo(() => contentQuery.QueryAsync(MatchsContentContext(), content.SchemaId.Id.ToString(),
+                A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$random=42" && x.NoTotal),
+                A<CancellationToken>._))
+            .Returns(ResultList.CreateFrom(0, content));
+
+        var actual = await ExecuteAsync(new TestQuery
+        {
+            Query = @"
+                query {
+                  queryMySchemaContents(random: 42) {
                     {fields}
                   }
                 }",

backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromJsonTests.cs

@@ -590,6 +590,14 @@ public sealed class QueryFromJsonTests
         AssertQuery(json, "FullText: 'Hello'");
     }
 
+    [Fact]
+    public void Should_parse_collation()
+    {
+        var json = new { Collation = "Collation" };
+
+        AssertQuery(json, "Collation: 'Collation'");
+    }
+
     [Fact]
     public void Should_parse_sort()
     {

backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromODataTests.cs

@@ -449,6 +449,24 @@ public class QueryFromODataTests
         Assert.Equal(o, i);
     }
 
+    [Fact]
+    public void Should_parse_collation1()
+    {
+        var i = _Q("collation=Collation");
+        var o = _C("Collation: 'Collation'");
+
+        Assert.Equal(o, i);
+    }
+
+    [Fact]
+    public void Should_parse_collation2()
+    {
+        var i = _Q("$collation=Collation");
+        var o = _C("Collation: 'Collation'");
+
+        Assert.Equal(o, i);
+    }
+
     [Fact]
     public void Should_text_and_multiple_terms()
     {

tools/TestSuite/TestSuite.ApiTests/AssetTests.cs

@@ -441,7 +441,7 @@ public class AssetTests : IClassFixture<CreatedAppFixture>
         {
             var downloaded = new MemoryStream();
 
-            using (var assetStream = await _.Client.Assets.GetAssetContentBySlugAsync(asset_2.Id, string.Empty))
+            using (var assetStream = await _.Client.Assets.GetAssetContentBySlugAsync(asset_2.Id))
             {
                 await assetStream.Stream.CopyToAsync(downloaded);
             }
@@ -796,7 +796,7 @@ public class AssetTests : IClassFixture<CreatedAppFixture>
 
         foreach (var asset in assets.Items)
         {
-            var content = await client.Assets.GetAssetContentBySlugAsync(asset.Id, string.Empty, deleted: true);
+            var content = await client.Assets.GetAssetContentBySlugAsync(asset.Id, deleted: true);
 
             await client.Assets.PostAssetAsync(id: asset.Id, file: new FileParameter(content.Stream, asset.FileName, asset.MimeType));
         }

tools/TestSuite/TestSuite.ApiTests/ContentCollationTests.cs

@@ -0,0 +1,93 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Newtonsoft.Json;
+using Squidex.ClientLibrary;
+using TestSuite.Fixtures;
+using TestSuite.Model;
+
+namespace TestSuite.ApiTests;
+
+#pragma warning disable SA1300 // Element should begin with upper-case letter
+#pragma warning disable SA1507 // Code should not contain multiple blank lines in a row
+
+public class ContentCollationTests : IClassFixture<CreatedAppFixture>
+{
+    private readonly string schemaName = $"schema-{Guid.NewGuid()}";
+
+    public CreatedAppFixture _ { get; }
+
+    public ContentCollationTests(CreatedAppFixture fixture)
+    {
+        _ = fixture;
+    }
+
+    public sealed class SimpleEntityData
+    {
+        public static readonly string StringField = nameof(String).ToLowerInvariant();
+
+        [JsonConverter(typeof(InvariantConverter))]
+        public string? String { get; set; }
+    }
+
+    public sealed class SimpleEntity : Content<SimpleEntityData>
+    {
+    }
+
+    [Fact]
+    public async Task Should_search_based_on_collation()
+    {
+        // STEP 0: Create schema.
+        var schemaRequest = new CreateSchemaDto
+        {
+            Name = schemaName,
+            Fields =
+            [
+                new UpsertSchemaFieldDto
+                {
+                    Name = SimpleEntityData.StringField,
+                    Properties = new StringFieldPropertiesDto()
+                },
+            ],
+            IsPublished = true
+        };
+
+        await _.Client.Schemas.PostSchemaAsync(schemaRequest);
+
+        // STEP 1: Create content.
+        var contents = _.Client.Contents<SimpleEntity, SimpleEntityData>(schemaName);
+
+        await contents.CreateAsync(new SimpleEntityData
+        {
+            String = "İstanbul"
+        }, ContentCreateOptions.AsPublish);
+
+        await contents.CreateAsync(new SimpleEntityData
+        {
+            String = "Mersin"
+        }, ContentCreateOptions.AsPublish);
+
+        await contents.CreateAsync(new SimpleEntityData
+        {
+            String = "Lüleburgaz"
+        }, ContentCreateOptions.AsPublish);
+
+
+        // STEP 2: Get sorted contents.
+        var sorted_1 = await contents.GetAsync(new ContentQuery { OrderBy = $"data/{TestEntityData.StringField}/iv asc" });
+        var sortedNames_1 = sorted_1.Items.Select(x => x.Data.String).ToList();
+
+        Assert.Equal(new string[] { "Lüleburgaz", "Mersin", "İstanbul" }, sortedNames_1);
+
+
+        // STEP 3: Get with collation.
+        var sorted_2 = await contents.GetAsync(new ContentQuery { OrderBy = $"data/{TestEntityData.StringField}/iv asc", Collation = "tr" });
+        var sortedNames_2 = sorted_2.Items.Select(x => x.Data.String).ToList();
+
+        Assert.Equal(new string[] { "İstanbul", "Lüleburgaz", "Mersin" }, sortedNames_2);
+    }
+}

tools/TestSuite/TestSuite.ApiTests/TestSuite.ApiTests.csproj

@@ -7,20 +7,20 @@
 	<Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="FluentAssertions" Version="6.12.0" />
+    <PackageReference Include="FluentAssertions" Version="6.12.2" />
     <PackageReference Include="GraphQL.Client" Version="6.1.0" />
     <PackageReference Include="GraphQL.Client.Serializer.Newtonsoft" Version="6.1.0" />
-    <PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageReference Include="NSwag.Core" Version="14.1.0" />
-    <PackageReference Include="Squidex.Assets" Version="6.18.0" />
+    <PackageReference Include="Squidex.Assets" Version="6.19.0" />
-    <PackageReference Include="Squidex.Assets.ImageSharp" Version="6.18.0" />
+    <PackageReference Include="Squidex.Assets.ImageSharp" Version="6.19.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
-    <PackageReference Include="Verify.Xunit" Version="26.2.0" />
+    <PackageReference Include="Verify.Xunit" Version="28.3.1" />
-    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

tools/TestSuite/TestSuite.LoadTests/TestSuite.LoadTests.csproj

@@ -7,13 +7,13 @@
 	<Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
-    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj

@@ -7,22 +7,22 @@
 	<Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
+    <PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="9.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="9.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0" />
     <PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
-    <PackageReference Include="Squidex.Assets" Version="6.18.0" />
+    <PackageReference Include="Squidex.Assets" Version="6.19.0" />
-    <PackageReference Include="Squidex.ClientLibrary" Version="19.7.0" />
+    <PackageReference Include="Squidex.ClientLibrary" Version="20.1.0" />
-    <PackageReference Include="Squidex.ClientLibrary.ServiceExtensions" Version="19.7.0" />
+    <PackageReference Include="Squidex.ClientLibrary.ServiceExtensions" Version="20.1.0" />
     <PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
-    <PackageReference Include="Verify" Version="26.2.0" />
+    <PackageReference Include="Verify" Version="28.3.1" />
-    <PackageReference Include="xunit" Version="2.9.0" />
+    <PackageReference Include="xunit" Version="2.9.2" />
   </ItemGroup>
   <ItemGroup>
     <AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />