Browse Source

Collation (#1139)

* Support for collation.

* Add to GraphQL.
pull/1144/head
Sebastian Stehle 2 years ago
committed by GitHub
parent
commit
a49ef9d430
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 18
      backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.Designer.cs
  2. 6
      backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.resx
  3. 23
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/Extensions.cs
  4. 12
      backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/Contents/ContentActions.cs
  5. 1
      backend/src/Squidex.Infrastructure/Queries/OData/EdmModelExtensions.cs
  6. 2
      backend/src/Squidex.Infrastructure/Queries/OData/FilterBuilder.cs
  7. 12
      backend/src/Squidex.Infrastructure/Queries/OData/LimitExtensions.cs
  8. 8
      backend/src/Squidex.Infrastructure/Queries/Query.cs
  9. 45
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLQueriesTests.cs
  10. 8
      backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromJsonTests.cs
  11. 18
      backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromODataTests.cs
  12. 4
      tools/TestSuite/TestSuite.ApiTests/AssetTests.cs
  13. 93
      tools/TestSuite/TestSuite.ApiTests/ContentCollationTests.cs
  14. 14
      tools/TestSuite/TestSuite.ApiTests/TestSuite.ApiTests.csproj
  15. 6
      tools/TestSuite/TestSuite.LoadTests/TestSuite.LoadTests.csproj
  16. 22
      tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj

18
backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.Designer.cs

@ -843,6 +843,15 @@ namespace Squidex.Domain.Apps.Core {
}
}
/// <summary>
/// Looks up a localized string similar to The collation or language code that should be used to compare strings, e.g. for sorting..
/// </summary>
public static string QueryCollation {
get {
return ResourceManager.GetString("QueryCollation", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Optional OData filter..
/// </summary>
@ -879,6 +888,15 @@ namespace Squidex.Domain.Apps.Core {
}
}
/// <summary>
/// Looks up a localized string similar to Picks N random of elements from the schema..
/// </summary>
public static string QueryRandom {
get {
return ResourceManager.GetString("QueryRandom", resourceCulture);
}
}
/// <summary>
/// Looks up a localized string similar to Optional OData full text search..
/// </summary>

6
backend/src/Squidex.Domain.Apps.Core.Model/FieldDescriptions.resx

@ -378,6 +378,9 @@
<data name="Operation" xml:space="preserve">
<value>The current operation.</value>
</data>
<data name="QueryCollation" xml:space="preserve">
<value>The collation or language code that should be used to compare strings, e.g. for sorting.</value>
</data>
<data name="QueryFilter" xml:space="preserve">
<value>Optional OData filter.</value>
</data>
@ -390,6 +393,9 @@
<data name="QueryQ" xml:space="preserve">
<value>JSON query as well formatted json string. Overrides all other query parameters, except 'ids'.</value>
</data>
<data name="QueryRandom" xml:space="preserve">
<value>Picks N random of elements from the schema.</value>
</data>
<data name="QuerySearch" xml:space="preserve">
<value>Optional OData full text search.</value>
</data>

23
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/Extensions.cs

@ -86,6 +86,13 @@ public static class Extensions
public static async Task<List<MongoContentEntity>> QueryContentsAsync(this IMongoCollection<MongoContentEntity> collection, FilterDefinition<MongoContentEntity> filter, ClrQuery query, Q q,
CancellationToken ct)
{
Collation? collation = null;
if (query.Collation != null)
{
collation = new Collation(query.Collation);
}
if (query.Skip > 0 && !query.IsSatisfiedByIndex())
{
// If we have to skip over items, we could reach the limit of the sort buffer, therefore get the ids and all filter fields only
@ -97,10 +104,15 @@ public static class Extensions
projection = projection.Include(field);
}
var aggregateOptions = new AggregateOptions
{
Collation = collation,
};
if (query.Random > 0)
{
var ids =
await collection.Aggregate()
await collection.Aggregate(aggregateOptions)
.Match(filter)
.Project<IdOnly>(projection)
.QuerySort(query)
@ -118,7 +130,7 @@ public static class Extensions
}
var joined =
await collection.Aggregate()
await collection.Aggregate(aggregateOptions)
.Match(filter)
.Project<IdOnly>(projection)
.QuerySort(query)
@ -138,8 +150,13 @@ public static class Extensions
return joined.Select(x => x.Joined[0]).ToList();
}
var findOptions = new FindOptions
{
Collation = collation,
};
var result =
collection.Find(filter)
collection.Find(filter, findOptions)
.QuerySort(query)
.QuerySkip(query)
.QueryLimit(query)

12
backend/src/Squidex.Domain.Apps.Entities/Contents/GraphQL/Types/Contents/ContentActions.cs

@ -185,6 +185,18 @@ internal static class ContentActions
Description = FieldDescriptions.QuerySearch,
DefaultValue = null
},
new QueryArgument(Scalars.String)
{
Name = "collation",
Description = FieldDescriptions.QueryCollation,
DefaultValue = null
},
new QueryArgument(Scalars.Int)
{
Name = "random",
Description = FieldDescriptions.QueryRandom,
DefaultValue = null
},
];
public static readonly IFieldResolver Query = Resolvers.Async<object, object>(async (_, fieldContext, context) =>

1
backend/src/Squidex.Infrastructure/Queries/OData/EdmModelExtensions.cs

@ -70,6 +70,7 @@ public static class EdmModelExtensions
parser.ParseFilter(query);
parser.ParseSort(query);
parser.ParseRandom(query);
parser.ParseCollation(query);
}
return query;

2
backend/src/Squidex.Infrastructure/Queries/OData/FilterBuilder.cs

@ -1,4 +1,4 @@
// ==========================================================================
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)

12
backend/src/Squidex.Infrastructure/Queries/OData/LimitExtensions.cs

@ -47,4 +47,16 @@ public static class LimitExtensions
result.Random = random;
}
}
public static void ParseCollation(this ODataUriParser query, ClrQuery result)
{
var customQueries = query.CustomQueryOptions;
var randomQuery = customQueries.FirstOrDefault(x =>
string.Equals(x.Key, "collation", StringComparison.OrdinalIgnoreCase) ||
string.Equals(x.Key, "collate", StringComparison.OrdinalIgnoreCase) ||
string.Equals(x.Key, "$collation", StringComparison.OrdinalIgnoreCase));
result.Collation = randomQuery.Value;
}
}

8
backend/src/Squidex.Infrastructure/Queries/Query.cs

@ -15,6 +15,8 @@ public class Query<TValue>
public string? FullText { get; set; }
public string? Collation { get; set; }
public long Skip { get; set; }
public long Take { get; set; } = long.MaxValue;
@ -61,6 +63,12 @@ public class Query<TValue>
sb.Append($"FullText: '{FullText.Replace('\'', '\'')}'");
}
if (Collation != null)
{
sb.AppendIfNotEmpty("; ");
sb.Append($"Collation: '{Collation}'");
}
if (Skip > 0)
{
sb.AppendIfNotEmpty("; ");

45
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/GraphQL/GraphQLQueriesTests.cs

@ -51,13 +51,13 @@ public class GraphQLQueriesTests : GraphQLTestBase
}
[Fact]
public async Task Should_query_contents_with_full_text()
public async Task Should_query_contents_with_full_text_and_collation()
{
var contentId = DomainId.NewGuid();
var content = TestContent.Create(contentId);
A.CallTo(() => contentQuery.QueryAsync(MatchsContentContext(), content.SchemaId.Id.ToString(),
A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$search=\"Hello\"" && x.NoTotal),
A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$search=\"Hello\"&$collation=tr" && x.NoTotal),
A<CancellationToken>._))
.Returns(ResultList.CreateFrom(0, content));
@ -65,7 +65,46 @@ public class GraphQLQueriesTests : GraphQLTestBase
{
Query = @"
query {
queryMySchemaContents(search: 'Hello') {
queryMySchemaContents(search: 'Hello', collation: 'tr') {
{fields}
}
}",
Args = new
{
fields = TestContent.AllFlatFields
}
});
var expected = new
{
data = new
{
queryMySchemaContents = new[]
{
TestContent.FlatResponse(content)
}
}
};
AssertResult(expected, actual);
}
[Fact]
public async Task Should_query_contents_with_random()
{
var contentId = DomainId.NewGuid();
var content = TestContent.Create(contentId);
A.CallTo(() => contentQuery.QueryAsync(MatchsContentContext(), content.SchemaId.Id.ToString(),
A<Q>.That.Matches(x => x.QueryAsOdata == "?$skip=0&$random=42" && x.NoTotal),
A<CancellationToken>._))
.Returns(ResultList.CreateFrom(0, content));
var actual = await ExecuteAsync(new TestQuery
{
Query = @"
query {
queryMySchemaContents(random: 42) {
{fields}
}
}",

8
backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromJsonTests.cs

@ -590,6 +590,14 @@ public sealed class QueryFromJsonTests
AssertQuery(json, "FullText: 'Hello'");
}
[Fact]
public void Should_parse_collation()
{
var json = new { Collation = "Collation" };
AssertQuery(json, "Collation: 'Collation'");
}
[Fact]
public void Should_parse_sort()
{

18
backend/tests/Squidex.Infrastructure.Tests/Queries/QueryFromODataTests.cs

@ -449,6 +449,24 @@ public class QueryFromODataTests
Assert.Equal(o, i);
}
[Fact]
public void Should_parse_collation1()
{
var i = _Q("collation=Collation");
var o = _C("Collation: 'Collation'");
Assert.Equal(o, i);
}
[Fact]
public void Should_parse_collation2()
{
var i = _Q("$collation=Collation");
var o = _C("Collation: 'Collation'");
Assert.Equal(o, i);
}
[Fact]
public void Should_text_and_multiple_terms()
{

4
tools/TestSuite/TestSuite.ApiTests/AssetTests.cs

@ -441,7 +441,7 @@ public class AssetTests : IClassFixture<CreatedAppFixture>
{
var downloaded = new MemoryStream();
using (var assetStream = await _.Client.Assets.GetAssetContentBySlugAsync(asset_2.Id, string.Empty))
using (var assetStream = await _.Client.Assets.GetAssetContentBySlugAsync(asset_2.Id))
{
await assetStream.Stream.CopyToAsync(downloaded);
}
@ -796,7 +796,7 @@ public class AssetTests : IClassFixture<CreatedAppFixture>
foreach (var asset in assets.Items)
{
var content = await client.Assets.GetAssetContentBySlugAsync(asset.Id, string.Empty, deleted: true);
var content = await client.Assets.GetAssetContentBySlugAsync(asset.Id, deleted: true);
await client.Assets.PostAssetAsync(id: asset.Id, file: new FileParameter(content.Stream, asset.FileName, asset.MimeType));
}

93
tools/TestSuite/TestSuite.ApiTests/ContentCollationTests.cs

@ -0,0 +1,93 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using Newtonsoft.Json;
using Squidex.ClientLibrary;
using TestSuite.Fixtures;
using TestSuite.Model;
namespace TestSuite.ApiTests;
#pragma warning disable SA1300 // Element should begin with upper-case letter
#pragma warning disable SA1507 // Code should not contain multiple blank lines in a row
public class ContentCollationTests : IClassFixture<CreatedAppFixture>
{
private readonly string schemaName = $"schema-{Guid.NewGuid()}";
public CreatedAppFixture _ { get; }
public ContentCollationTests(CreatedAppFixture fixture)
{
_ = fixture;
}
public sealed class SimpleEntityData
{
public static readonly string StringField = nameof(String).ToLowerInvariant();
[JsonConverter(typeof(InvariantConverter))]
public string? String { get; set; }
}
public sealed class SimpleEntity : Content<SimpleEntityData>
{
}
[Fact]
public async Task Should_search_based_on_collation()
{
// STEP 0: Create schema.
var schemaRequest = new CreateSchemaDto
{
Name = schemaName,
Fields =
[
new UpsertSchemaFieldDto
{
Name = SimpleEntityData.StringField,
Properties = new StringFieldPropertiesDto()
},
],
IsPublished = true
};
await _.Client.Schemas.PostSchemaAsync(schemaRequest);
// STEP 1: Create content.
var contents = _.Client.Contents<SimpleEntity, SimpleEntityData>(schemaName);
await contents.CreateAsync(new SimpleEntityData
{
String = "İstanbul"
}, ContentCreateOptions.AsPublish);
await contents.CreateAsync(new SimpleEntityData
{
String = "Mersin"
}, ContentCreateOptions.AsPublish);
await contents.CreateAsync(new SimpleEntityData
{
String = "Lüleburgaz"
}, ContentCreateOptions.AsPublish);
// STEP 2: Get sorted contents.
var sorted_1 = await contents.GetAsync(new ContentQuery { OrderBy = $"data/{TestEntityData.StringField}/iv asc" });
var sortedNames_1 = sorted_1.Items.Select(x => x.Data.String).ToList();
Assert.Equal(new string[] { "Lüleburgaz", "Mersin", "İstanbul" }, sortedNames_1);
// STEP 3: Get with collation.
var sorted_2 = await contents.GetAsync(new ContentQuery { OrderBy = $"data/{TestEntityData.StringField}/iv asc", Collation = "tr" });
var sortedNames_2 = sorted_2.Items.Select(x => x.Data.String).ToList();
Assert.Equal(new string[] { "İstanbul", "Lüleburgaz", "Mersin" }, sortedNames_2);
}
}

14
tools/TestSuite/TestSuite.ApiTests/TestSuite.ApiTests.csproj

@ -7,20 +7,20 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.0" />
<PackageReference Include="FluentAssertions" Version="6.12.2" />
<PackageReference Include="GraphQL.Client" Version="6.1.0" />
<PackageReference Include="GraphQL.Client.Serializer.Newtonsoft" Version="6.1.0" />
<PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
<PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="NSwag.Core" Version="14.1.0" />
<PackageReference Include="Squidex.Assets" Version="6.18.0" />
<PackageReference Include="Squidex.Assets.ImageSharp" Version="6.18.0" />
<PackageReference Include="Squidex.Assets" Version="6.19.0" />
<PackageReference Include="Squidex.Assets.ImageSharp" Version="6.19.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="Verify.Xunit" Version="26.2.0" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="Verify.Xunit" Version="28.3.1" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

6
tools/TestSuite/TestSuite.LoadTests/TestSuite.LoadTests.csproj

@ -7,13 +7,13 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
<PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>

22
tools/TestSuite/TestSuite.Shared/TestSuite.Shared.csproj

@ -7,22 +7,22 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Meziantou.Analyzer" Version="2.0.163">
<PackageReference Include="Meziantou.Analyzer" Version="2.0.179">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="9.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="9.0.0" />
<PackageReference Include="RefactoringEssentials" Version="5.6.0" PrivateAssets="all" />
<PackageReference Include="Squidex.Assets" Version="6.18.0" />
<PackageReference Include="Squidex.ClientLibrary" Version="19.7.0" />
<PackageReference Include="Squidex.ClientLibrary.ServiceExtensions" Version="19.7.0" />
<PackageReference Include="Squidex.Assets" Version="6.19.0" />
<PackageReference Include="Squidex.ClientLibrary" Version="20.1.0" />
<PackageReference Include="Squidex.ClientLibrary.ServiceExtensions" Version="20.1.0" />
<PackageReference Include="StyleCop.Analyzers" Version="1.1.118" PrivateAssets="all" />
<PackageReference Include="Verify" Version="26.2.0" />
<PackageReference Include="xunit" Version="2.9.0" />
<PackageReference Include="Verify" Version="28.3.1" />
<PackageReference Include="xunit" Version="2.9.2" />
</ItemGroup>
<ItemGroup>
<AdditionalFiles Include="..\..\stylecop.json" Link="stylecop.json" />

Loading…
Cancel
Save