From 3534fe76089eb15eebbfcacba060dd0124fb2798 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Thu, 28 May 2020 18:26:58 +0200 Subject: [PATCH] Feature/content sorting (#523) * Improve content sorting using two stage aggregation framework. * Sorting improvements. --- .../Assets/MongoAssetRepository.cs | 11 +-- .../Operations/QueryContentsByQuery.cs | 72 +++++++++++++++---- .../MongoDb/Queries/LimitExtensions.cs | 25 +++++++ .../Queries/CompareFilter.cs | 7 ++ .../Queries/FilterNode.cs | 4 ++ .../Queries/LogicalFilter.cs | 8 +++ .../Queries/NegateFilter.cs | 7 ++ .../Squidex.Infrastructure/Queries/Query.cs | 17 +++++ .../Contents/MongoDb/ContentsQueryFixture.cs | 10 +-- .../Contents/MongoDb/ContentsQueryTests.cs | 50 ++++++++++--- .../Squidex.Domain.Apps.Entities.Tests.csproj | 1 + .../Queries/QueryTests.cs | 64 +++++++++++++++++ 12 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index 06c44509f..82578a00c 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -85,16 +85,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets return ResultList.Create(assetCount.Result, assetItems.Result); } - catch (MongoQueryException ex) + catch (MongoQueryException ex) when (ex.Message.Contains("17406")) { - if (ex.Message.Contains("17406")) - { - throw new DomainException("Result set is too large to be retrieved. Use $top parameter to reduce the number of items."); - } - else - { - throw; - } + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs index fcf3dec34..03e07343b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryContentsByQuery.cs @@ -7,8 +7,11 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; using MongoDB.Driver; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Contents; @@ -22,9 +25,21 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations { internal sealed class QueryContentsByQuery : OperationBase { + private static readonly PropertyPath DefaultOrderField = "mt"; private readonly DataConverter converter; private readonly ITextIndex indexer; + [BsonIgnoreExtraElements] + internal sealed class IdOnly + { + [BsonId] + [BsonElement("_id")] + [BsonRepresentation(BsonType.String)] + public Guid Id { get; set; } + + public MongoContentEntity[] Joined { get; set; } + } + public QueryContentsByQuery(DataConverter converter, ITextIndex indexer) { this.converter = converter; @@ -71,12 +86,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations var filter = CreateFilter(schema.Id, fullTextIds, query); var contentCount = Collection.Find(filter).CountDocumentsAsync(); - var contentItems = - Collection.Find(filter) - .QueryLimit(query) - .QuerySkip(query) - .QuerySort(query) - .ToListAsync(); + var contentItems = FindContentsAsync(query, filter); await Task.WhenAll(contentItems, contentCount); @@ -87,17 +97,53 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations return ResultList.Create(contentCount.Result, contentItems.Result); } - catch (MongoQueryException ex) + catch (MongoCommandException ex) when (ex.Code == 96) { - if (ex.Message.Contains("17406")) - { - throw new DomainException("Result set is too large to be retrieved. Use $top parameter to reduce the number of items."); - } - else + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); + } + catch (MongoQueryException ex) when (ex.Message.Contains("17406")) + { + throw new DomainException("Result set is too large to be retrieved. Use $take parameter to reduce the number of items."); + } + } + + private async Task> FindContentsAsync(ClrQuery query, FilterDefinition filter) + { + if (query.Skip > 0 && !IsSatisfiedByIndex(query)) + { + var projection = Projection.Include("_id"); + + foreach (var field in query.GetAllFields()) { - throw; + projection = Projection.Include(field); } + + var joined = + await Collection.Aggregate() + .Match(filter) + .Project(projection) + .QuerySort(query) + .QuerySkip(query) + .QueryLimit(query) + .Lookup(Collection, x => x.Id, x => x.Id, x => x.Joined) + .ToListAsync(); + + return joined.Select(x => x.Joined[0]).ToList(); } + + var result = + Collection.Find(filter) + .QuerySort(query) + .QueryLimit(query) + .QuerySkip(query) + .ToListAsync(); + + return await result; + } + + private static bool IsSatisfiedByIndex(ClrQuery query) + { + return query.Sort?.Any(x => x.Path == DefaultOrderField && x.Order == SortOrder.Descending) == true; } private static FilterDefinition CreateFilter(Guid schemaId, ICollection? ids, ClrQuery? query) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs index 355138c98..4cc07fe2d 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/Queries/LimitExtensions.cs @@ -12,6 +12,16 @@ namespace Squidex.Infrastructure.MongoDb.Queries { public static class LimitExtensions { + public static IAggregateFluent QueryLimit(this IAggregateFluent cursor, ClrQuery query) + { + if (query.Take < long.MaxValue) + { + cursor = cursor.Limit((int)query.Take); + } + + return cursor; + } + public static IFindFluent QueryLimit(this IFindFluent cursor, ClrQuery query) { if (query.Take < long.MaxValue) @@ -22,6 +32,16 @@ namespace Squidex.Infrastructure.MongoDb.Queries return cursor; } + public static IAggregateFluent QuerySkip(this IAggregateFluent cursor, ClrQuery query) + { + if (query.Skip > 0) + { + cursor = cursor.Skip((int)query.Skip); + } + + return cursor; + } + public static IFindFluent QuerySkip(this IFindFluent cursor, ClrQuery query) { if (query.Skip > 0) @@ -36,5 +56,10 @@ namespace Squidex.Infrastructure.MongoDb.Queries { return cursor.Sort(query.BuildSort()); } + + public static IAggregateFluent QuerySort(this IAggregateFluent cursor, ClrQuery query) + { + return cursor.Sort(query.BuildSort()); + } } } diff --git a/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs b/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs index 5b522e6c9..c33d039f2 100644 --- a/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/CompareFilter.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public sealed class CompareFilter : FilterNode @@ -28,6 +30,11 @@ namespace Squidex.Infrastructure.Queries Value = value; } + public override void AddFields(HashSet fields) + { + fields.Add(Path.ToString()); + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs b/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs index 125787f11..0ca210b6e 100644 --- a/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs +++ b/backend/src/Squidex.Infrastructure/Queries/FilterNode.cs @@ -5,12 +5,16 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public abstract class FilterNode { public abstract T Accept(FilterNodeVisitor visitor); + public abstract void AddFields(HashSet fields); + public abstract override string ToString(); } } diff --git a/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs b/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs index 1fc2a1416..f2a49a75e 100644 --- a/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/LogicalFilter.cs @@ -25,6 +25,14 @@ namespace Squidex.Infrastructure.Queries Type = type; } + public override void AddFields(HashSet fields) + { + foreach (var filter in Filters) + { + filter.AddFields(fields); + } + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs b/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs index 09583a0f8..83f04331a 100644 --- a/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs +++ b/backend/src/Squidex.Infrastructure/Queries/NegateFilter.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; + namespace Squidex.Infrastructure.Queries { public sealed class NegateFilter : FilterNode @@ -18,6 +20,11 @@ namespace Squidex.Infrastructure.Queries Filter = filter; } + public override void AddFields(HashSet fields) + { + Filter.AddFields(fields); + } + public override T Accept(FilterNodeVisitor visitor) { return visitor.Visit(this); diff --git a/backend/src/Squidex.Infrastructure/Queries/Query.cs b/backend/src/Squidex.Infrastructure/Queries/Query.cs index d9e814d10..136999e10 100644 --- a/backend/src/Squidex.Infrastructure/Queries/Query.cs +++ b/backend/src/Squidex.Infrastructure/Queries/Query.cs @@ -26,6 +26,23 @@ namespace Squidex.Infrastructure.Queries public List Sort { get; set; } = new List(); + public HashSet GetAllFields() + { + var result = new HashSet(); + + if (Sort != null) + { + foreach (var sorting in Sort) + { + result.Add(sorting.Path.ToString()); + } + } + + Filter?.AddFields(result); + + return result; + } + public override string ToString() { var parts = new List(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs index 6d58727a5..fc85d18db 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryFixture.cs @@ -9,6 +9,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using FakeItEasy; +using LoremNET; using MongoDB.Bson; using MongoDB.Driver; using Newtonsoft.Json; @@ -102,13 +103,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { for (var i = 0; i < numValues; i++) { - var value = i.ToString(); - var data = new IdContentData() .AddField(1, new ContentFieldData() - .AddJsonValue(JsonValue.Create(value))); + .AddJsonValue(JsonValue.Create(i))) + .AddField(2, + new ContentFieldData() + .AddJsonValue(JsonValue.Create(Lorem.Paragraph(200, 20)))); var content = new MongoContentEntity { @@ -198,7 +200,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { var schemaDef = new Schema("my-schema") - .AddField(Fields.String(1, "value", Partitioning.Invariant)); + .AddField(Fields.Number(1, "value", Partitioning.Invariant)); var schema = Mocks.Schema( diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs index 48bdb6981..abb9142a3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryTests.cs @@ -62,11 +62,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb [Fact] public async Task Should_query_contents_by_filter() { - var filter = F.Eq("data.value.iv", _.RandomValue()); + var filter = F.Eq("data.value.iv", 12); var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter); - Assert.NotNull(contents); + Assert.NotEmpty(contents); } [Fact] @@ -84,7 +84,23 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb var contents = await QueryAsync(query); - Assert.NotNull(contents); + Assert.NotEmpty(contents); + } + + [Fact] + public async Task Should_query_contents_with_large_skip() + { + var query = new ClrQuery + { + Sort = new List + { + new SortNode("data.value.iv", SortOrder.Ascending) + } + }; + + var contents = await QueryAsync(query, 1000, 9000); + + Assert.NotEmpty(contents); } [Fact] @@ -105,19 +121,33 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb { var query = new ClrQuery { - Filter = F.Eq("data.value.iv", _.RandomValue()) + Filter = F.Eq("data.value.iv", 200) }; - var contents = await QueryAsync(query); + var contents = await QueryAsync(query, 1000, 0); - Assert.NotNull(contents); + Assert.NotEmpty(contents); } - private async Task> QueryAsync(ClrQuery clrQuery) + private async Task> QueryAsync(ClrQuery clrQuery, int take = 1000, int skip = 100) { - clrQuery.Top = 1000; - clrQuery.Skip = 100; - clrQuery.Sort = new List { new SortNode("LastModified", SortOrder.Descending) }; + if (clrQuery.Take == long.MaxValue) + { + clrQuery.Take = take; + } + + if (clrQuery.Skip == 0) + { + clrQuery.Skip = skip; + } + + if (clrQuery.Sort.Count == 0) + { + clrQuery.Sort = new List + { + new SortNode("LastModified", SortOrder.Descending) + }; + } var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), clrQuery, SearchScope.All); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj index 0d756ab9b..c60353b65 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj @@ -20,6 +20,7 @@ + diff --git a/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs new file mode 100644 index 000000000..fa1e635a6 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/Queries/QueryTests.cs @@ -0,0 +1,64 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Collections.Generic; +using Xunit; + +namespace Squidex.Infrastructure.Queries +{ + public class QueryTests + { + [Fact] + public void Should_add_fields_from_sorting() + { + var query = new ClrQuery + { + Sort = new List + { + new SortNode("field1", SortOrder.Ascending), + new SortNode("field1", SortOrder.Ascending), + new SortNode("field2", SortOrder.Ascending) + } + }; + + var fields = query.GetAllFields(); + + var expected = new HashSet + { + "field1", + "field2" + }; + + Assert.Equal(expected, fields); + } + + [Fact] + public void Should_add_fields_from_filters() + { + var query = new ClrQuery + { + Filter = + ClrFilter.And( + ClrFilter.Not( + ClrFilter.Eq("field1", 1)), + ClrFilter.Or( + ClrFilter.Eq("field2", 2), + ClrFilter.Eq("field2", 4))) + }; + + var fields = query.GetAllFields(); + + var expected = new HashSet + { + "field1", + "field2" + }; + + Assert.Equal(expected, fields); + } + } +}