tsai
/
squidex
mirror of https://github.com/Squidex/squidex.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Optimize query performance.

pull/719/head
Sebastian 5 years ago
parent
5e1dcac6ac
commit
f700d28061
6 changed files with 75 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs
  2. 2
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs
  3. 16
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs
  4. 1
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs
  5. 20
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs
  6. 106
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs

7
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs
View File

@ -46,11 +46,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{ {
new CreateIndexModel<MongoAssetEntity>( new CreateIndexModel<MongoAssetEntity>(
Index Index
.Descending(x => x.LastModified)
.Ascending(x => x.Id)
.Ascending(x => x.IndexedAppId) .Ascending(x => x.IndexedAppId)
.Ascending(x => x.IsDeleted) .Ascending(x => x.IsDeleted)
.Ascending(x => x.ParentId) .Ascending(x => x.ParentId)
.Ascending(x => x.Tags) .Ascending(x => x.Tags)),
.Descending(x => x.LastModified)),
new CreateIndexModel<MongoAssetEntity>( new CreateIndexModel<MongoAssetEntity>(
Index Index
.Ascending(x => x.IndexedAppId) .Ascending(x => x.IndexedAppId)
@ -99,7 +100,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
var filter = BuildFilter(appId, q.Ids.ToHashSet()); var filter = BuildFilter(appId, q.Ids.ToHashSet());
var assetEntities = var assetEntities =
await Collection.Find(filter).SortByDescending(x => x.LastModified) await Collection.Find(filter).SortByDescending(x => x.LastModified).ThenBy(x => x.Id)
.QueryLimit(q.Query) .QueryLimit(q.Query)
.QuerySkip(q.Query) .QuerySkip(q.Query)
.ToListAsync(ct = default); .ToListAsync(ct = default);

2
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs
View File

@ -42,6 +42,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets.Visitors
{ {
var filters = new List<FilterDefinition<MongoAssetEntity>> var filters = new List<FilterDefinition<MongoAssetEntity>>
{ {
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId) Filter.Eq(x => x.IndexedAppId, appId)
}; };

16
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs
View File

@ -45,11 +45,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
{ {
var indexBySchemaWithRefs = var indexBySchemaWithRefs =
new CreateIndexModel<MongoContentEntity>(Index new CreateIndexModel<MongoContentEntity>(Index
.Descending(x => x.LastModified)
.Ascending(x => x.Id)
.Ascending(x => x.IndexedAppId) .Ascending(x => x.IndexedAppId)
.Ascending(x => x.IndexedSchemaId) .Ascending(x => x.IndexedSchemaId)
.Ascending(x => x.IsDeleted) .Ascending(x => x.IsDeleted)
.Ascending(x => x.ReferencedIds) .Ascending(x => x.ReferencedIds));
.Descending(x => x.LastModified));
await Collection.Indexes.CreateOneAsync(indexBySchemaWithRefs, cancellationToken: ct); await Collection.Indexes.CreateOneAsync(indexBySchemaWithRefs, cancellationToken: ct);
@ -202,13 +203,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
private static bool IsSatisfiedByIndex(ClrQuery query) private static bool IsSatisfiedByIndex(ClrQuery query)
{ {
return query.Sort?.All(x => x.Path.ToString() == "mt" && x.Order == SortOrder.Descending) == true; return query.Sort != null &&
query.Sort.Count == 2 &&
query.Sort[0].Path.ToString() == "mt" &&
query.Sort[0].Order == SortOrder.Descending &&
query.Sort[1].Path.ToString() == "id" &&
query.Sort[1].Order == SortOrder.Ascending;
} }
private static FilterDefinition<MongoContentEntity> BuildFilter(DomainId appId, DomainId schemaId, FilterNode<ClrValue>? filter) private static FilterDefinition<MongoContentEntity> BuildFilter(DomainId appId, DomainId schemaId, FilterNode<ClrValue>? filter)
{ {
var filters = new List<FilterDefinition<MongoContentEntity>> var filters = new List<FilterDefinition<MongoContentEntity>>
{ {
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId), Filter.Eq(x => x.IndexedAppId, appId),
Filter.Eq(x => x.IndexedSchemaId, schemaId) Filter.Eq(x => x.IndexedSchemaId, schemaId)
}; };
@ -231,6 +239,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
{ {
var filters = new List<FilterDefinition<MongoContentEntity>> var filters = new List<FilterDefinition<MongoContentEntity>>
{ {
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId), Filter.Eq(x => x.IndexedAppId, appId),
Filter.In(x => x.IndexedSchemaId, schemaIds), Filter.In(x => x.IndexedSchemaId, schemaIds),
}; };

1
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs
View File

@ -87,6 +87,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
{ {
DocumentId = DomainId.NewGuid(), DocumentId = DomainId.NewGuid(),
Tags = new HashSet<string> { tag }, Tags = new HashSet<string> { tag },
Id = DomainId.NewGuid(),
FileHash = fileName, FileHash = fileName,
FileName = fileName, FileName = fileName,
FileSize = 1024, FileSize = 1024,

20
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs
View File

@ -138,18 +138,22 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
yield return new object?[] { DomainId.Empty }; yield return new object?[] { DomainId.Empty };
} }
private async Task<IResultList<IAssetEntity>> QueryAsync(DomainId? parentId, ClrQuery query) private async Task<IResultList<IAssetEntity>> QueryAsync(DomainId? parentId, ClrQuery clrQuery)
{ {
query.Top = 1000; clrQuery.Top = 1000;
query.Skip = 100; clrQuery.Skip = 100;
query.Sort = new List<SortNode> if (clrQuery.Sort.Count == 0)
{ {
new SortNode("LastModified", SortOrder.Descending) clrQuery.Sort = new List<SortNode>
}; {
new SortNode("LastModified", SortOrder.Descending),
var assets = await _.AssetRepository.QueryAsync(_.RandomAppId(), parentId, Q.Empty.WithQuery(query)); new SortNode("Id", SortOrder.Descending)
};
}
var assets = await _.AssetRepository.QueryAsync(_.RandomAppId(), parentId, Q.Empty.WithQuery(clrQuery));
return assets; return assets;
} }

106
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs
View File

@ -30,25 +30,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
_ = fixture; _ = fixture;
} }
public IEnumerable<object[]> Collections() [Fact]
{ public async Task Should_verify_ids()
yield return new[] { _.ContentRepository };
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_verify_ids(IContentRepository repository)
{ {
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet(); var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
var contents = await repository.QueryIdsAsync(_.RandomAppId(), ids, SearchScope.Published); var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), ids, SearchScope.Published);
Assert.NotNull(contents); Assert.NotNull(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_by_ids()
public async Task Should_query_contents_by_ids(IContentRepository repository)
{ {
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet(); var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
@ -57,85 +50,74 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
_.RandomSchema() _.RandomSchema()
}; };
var contents = await repository.QueryAsync(_.RandomApp(), schemas, Q.Empty.WithIds(ids), SearchScope.All); var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), schemas, Q.Empty.WithIds(ids), SearchScope.All);
Assert.NotNull(contents); Assert.NotNull(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_by_ids_and_schema()
public async Task Should_query_contents_by_ids_and_schema(IContentRepository repository)
{ {
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet(); var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithIds(ids), SearchScope.All); var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithIds(ids), SearchScope.All);
Assert.NotNull(contents); Assert.NotNull(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_ids_by_filter()
public async Task Should_query_contents_ids_by_filter(IContentRepository repository)
{ {
var filter = F.Eq("data.value.iv", 12); var filter = F.Eq("data.field1.iv", 12);
var contents = await repository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter); var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter);
Assert.NotEmpty(contents); Assert.NotEmpty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_by_filter()
public async Task Should_query_contents_by_filter(IContentRepository repository)
{ {
var query = new ClrQuery var query = new ClrQuery
{ {
Sort = new List<SortNode> Filter = F.Eq("data.field1.iv", 12)
{
new SortNode("lastModified", SortOrder.Descending)
},
Filter = F.Eq("data.value.iv", 12)
}; };
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithQuery(query), SearchScope.Published); var contents = await QueryAsync(_.ContentRepository, query, 1000, 0);
Assert.NotEmpty(contents); Assert.NotEmpty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_scheduled()
public async Task Should_query_contents_scheduled(IContentRepository repository)
{ {
var time = SystemClock.Instance.GetCurrentInstant(); var time = SystemClock.Instance.GetCurrentInstant();
await repository.QueryScheduledWithoutDataAsync(time, _ => Task.CompletedTask); await _.ContentRepository.QueryScheduledWithoutDataAsync(time, _ => Task.CompletedTask);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_default_query()
public async Task Should_query_contents_with_default_query(IContentRepository repository)
{ {
var query = new ClrQuery(); var query = new ClrQuery();
var contents = await QueryAsync(repository, query); var contents = await QueryAsync(_.ContentRepository, query);
Assert.NotEmpty(contents); Assert.NotEmpty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_default_query_and_id()
public async Task Should_query_contents_with_default_query_and_id(IContentRepository repository)
{ {
var query = new ClrQuery(); var query = new ClrQuery();
var contents = await QueryAsync(repository, query, reference: DomainId.NewGuid()); var contents = await QueryAsync(_.ContentRepository, query, reference: DomainId.NewGuid());
Assert.Empty(contents); Assert.Empty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_large_skip()
public async Task Should_query_contents_with_large_skip(IContentRepository repository)
{ {
var query = new ClrQuery var query = new ClrQuery
{ {
@ -145,54 +127,51 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
} }
}; };
var contents = await QueryAsync(repository, query, 1000, 9000); var contents = await QueryAsync(_.ContentRepository, query, 1000, 9000);
Assert.NotEmpty(contents); Assert.NotEmpty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_query_fulltext()
public async Task Should_query_contents_with_query_fulltext(IContentRepository repository)
{ {
var query = new ClrQuery var query = new ClrQuery
{ {
FullText = "hello" FullText = "hello"
}; };
var contents = await QueryAsync(repository, query); var contents = await QueryAsync(_.ContentRepository, query);
Assert.NotNull(contents); Assert.NotNull(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_query_filter()
public async Task Should_query_contents_with_query_filter(IContentRepository repository)
{ {
var query = new ClrQuery var query = new ClrQuery
{ {
Filter = F.Eq("data.value.iv", 200) Filter = F.Eq("data.field1.iv", 200)
}; };
var contents = await QueryAsync(repository, query, 1000, 0); var contents = await QueryAsync(_.ContentRepository, query, 1000, 0);
Assert.NotEmpty(contents); Assert.NotEmpty(contents);
} }
[Theory] [Fact]
[MemberData(nameof(Collections))] public async Task Should_query_contents_with_query_filter_and_id()
public async Task Should_query_contents_with_query_filter_and_id(IContentRepository repository)
{ {
var query = new ClrQuery var query = new ClrQuery
{ {
Filter = F.Eq("data.value.iv", 12) Filter = F.Eq("data.value.iv", 12)
}; };
var contents = await QueryAsync(repository, query, 1000, 0, reference: DomainId.NewGuid()); var contents = await QueryAsync(_.ContentRepository, query, 1000, 0, reference: DomainId.NewGuid());
Assert.Empty(contents); Assert.Empty(contents);
} }
private async Task<IResultList<IContentEntity>> QueryAsync(IContentRepository repository, private async Task<IResultList<IContentEntity>> QueryAsync(IContentRepository contentRepository,
ClrQuery clrQuery, ClrQuery clrQuery,
int take = 1000, int take = 1000,
int skip = 100, int skip = 100,
@ -212,7 +191,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
{ {
clrQuery.Sort = new List<SortNode> clrQuery.Sort = new List<SortNode>
{ {
new SortNode("LastModified", SortOrder.Descending) new SortNode("LastModified", SortOrder.Descending),
new SortNode("Id", SortOrder.Ascending)
}; };
} }
@ -221,7 +201,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
.WithQuery(clrQuery) .WithQuery(clrQuery)
.WithReference(reference); .WithReference(reference);
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), q, SearchScope.All); var contents = await contentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), q, SearchScope.All);
return contents; return contents;
} }

Write Preview
Loading…
Cancel
Save