tsai
/
squidex
mirror of https://github.com/Squidex/squidex.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Optimize query performance.

pull/719/head
Sebastian 5 years ago
parent
5e1dcac6ac
commit
f700d28061
6 changed files with 75 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs
  2. 2
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs
  3. 16
      backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs
  4. 1
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs
  5. 16
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs
  6. 106
      backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs

7
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs
View File

@ -46,11 +46,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
{
new CreateIndexModel<MongoAssetEntity>(
Index
.Descending(x => x.LastModified)
.Ascending(x => x.Id)
.Ascending(x => x.IndexedAppId)
.Ascending(x => x.IsDeleted)
.Ascending(x => x.ParentId)
.Ascending(x => x.Tags)
.Descending(x => x.LastModified)),
.Ascending(x => x.Tags)),
new CreateIndexModel<MongoAssetEntity>(
Index
.Ascending(x => x.IndexedAppId)
@ -99,7 +100,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
var filter = BuildFilter(appId, q.Ids.ToHashSet());
var assetEntities =
await Collection.Find(filter).SortByDescending(x => x.LastModified)
await Collection.Find(filter).SortByDescending(x => x.LastModified).ThenBy(x => x.Id)
.QueryLimit(q.Query)
.QuerySkip(q.Query)
.ToListAsync(ct = default);

2
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs
View File

@ -42,6 +42,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets.Visitors
{
var filters = new List<FilterDefinition<MongoAssetEntity>>
{
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId)
};

16
backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs
View File

@ -45,11 +45,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
{
var indexBySchemaWithRefs =
new CreateIndexModel<MongoContentEntity>(Index
.Descending(x => x.LastModified)
.Ascending(x => x.Id)
.Ascending(x => x.IndexedAppId)
.Ascending(x => x.IndexedSchemaId)
.Ascending(x => x.IsDeleted)
.Ascending(x => x.ReferencedIds)
.Descending(x => x.LastModified));
.Ascending(x => x.ReferencedIds));
await Collection.Indexes.CreateOneAsync(indexBySchemaWithRefs, cancellationToken: ct);
@ -202,13 +203,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
private static bool IsSatisfiedByIndex(ClrQuery query)
{
return query.Sort?.All(x => x.Path.ToString() == "mt" && x.Order == SortOrder.Descending) == true;
return query.Sort != null &&
query.Sort.Count == 2 &&
query.Sort[0].Path.ToString() == "mt" &&
query.Sort[0].Order == SortOrder.Descending &&
query.Sort[1].Path.ToString() == "id" &&
query.Sort[1].Order == SortOrder.Ascending;
}
private static FilterDefinition<MongoContentEntity> BuildFilter(DomainId appId, DomainId schemaId, FilterNode<ClrValue>? filter)
{
var filters = new List<FilterDefinition<MongoContentEntity>>
{
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId),
Filter.Eq(x => x.IndexedSchemaId, schemaId)
};
@ -231,6 +239,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
{
var filters = new List<FilterDefinition<MongoContentEntity>>
{
Filter.Exists(x => x.LastModified),
Filter.Exists(x => x.Id),
Filter.Eq(x => x.IndexedAppId, appId),
Filter.In(x => x.IndexedSchemaId, schemaIds),
};

1
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs
View File

@ -87,6 +87,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
{
DocumentId = DomainId.NewGuid(),
Tags = new HashSet<string> { tag },
Id = DomainId.NewGuid(),
FileHash = fileName,
FileName = fileName,
FileSize = 1024,

16
backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryIntegrationTests.cs
View File

@ -138,18 +138,22 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb
yield return new object?[] { DomainId.Empty };
}
private async Task<IResultList<IAssetEntity>> QueryAsync(DomainId? parentId, ClrQuery query)
private async Task<IResultList<IAssetEntity>> QueryAsync(DomainId? parentId, ClrQuery clrQuery)
{
query.Top = 1000;
clrQuery.Top = 1000;
query.Skip = 100;
clrQuery.Skip = 100;
query.Sort = new List<SortNode>
if (clrQuery.Sort.Count == 0)
{
new SortNode("LastModified", SortOrder.Descending)
clrQuery.Sort = new List<SortNode>
{
new SortNode("LastModified", SortOrder.Descending),
new SortNode("Id", SortOrder.Descending)
};
}
var assets = await _.AssetRepository.QueryAsync(_.RandomAppId(), parentId, Q.Empty.WithQuery(query));
var assets = await _.AssetRepository.QueryAsync(_.RandomAppId(), parentId, Q.Empty.WithQuery(clrQuery));
return assets;
}

106
backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentsQueryIntegrationTests.cs
View File

@ -30,25 +30,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
_ = fixture;
}
public IEnumerable<object[]> Collections()
{
yield return new[] { _.ContentRepository };
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_verify_ids(IContentRepository repository)
[Fact]
public async Task Should_verify_ids()
{
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
var contents = await repository.QueryIdsAsync(_.RandomAppId(), ids, SearchScope.Published);
var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), ids, SearchScope.Published);
Assert.NotNull(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_by_ids(IContentRepository repository)
[Fact]
public async Task Should_query_contents_by_ids()
{
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
@ -57,85 +50,74 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
_.RandomSchema()
};
var contents = await repository.QueryAsync(_.RandomApp(), schemas, Q.Empty.WithIds(ids), SearchScope.All);
var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), schemas, Q.Empty.WithIds(ids), SearchScope.All);
Assert.NotNull(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_by_ids_and_schema(IContentRepository repository)
[Fact]
public async Task Should_query_contents_by_ids_and_schema()
{
var ids = Enumerable.Repeat(0, 50).Select(_ => DomainId.NewGuid()).ToHashSet();
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithIds(ids), SearchScope.All);
var contents = await _.ContentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithIds(ids), SearchScope.All);
Assert.NotNull(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_ids_by_filter(IContentRepository repository)
[Fact]
public async Task Should_query_contents_ids_by_filter()
{
var filter = F.Eq("data.value.iv", 12);
var filter = F.Eq("data.field1.iv", 12);
var contents = await repository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter);
var contents = await _.ContentRepository.QueryIdsAsync(_.RandomAppId(), _.RandomSchemaId(), filter);
Assert.NotEmpty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_by_filter(IContentRepository repository)
[Fact]
public async Task Should_query_contents_by_filter()
{
var query = new ClrQuery
{
Sort = new List<SortNode>
{
new SortNode("lastModified", SortOrder.Descending)
},
Filter = F.Eq("data.value.iv", 12)
Filter = F.Eq("data.field1.iv", 12)
};
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), Q.Empty.WithQuery(query), SearchScope.Published);
var contents = await QueryAsync(_.ContentRepository, query, 1000, 0);
Assert.NotEmpty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_scheduled(IContentRepository repository)
[Fact]
public async Task Should_query_contents_scheduled()
{
var time = SystemClock.Instance.GetCurrentInstant();
await repository.QueryScheduledWithoutDataAsync(time, _ => Task.CompletedTask);
await _.ContentRepository.QueryScheduledWithoutDataAsync(time, _ => Task.CompletedTask);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_default_query(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_default_query()
{
var query = new ClrQuery();
var contents = await QueryAsync(repository, query);
var contents = await QueryAsync(_.ContentRepository, query);
Assert.NotEmpty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_default_query_and_id(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_default_query_and_id()
{
var query = new ClrQuery();
var contents = await QueryAsync(repository, query, reference: DomainId.NewGuid());
var contents = await QueryAsync(_.ContentRepository, query, reference: DomainId.NewGuid());
Assert.Empty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_large_skip(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_large_skip()
{
var query = new ClrQuery
{
@ -145,54 +127,51 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
}
};
var contents = await QueryAsync(repository, query, 1000, 9000);
var contents = await QueryAsync(_.ContentRepository, query, 1000, 9000);
Assert.NotEmpty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_query_fulltext(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_query_fulltext()
{
var query = new ClrQuery
{
FullText = "hello"
};
var contents = await QueryAsync(repository, query);
var contents = await QueryAsync(_.ContentRepository, query);
Assert.NotNull(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_query_filter(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_query_filter()
{
var query = new ClrQuery
{
Filter = F.Eq("data.value.iv", 200)
Filter = F.Eq("data.field1.iv", 200)
};
var contents = await QueryAsync(repository, query, 1000, 0);
var contents = await QueryAsync(_.ContentRepository, query, 1000, 0);
Assert.NotEmpty(contents);
}
[Theory]
[MemberData(nameof(Collections))]
public async Task Should_query_contents_with_query_filter_and_id(IContentRepository repository)
[Fact]
public async Task Should_query_contents_with_query_filter_and_id()
{
var query = new ClrQuery
{
Filter = F.Eq("data.value.iv", 12)
};
var contents = await QueryAsync(repository, query, 1000, 0, reference: DomainId.NewGuid());
var contents = await QueryAsync(_.ContentRepository, query, 1000, 0, reference: DomainId.NewGuid());
Assert.Empty(contents);
}
private async Task<IResultList<IContentEntity>> QueryAsync(IContentRepository repository,
private async Task<IResultList<IContentEntity>> QueryAsync(IContentRepository contentRepository,
ClrQuery clrQuery,
int take = 1000,
int skip = 100,
@ -212,7 +191,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
{
clrQuery.Sort = new List<SortNode>
{
new SortNode("LastModified", SortOrder.Descending)
new SortNode("LastModified", SortOrder.Descending),
new SortNode("Id", SortOrder.Ascending)
};
}
@ -221,7 +201,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.MongoDb
.WithQuery(clrQuery)
.WithReference(reference);
var contents = await repository.QueryAsync(_.RandomApp(), _.RandomSchema(), q, SearchScope.All);
var contents = await contentRepository.QueryAsync(_.RandomApp(), _.RandomSchema(), q, SearchScope.All);
return contents;
}

Write Preview
Loading…
Cancel
Save