From 64b62763506e7848c3d7dc75ee7a419d17d2ec5c Mon Sep 17 00:00:00 2001 From: Sebastian Date: Thu, 5 Dec 2019 14:42:17 +0100 Subject: [PATCH] Improved indices for contents. --- .../Contents/MongoContentCollection.cs | 29 ++++++-- .../Contents/MongoContentRepository.cs | 2 +- .../Contents/Visitors/FilterFactory.cs | 5 -- .../Contents/Text/Extensions.cs | 2 +- .../EventSourcing/MongoEventStore_Writer.cs | 2 +- backend/tools/LoadTest/LoadTest.csproj | 4 +- backend/tools/LoadTest/ManyItemsTests.cs | 67 +++++++++++++++++++ backend/tools/LoadTest/Model/TestClient.cs | 13 +++- backend/tools/LoadTest/Model/TestEntity.cs | 5 +- backend/tools/LoadTest/ReadingFixture.cs | 2 +- backend/tools/LoadTest/Utils/RandomHash.cs | 48 +++++++++++++ backend/tools/LoadTest/Utils/RandomString.cs | 28 ++++++++ backend/tools/LoadTest/WritingBenchmarks.cs | 2 +- backend/tools/LoadTest/WritingFixture.cs | 2 +- 14 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 backend/tools/LoadTest/ManyItemsTests.cs create mode 100644 backend/tools/LoadTest/Utils/RandomHash.cs create mode 100644 backend/tools/LoadTest/Utils/RandomString.cs diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index 9485857b9..22120a31d 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -45,16 +45,29 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { return collection.Indexes.CreateManyAsync(new[] { + new CreateIndexModel(Index + .Ascending(x => x.IndexedAppId) + .Ascending(x => x.IsDeleted) + .Ascending(x => x.Status) + .Descending(x => x.LastModified)), new CreateIndexModel(Index .Ascending(x => x.IndexedAppId) .Ascending(x => x.IsDeleted) .Ascending(x => x.Status) .Ascending(x => x.Id)), + new CreateIndexModel(Index + .Ascending(x => x.IndexedSchemaId) + .Ascending(x => x.IsDeleted) + .Ascending(x => x.Status) + .Descending(x => x.LastModified)), new CreateIndexModel(Index .Ascending(x => x.IndexedSchemaId) .Ascending(x => x.IsDeleted) .Ascending(x => x.Status) .Ascending(x => x.Id)), + new CreateIndexModel(Index + .Ascending(x => x.IndexedAppId) + .Ascending(x => x.Id)), new CreateIndexModel(Index .Ascending(x => x.ScheduledAt) .Ascending(x => x.IsDeleted)), @@ -149,11 +162,19 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents public async Task FindContentAsync(ISchemaEntity schema, Guid id, Status[]? status, bool includeDraft) { - var find = Collection.Find(FilterFactory.Build(schema.Id, id, status)); + var find = Collection.Find(x => x.Id == id); var contentEntity = await find.WithoutDraft(includeDraft).FirstOrDefaultAsync(); - contentEntity?.ParseData(schema.SchemaDef, serializer); + if (contentEntity != null) + { + if (contentEntity.IndexedSchemaId != schema.Id || status?.Contains(contentEntity.Status) == false) + { + return null; + } + + contentEntity?.ParseData(schema.SchemaDef, serializer); + } return contentEntity; } @@ -180,10 +201,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return contentEntities.Select(x => (Guid.Parse(x["_si"].AsString), Guid.Parse(x["_id"].AsString))).ToList(); } - public async Task> QueryIdsAsync(HashSet ids) + public async Task> QueryIdsAsync(Guid appId, HashSet ids) { var contentEntities = - await Collection.Find(Filter.In(x => x.Id, ids)).Only(x => x.Id, x => x.IndexedSchemaId) + await Collection.Find(Filter.And(Filter.Eq(x => x.IndexedAppId, appId), Filter.In(x => x.Id, ids))).Only(x => x.Id, x => x.IndexedSchemaId) .ToListAsync(); return contentEntities.Select(x => (Guid.Parse(x["_si"].AsString), Guid.Parse(x["_id"].AsString))).ToList(); diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 402b89427..d4a923d30 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -129,7 +129,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { using (Profiler.TraceMethod()) { - return await contents.QueryIdsAsync(ids); + return await contents.QueryIdsAsync(appId, ids); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Visitors/FilterFactory.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Visitors/FilterFactory.cs index 95efcb736..18c95a436 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Visitors/FilterFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Visitors/FilterFactory.cs @@ -62,11 +62,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Visitors return !includeDraft ? cursor.Not(x => x.DataDraftByIds, x => x.IsDeleted) : cursor; } - public static FilterDefinition Build(Guid schemaId, Guid id, Status[]? status) - { - return CreateFilter(null, schemaId, new List { id }, status, null); - } - public static FilterDefinition IdsByApp(Guid appId, ICollection ids, Status[]? status) { return CreateFilter(appId, null, ids, status, null); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Extensions.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Extensions.cs index e458c09a0..dd860bf69 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Extensions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Extensions.cs @@ -111,7 +111,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text result = new BytesRef(); } - if (reader == null) + if (reader == null || docId < 0) { return result; } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs index 5b50bbe90..81910f65e 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Writer.cs @@ -16,7 +16,7 @@ namespace Squidex.Infrastructure.EventSourcing { public partial class MongoEventStore { - private const int MaxCommitSize = 10; + private const int MaxCommitSize = 100; private const int MaxWriteAttempts = 20; private static readonly BsonTimestamp EmptyTimestamp = new BsonTimestamp(0); diff --git a/backend/tools/LoadTest/LoadTest.csproj b/backend/tools/LoadTest/LoadTest.csproj index 6f04598b2..5f3b3d534 100644 --- a/backend/tools/LoadTest/LoadTest.csproj +++ b/backend/tools/LoadTest/LoadTest.csproj @@ -4,9 +4,9 @@ netcoreapp3.0 - + - + diff --git a/backend/tools/LoadTest/ManyItemsTests.cs b/backend/tools/LoadTest/ManyItemsTests.cs new file mode 100644 index 000000000..ecbe3fa35 --- /dev/null +++ b/backend/tools/LoadTest/ManyItemsTests.cs @@ -0,0 +1,67 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Threading.Tasks; +using LoadTest.Model; +using LoadTest.Utils; +using Squidex.ClientLibrary; +using Xunit; + +namespace LoadTest +{ + public class ManyItemsTests + { + private readonly SquidexClient client; + + public ManyItemsTests() + { + client = TestClient.BuildAsync("multiple").Result; + } + + [Fact] + public async Task Should_read_many_async() + { + var contents = await client.GetAsync(); + + for (var i = contents.Total; i < 20000; i++) + { + await client.CreateAsync(new TestEntityData + { + String = RandomString.Create(1000) + }, publish: true); + } + + var found = await client.GetAll2Async(); + + Assert.Equal(20000, found.Items.Count); + } + } + + public static class SquidexClientExtensions + { + public static async Task> GetAll2Async(this SquidexClient client, int batchSize = 200) + where TEntity : SquidexEntityBase + where TData : class, new() + { + var query = new ODataQuery { Top = batchSize, Skip = 0 }; + + var entities = new SquidexEntities(); + do + { + var getResult = await client.GetAsync(query); + + entities.Total = getResult.Total; + entities.Items.AddRange(getResult.Items); + + query.Skip += getResult.Items.Count; + } + while (query.Skip < entities.Total); + + return entities; + } + } +} diff --git a/backend/tools/LoadTest/Model/TestClient.cs b/backend/tools/LoadTest/Model/TestClient.cs index e8998ed10..edbd61294 100644 --- a/backend/tools/LoadTest/Model/TestClient.cs +++ b/backend/tools/LoadTest/Model/TestClient.cs @@ -72,11 +72,18 @@ namespace LoadTest.Model { new UpsertSchemaFieldDto { - Name = "value", + Name = "number", Properties = new NumberFieldPropertiesDto { - IsRequired = true, - IsListField = true + IsRequired = true + } + }, + new UpsertSchemaFieldDto + { + Name = "string", + Properties = new StringFieldPropertiesDto + { + IsRequired = false } } }, diff --git a/backend/tools/LoadTest/Model/TestEntity.cs b/backend/tools/LoadTest/Model/TestEntity.cs index 0bda9b364..b96ed2975 100644 --- a/backend/tools/LoadTest/Model/TestEntity.cs +++ b/backend/tools/LoadTest/Model/TestEntity.cs @@ -17,6 +17,9 @@ namespace LoadTest.Model public sealed class TestEntityData { [JsonConverter(typeof(InvariantConverter))] - public int Value { get; set; } + public int Number { get; set; } + + [JsonConverter(typeof(InvariantConverter))] + public string String { get; set; } } } diff --git a/backend/tools/LoadTest/ReadingFixture.cs b/backend/tools/LoadTest/ReadingFixture.cs index 0187b15f3..dbad0db64 100644 --- a/backend/tools/LoadTest/ReadingFixture.cs +++ b/backend/tools/LoadTest/ReadingFixture.cs @@ -36,7 +36,7 @@ namespace LoadTest for (var i = 10; i > 0; i--) { - await Client.CreateAsync(new TestEntityData { Value = i }, true); + await Client.CreateAsync(new TestEntityData { Number = i }, true); } } diff --git a/backend/tools/LoadTest/Utils/RandomHash.cs b/backend/tools/LoadTest/Utils/RandomHash.cs new file mode 100644 index 000000000..9dd29bd62 --- /dev/null +++ b/backend/tools/LoadTest/Utils/RandomHash.cs @@ -0,0 +1,48 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Security.Cryptography; +using System.Text; + +namespace LoadTest.Utils +{ + public static class RandomHash + { + public static string New() + { + return Guid.NewGuid() + .ToString().Sha256Base64() + .ToLowerInvariant() + .Replace("+", "x") + .Replace("=", "x") + .Replace("/", "x"); + } + + public static string Simple() + { + return Guid.NewGuid().ToString().Replace("-", string.Empty); + } + + public static string Sha256Base64(this string value) + { + return Sha256Base64(Encoding.UTF8.GetBytes(value)); + } + + public static string Sha256Base64(this byte[] bytes) + { + using (var sha = SHA256.Create()) + { + var bytesHash = sha.ComputeHash(bytes); + + var result = Convert.ToBase64String(bytesHash); + + return result; + } + } + } +} diff --git a/backend/tools/LoadTest/Utils/RandomString.cs b/backend/tools/LoadTest/Utils/RandomString.cs new file mode 100644 index 000000000..1f626cb1e --- /dev/null +++ b/backend/tools/LoadTest/Utils/RandomString.cs @@ -0,0 +1,28 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; + +namespace LoadTest.Utils +{ + public static class RandomString + { + private static readonly Random Random = new Random(); + + public static string Create(int length) + { + var chars = new char[length]; + + for (var i = 0; i < length; i++) + { + chars[i] = (char)Random.Next(48, 122); + } + + return new string(chars); + } + } +} diff --git a/backend/tools/LoadTest/WritingBenchmarks.cs b/backend/tools/LoadTest/WritingBenchmarks.cs index 185059974..af45c9ca1 100644 --- a/backend/tools/LoadTest/WritingBenchmarks.cs +++ b/backend/tools/LoadTest/WritingBenchmarks.cs @@ -62,7 +62,7 @@ namespace LoadTest await Run.Parallel(numUsers, numIterationsPerUser, async () => { - await Fixture.Client.CreateAsync(new TestEntityData { Value = random.Next() }, true); + await Fixture.Client.CreateAsync(new TestEntityData { Number = random.Next() }, true); }); } } diff --git a/backend/tools/LoadTest/WritingFixture.cs b/backend/tools/LoadTest/WritingFixture.cs index 70af0d11c..b1a2ffe3a 100644 --- a/backend/tools/LoadTest/WritingFixture.cs +++ b/backend/tools/LoadTest/WritingFixture.cs @@ -20,7 +20,7 @@ namespace LoadTest { Task.Run(async () => { - Client = await TestClient.BuildAsync("reading"); + Client = await TestClient.BuildAsync("writing"); }).Wait(); }