From c9f578df70939ecc2031ae4533ad2779556a2254 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Sun, 4 Oct 2020 22:28:40 +0200 Subject: [PATCH] Performance improvements. --- backend/src/Migrations/MigrationPath.cs | 24 ++- .../MongoDb/AddAppIdToEventStream.cs | 85 ++++++----- .../Migrations/MongoDb/ConvertDocumentIds.cs | 137 ++++++++++++++++++ .../Assets/MongoAssetFolderRepository.cs | 2 +- .../Assets/MongoAssetRepository.cs | 2 +- .../Contents/MongoContentCollectionAll.cs | 2 +- .../MongoContentCollectionPublished.cs | 2 +- .../EventSourcing/MongoEventStore.cs | 2 +- .../Commands/DomainObjectBase.cs | 45 +++--- .../Migrations/Migrator.cs | 2 +- .../Squidex/Config/Domain/StoreServices.cs | 3 + .../Commands/DomainObjectTests.cs | 17 ++- .../Commands/LogSnapshotDomainObjectTests.cs | 17 ++- .../angular/image-source.directive.ts | 5 +- 14 files changed, 267 insertions(+), 78 deletions(-) create mode 100644 backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs diff --git a/backend/src/Migrations/MigrationPath.cs b/backend/src/Migrations/MigrationPath.cs index f24eb8bae..1252c82b8 100644 --- a/backend/src/Migrations/MigrationPath.cs +++ b/backend/src/Migrations/MigrationPath.cs @@ -48,7 +48,7 @@ namespace Migrations yield return serviceProvider.GetRequiredService(); } - // Version 22: Also add app id to aggregate id. + // Version 22: Integrate Domain Id. if (version < 22) { yield return serviceProvider.GetRequiredService(); @@ -80,7 +80,7 @@ namespace Migrations } // Version 14: Schema refactoring - // Version 22: Also add app id to aggregate id. + // Version 22: Introduce domain id. if (version < 22) { yield return serviceProvider.GetRequiredService(); @@ -88,8 +88,7 @@ namespace Migrations } // Version 18: Rebuild assets. - // Version 22: Introduce domain id. - if (version < 22) + if (version < 18) { yield return serviceProvider.GetService(); yield return serviceProvider.GetService(); @@ -107,14 +106,27 @@ namespace Migrations { yield return serviceProvider.GetService(); } + + // Version 22: Introduce domain id. + if (version < 22) + { + yield return serviceProvider.GetRequiredService().ForAssets(); + } } // Version 21: Introduce content drafts V2. - // Version 22: Introduce domain id. - if (version < 22) + if (version < 21) { yield return serviceProvider.GetRequiredService(); } + else + { + // Version 22: Introduce domain id. + if (version < 22) + { + yield return serviceProvider.GetRequiredService().ForContents(); + } + } } // Version 13: Json refactoring diff --git a/backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs b/backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs index 9e1e89278..b1e50f5e3 100644 --- a/backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs +++ b/backend/src/Migrations/Migrations/MongoDb/AddAppIdToEventStream.cs @@ -7,11 +7,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Threading.Tasks; using System.Threading.Tasks.Dataflow; using MongoDB.Bson; using MongoDB.Driver; +using Squidex.Infrastructure; using Squidex.Infrastructure.Migrations; namespace Migrations.Migrations.MongoDb @@ -27,70 +29,63 @@ namespace Migrations.Migrations.MongoDb public async Task UpdateAsync() { - var collection = database.GetCollection("Events"); - - const int SizeOfBatch = 200; + const int SizeOfBatch = 1000; const int SizeOfQueue = 20; + var collectionOld = database.GetCollection("Events"); + var collectionNew = database.GetCollection("Events2"); + var batchBlock = new BatchBlock(SizeOfBatch, new GroupingDataflowBlockOptions { BoundedCapacity = SizeOfQueue * SizeOfBatch }); + var writeOptions = new BulkWriteOptions + { + IsOrdered = false + }; + var actionBlock = new ActionBlock(async batch => { var updates = new List>(); - foreach (var commit in batch) + foreach (var document in batch) { - var eventStream = commit["EventStream"].AsString; + var eventStream = document["EventStream"].AsString; - string? appId = null; - - foreach (var @event in commit["Events"].AsBsonArray) - { - var metadata = @event["Metadata"].AsBsonDocument; - - if (metadata.TryGetValue("AppId", out var value)) - { - appId = value.AsString; - } - } - - if (appId != null) + if (TryGetAppId(document, out var appId)) { var parts = eventStream.Split("-"); var domainType = parts[0]; var domainId = string.Join("-", parts.Skip(1)); - var newStreamName = $"{domainType}-{appId}--{domainId}"; - - var update = Builders.Update.Set("EventStream", newStreamName); + var newDomainId = DomainId.Combine(appId, domainId).ToString(); + var newStreamName = $"{domainType}-{newDomainId}"; - var i = 0; + document["EventStream"] = newStreamName; - foreach (var @event in commit["Events"].AsBsonArray) + foreach (var @event in document["Events"].AsBsonArray) { - update = update.Set($"Events.{i}.Metadata.AggregateId", $"{appId}--{domainId}"); - update = update.Unset($"Events.{i}.Metadata.AppId"); + var metadata = @event["Metadata"].AsBsonDocument; - i++; + metadata["AggregateId"] = newDomainId; + metadata.Remove("AppId"); } - var filter = Builders.Filter.Eq("_id", commit["_id"].AsString); + var filter = Builders.Filter.Eq("_id", document["_id"].AsString); - updates.Add(new UpdateOneModel(filter, update)); + updates.Add(new ReplaceOneModel(filter, document) + { + IsUpsert = true + }); } } - if (updates.Count > 0) - { - await collection.BulkWriteAsync(updates); - } + await collectionNew.BulkWriteAsync(updates, writeOptions); }, new ExecutionDataflowBlockOptions { - MaxDegreeOfParallelism = 4, + MaxDegreeOfParallelism = Environment.ProcessorCount * 2, MaxMessagesPerTask = 1, BoundedCapacity = SizeOfQueue }); @@ -100,19 +95,29 @@ namespace Migrations.Migrations.MongoDb PropagateCompletion = true }); - await collection.Find(new BsonDocument()).ForEachAsync(async commit => + await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync); + + batchBlock.Complete(); + + await actionBlock.Completion; + } + + private static bool TryGetAppId(BsonDocument document, [MaybeNullWhen(false)] out string appId) + { + foreach (var @event in document["Events"].AsBsonArray) { - var eventStream = commit["EventStream"].AsString; + var metadata = @event["Metadata"].AsBsonDocument; - if (!eventStream.Contains("--") && !eventStream.StartsWith("app-", StringComparison.OrdinalIgnoreCase)) + if (metadata.TryGetValue("AppId", out var value)) { - await batchBlock.SendAsync(commit); + appId = value.AsString; + return true; } - }); + } - batchBlock.Complete(); + appId = null; - await actionBlock.Completion; + return false; } } } diff --git a/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs b/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs new file mode 100644 index 000000000..6a571eaf4 --- /dev/null +++ b/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs @@ -0,0 +1,137 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using MongoDB.Bson; +using MongoDB.Driver; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Migrations; + +namespace Migrations.Migrations.MongoDb +{ + public sealed class ConvertDocumentIds : IMigration + { + private readonly IMongoDatabase database; + private readonly IMongoDatabase databaseContent; + private Scope scope; + + private enum Scope + { + None, + Assets, + Contents + } + + public ConvertDocumentIds(IMongoDatabase database, IMongoDatabase databaseContent) + { + this.database = database; + this.databaseContent = databaseContent; + } + + public override string ToString() + { + return $"{base.ToString()}({scope})"; + } + + public ConvertDocumentIds ForContents() + { + scope = Scope.Contents; + + return this; + } + + public ConvertDocumentIds ForAssets() + { + scope = Scope.Assets; + + return this; + } + + public async Task UpdateAsync() + { + switch (scope) + { + case Scope.Assets: + await RebuildAsync(database, "States_Assets"); + await RebuildAsync(database, "States_AssetFolders"); + break; + case Scope.Contents: + await RebuildAsync(databaseContent, "State_Contents_All", "States_Contents_All2"); + await RebuildAsync(databaseContent, "State_Contents_Published", "States_Contents_Published2"); + break; + } + } + + private static async Task RebuildAsync(IMongoDatabase database, string collectionNameOld, string? collectionNameNew = null) + { + const int SizeOfBatch = 1000; + const int SizeOfQueue = 10; + + if (string.IsNullOrWhiteSpace(collectionNameNew)) + { + collectionNameNew = $"{collectionNameOld}2"; + } + + var collectionOld = database.GetCollection(collectionNameOld); + var collectionNew = database.GetCollection(collectionNameNew); + + var batchBlock = new BatchBlock(SizeOfBatch, new GroupingDataflowBlockOptions + { + BoundedCapacity = SizeOfQueue * SizeOfBatch + }); + + var writeOptions = new BulkWriteOptions + { + IsOrdered = false + }; + + var actionBlock = new ActionBlock(async batch => + { + var updates = new List>(); + + foreach (var document in batch) + { + var appId = document["_ai"].AsString; + + var documentIdOld = document["_id"].AsString; + var documentIdNew = DomainId.Combine(appId, documentIdOld).ToString(); + + document["id"] = documentIdOld; + document["_id"] = documentIdNew; + + var filter = Builders.Filter.Eq("_id", documentIdNew); + + updates.Add(new ReplaceOneModel(filter, document) + { + IsUpsert = true + }); + } + + await collectionNew.BulkWriteAsync(updates, writeOptions); + }, new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = Environment.ProcessorCount * 2, + MaxMessagesPerTask = 1, + BoundedCapacity = SizeOfQueue + }); + + batchBlock.LinkTo(actionBlock, new DataflowLinkOptions + { + PropagateCompletion = true + }); + + await collectionOld.Find(new BsonDocument()).ForEachAsync(batchBlock.SendAsync); + + batchBlock.Complete(); + + await actionBlock.Completion; + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository.cs index 112c5cafe..9e11f4e98 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository.cs @@ -27,7 +27,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets protected override string CollectionName() { - return "States_AssetFolders"; + return "States_AssetFolders2"; } protected override Task SetupCollectionAsync(IMongoCollection collection, CancellationToken ct = default) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index c0eab4ea4..d1ca3a4b2 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -37,7 +37,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets protected override string CollectionName() { - return "States_Assets"; + return "States_Assets2"; } protected override Task SetupCollectionAsync(IMongoCollection collection, CancellationToken ct = default) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionAll.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionAll.cs index f06b49f3d..0d09015ac 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionAll.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionAll.cs @@ -51,7 +51,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents protected override string CollectionName() { - return "State_Contents_All"; + return "States_Contents_All2"; } protected override async Task SetupCollectionAsync(IMongoCollection collection, CancellationToken ct = default) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionPublished.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionPublished.cs index 7ace0278e..9fea155a5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionPublished.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollectionPublished.cs @@ -54,7 +54,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents protected override string CollectionName() { - return "State_Contents_Published"; + return "State_Contents_Published2"; } protected override async Task SetupCollectionAsync(IMongoCollection collection, CancellationToken ct = default) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs index 9a1e0c534..c8fd6da02 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore.cs @@ -44,7 +44,7 @@ namespace Squidex.Infrastructure.EventSourcing protected override string CollectionName() { - return "Events"; + return "Events2"; } protected override MongoCollectionSettings CollectionSettings() diff --git a/backend/src/Squidex.Infrastructure/Commands/DomainObjectBase.cs b/backend/src/Squidex.Infrastructure/Commands/DomainObjectBase.cs index 20d08a311..d011d2e74 100644 --- a/backend/src/Squidex.Infrastructure/Commands/DomainObjectBase.cs +++ b/backend/src/Squidex.Infrastructure/Commands/DomainObjectBase.cs @@ -10,6 +10,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Log; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.Tasks; namespace Squidex.Infrastructure.Commands @@ -146,15 +147,25 @@ namespace Squidex.Infrastructure.Commands Guard.NotNull(command, nameof(command)); Guard.NotNull(handler, nameof(handler)); - await EnsureLoadedAsync(); - - if (IsDeleted()) - { - throw new DomainException("Object has already been deleted."); - } - if (isUpdate) { + await EnsureLoadedAsync(); + + if (Version < 0) + { + throw new DomainObjectNotFoundException(uniqueId.ToString()); + } + + if (Version != command.ExpectedVersion && command.ExpectedVersion > EtagVersion.Any) + { + throw new DomainObjectVersionException(uniqueId.ToString(), Version, command.ExpectedVersion); + } + + if (IsDeleted()) + { + throw new DomainException("Object has already been deleted."); + } + if (!CanAccept(command)) { throw new DomainException("Invalid command."); @@ -162,7 +173,9 @@ namespace Squidex.Infrastructure.Commands } else { - if (Version > EtagVersion.Empty) + command.ExpectedVersion = EtagVersion.Empty; + + if (Version != command.ExpectedVersion) { throw new DomainObjectConflictException(uniqueId.ToString()); } @@ -173,16 +186,6 @@ namespace Squidex.Infrastructure.Commands } } - if (command.ExpectedVersion > EtagVersion.Any && command.ExpectedVersion != Version) - { - throw new DomainObjectVersionException(uniqueId.ToString(), Version, command.ExpectedVersion); - } - - if (isUpdate && Version < 0) - { - throw new DomainObjectNotFoundException(uniqueId.ToString()); - } - var previousSnapshot = Snapshot; var previousVersion = Version; try @@ -209,6 +212,12 @@ namespace Squidex.Infrastructure.Commands return result; } + catch (InconsistentStateException) when (!isUpdate) + { + RestorePreviousSnapshot(previousSnapshot, previousVersion); + + throw new DomainObjectConflictException(uniqueId.ToString()); + } catch { RestorePreviousSnapshot(previousSnapshot, previousVersion); diff --git a/backend/src/Squidex.Infrastructure/Migrations/Migrator.cs b/backend/src/Squidex.Infrastructure/Migrations/Migrator.cs index fd03b0cc3..6ded830f3 100644 --- a/backend/src/Squidex.Infrastructure/Migrations/Migrator.cs +++ b/backend/src/Squidex.Infrastructure/Migrations/Migrator.cs @@ -61,7 +61,7 @@ namespace Squidex.Infrastructure.Migrations foreach (var migration in migrations) { - var name = migration.GetType().ToString(); + var name = migration.ToString()!; log.LogInformation(w => w .WriteProperty("action", "Migration") diff --git a/backend/src/Squidex/Config/Domain/StoreServices.cs b/backend/src/Squidex/Config/Domain/StoreServices.cs index 0ea1b907e..44d7da8b5 100644 --- a/backend/src/Squidex/Config/Domain/StoreServices.cs +++ b/backend/src/Squidex/Config/Domain/StoreServices.cs @@ -72,6 +72,9 @@ namespace Squidex.Config.Domain services.AddTransientAs() .As(); + services.AddTransientAs(c => new ConvertDocumentIds(GetDatabase(c, mongoDatabaseName), GetDatabase(c, mongoContentDatabaseName))) + .As(); + services.AddTransientAs() .As(); diff --git a/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs index 0cea7973e..cd49e3fd5 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs @@ -112,7 +112,7 @@ namespace Squidex.Infrastructure.Commands A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count() == 1))) .MustHaveHappened(); A.CallTo(() => persistence.ReadAsync(A._)) - .MustHaveHappened(); + .MustNotHaveHappened(); Assert.True(result is EntityCreatedResult); @@ -136,7 +136,7 @@ namespace Squidex.Infrastructure.Commands A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count() == 1))) .MustHaveHappened(); A.CallTo(() => persistence.ReadAsync(A._)) - .MustHaveHappened(); + .MustNotHaveHappened(); Assert.True(result is EntitySavedResult); @@ -230,7 +230,18 @@ namespace Squidex.Infrastructure.Commands [Fact] public async Task Should_throw_exception_when_already_created() { - SetupCreated(4); + SetupEmpty(); + + A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + .Throws(new InconsistentStateException(4, EtagVersion.NotFound)); + + await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); + } + + [Fact] + public async Task Should_throw_exception_when_already_created_after_creation() + { + await sut.ExecuteAsync(new CreateAuto()); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); } diff --git a/backend/tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectTests.cs index 95ae73b2a..a2a2c732a 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Commands/LogSnapshotDomainObjectTests.cs @@ -165,7 +165,7 @@ namespace Squidex.Infrastructure.Commands A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count() == 1))) .MustHaveHappened(); A.CallTo(() => persistence.ReadAsync(A._)) - .MustHaveHappened(); + .MustNotHaveHappened(); Assert.True(result is EntityCreatedResult); @@ -189,7 +189,7 @@ namespace Squidex.Infrastructure.Commands A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count() == 1))) .MustHaveHappened(); A.CallTo(() => persistence.ReadAsync(A._)) - .MustHaveHappened(); + .MustNotHaveHappened(); Assert.True(result is EntitySavedResult); @@ -275,7 +275,18 @@ namespace Squidex.Infrastructure.Commands [Fact] public async Task Should_throw_exception_when_already_created() { - SetupCreated(4); + SetupEmpty(); + + A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + .Throws(new InconsistentStateException(4, EtagVersion.NotFound)); + + await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); + } + + [Fact] + public async Task Should_throw_exception_when_already_created_after_creation() + { + await sut.ExecuteAsync(new CreateAuto()); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); } diff --git a/frontend/app/framework/angular/image-source.directive.ts b/frontend/app/framework/angular/image-source.directive.ts index ed2fe2976..3f6f008f1 100644 --- a/frontend/app/framework/angular/image-source.directive.ts +++ b/frontend/app/framework/angular/image-source.directive.ts @@ -23,7 +23,8 @@ export class ImageSourceDirective extends ResourceOwner implements OnChanges, On public imageSource: string; @Input() - public retryCount = 10; + public retryCount = 0; + @Input() public layoutKey: string; @@ -136,7 +137,7 @@ export class ImageSourceDirective extends ResourceOwner implements OnChanges, On private retryLoadingImage() { this.loadRetries++; - if (this.loadRetries <= 3) { + if (this.loadRetries <= this.retryCount) { this.loadTimer = setTimeout(() => { this.loadQuery = MathHelper.guid();