From 3836c426172926f1015188716ecf0182fdda2be3 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Wed, 22 Jun 2022 14:57:14 +0200 Subject: [PATCH] Feature/experimental count collection (#890) * Always add scope. * Storage performance. * Storage improvements. * Fix tests. * Assets counts. * Fix for deletion. * Simplify deletion. * Fix bulk insert. * Added migration for total collection. * More flexible total calculation. * Fix reload. * Updates * Introduce persistence action. * Fix tests * Fix tests * Improve queries. * Improved mapping. * Temp * Revert count collection. * Easier count cache. --- backend/i18n/frontend_en.json | 2 + backend/i18n/frontend_it.json | 2 + backend/i18n/frontend_nl.json | 2 + backend/i18n/source/frontend_en.json | 2 + backend/src/Migrations/MigrationPath.cs | 2 +- .../Migrations/Migrations/CreateAssetSlugs.cs | 7 +- .../Contents/GeoJsonValue.cs | 1 - .../DefaultValues/DefaultValueExtensions.cs | 1 - .../ValueReferencesConverter.cs | 1 - .../Scripting/ContentWrapper/JsonMapper.cs | 1 - .../Assets/MongoAssetEntity.cs | 18 ++ .../Assets/MongoAssetFolderEntity.cs | 18 ++ ...ongoAssetFolderRepository_SnapshotStore.cs | 45 ++--- .../Assets/MongoAssetRepository.cs | 66 +++++--- .../MongoAssetRepository_SnapshotStore.cs | 47 ++---- .../Assets/Visitors/FindExtensions.cs | 16 +- .../Contents/MongoContentCollection.cs | 39 ++--- .../Contents/MongoContentEntity.cs | 81 +++++++++ .../MongoContentRepository_SnapshotStore.cs | 116 ++++--------- .../Contents/Operations/QueryByIds.cs | 15 +- .../Contents/Operations/QueryByQuery.cs | 71 +++++--- .../MongoCountCollection.cs | 93 ++++++++++ .../MongoCountEntity.cs | 25 +++ .../Assets/Queries/AssetQueryParser.cs | 4 + .../Assets/Transformations.cs | 3 +- .../Contents/ContentOptions.cs | 2 + .../Contents/ContentsSearchSource.cs | 2 +- .../Contents/Queries/ContentQueryParser.cs | 4 + .../Queries/Steps/ResolveReferences.cs | 2 +- .../ContextExtensions.cs | 11 ++ backend/src/Squidex.Domain.Apps.Entities/Q.cs | 7 + .../Squidex.Domain.Users/DefaultKeyStore.cs | 6 +- .../DefaultXmlRepository.cs | 4 +- .../MongoDb/InstantSerializer.cs | 54 +++++- .../MongoDb/MongoExtensions.cs | 33 +++- .../States/MongoSnapshotStoreBase.cs | 22 +-- .../EventSourcing/EnvelopeExtensions.cs | 1 - .../InstantExtensions.cs | 5 + .../Json/Newtonsoft/JsonValueConverter.cs | 1 - .../Json/Objects/JsonObject.cs | 2 - .../Orleans/GrainState.cs | 2 +- .../States/BatchContext.cs | 12 +- .../States/BatchPersistence.cs | 12 +- .../States/IPersistence.cs | 12 +- .../States/ISnapshotStore.cs | 16 +- .../States/Persistence.cs | 39 +++-- .../Squidex/Config/Domain/AssetServices.cs | 2 - .../Config/Domain/EventSourcingServices.cs | 1 - .../Assets/MongoDb/AssetMappingTests.cs | 89 ++++++++++ .../Assets/MongoDb/AssetsQueryFixture.cs | 4 +- .../Contents/MongoDb/ContentMappingTests.cs | 159 ++++++++++++++++++ .../Properties/Resources.Designer.cs | 2 +- .../TestHelpers/HandlerTestBase.cs | 6 +- .../DefaultKeyStoreTests.cs | 16 +- .../DefaultXmlRepositoryTests.cs | 6 +- .../Commands/DomainObjectTests.cs | 64 +++---- .../MongoDb/Entities.cs | 50 ++++++ .../MongoDb/InstantSerializerTests.cs | 93 ++++++++++ .../States/PersistenceBatchTests.cs | 18 +- .../States/PersistenceEventSourcingTests.cs | 28 +-- .../States/PersistenceSnapshotTests.cs | 22 +-- .../framework/angular/pager.component.html | 28 +++ .../administration/state/users.state.ts | 2 +- .../assets/pages/assets-page.component.html | 2 +- .../assets/pages/assets-page.component.ts | 6 +- .../contents/contents-page.component.html | 2 +- .../pages/contents/contents-page.component.ts | 6 +- .../framework/angular/pager.component.html | 12 +- .../framework/angular/pager.component.scss | 6 +- .../framework/angular/pager.component.spec.ts | 27 +++ .../app/framework/angular/pager.component.ts | 20 ++- .../assets/assets-list.component.html | 2 +- .../assets/assets-list.component.ts | 4 + .../content-selector.component.html | 2 +- .../references/content-selector.component.ts | 4 + .../shared/services/assets.service.spec.ts | 61 ++++--- .../src/app/shared/services/assets.service.ts | 30 ++-- .../shared/services/contents.service.spec.ts | 39 ++++- .../app/shared/services/contents.service.ts | 61 +++---- .../src/app/shared/state/assets.state.spec.ts | 41 +++-- frontend/src/app/shared/state/assets.state.ts | 20 +-- .../src/app/shared/state/contents.state.ts | 26 +-- 82 files changed, 1374 insertions(+), 514 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountCollection.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountEntity.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetMappingTests.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentMappingTests.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/MongoDb/Entities.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/MongoDb/InstantSerializerTests.cs create mode 100644 frontend/app/framework/angular/pager.component.html diff --git a/backend/i18n/frontend_en.json b/backend/i18n/frontend_en.json index c4ec2974f..5666528c8 100644 --- a/backend/i18n/frontend_en.json +++ b/backend/i18n/frontend_en.json @@ -320,6 +320,8 @@ "common.openAPI": "Open API", "common.or": "or", "common.pagerInfo": "{itemFirst}-{itemLast} of {numberOfItems}", + "common.pagerInfoNoTotal": "{itemFirst}-{itemLast} of total?", + "common.pagerReload": "Click to reload view and get total number of items", "common.password": "Password", "common.passwordConfirm": "Confirm Password", "common.pattern": "Pattern", diff --git a/backend/i18n/frontend_it.json b/backend/i18n/frontend_it.json index 91afe9cfa..4a6911914 100644 --- a/backend/i18n/frontend_it.json +++ b/backend/i18n/frontend_it.json @@ -320,6 +320,8 @@ "common.openAPI": "Open API", "common.or": "o", "common.pagerInfo": "{itemFirst}-{itemLast} of {numberOfItems}", + "common.pagerInfoNoTotal": "{itemFirst}-{itemLast} of total?", + "common.pagerReload": "Click to reload view and get total number of items", "common.password": "Password", "common.passwordConfirm": "Conferma Password", "common.pattern": "Pattern", diff --git a/backend/i18n/frontend_nl.json b/backend/i18n/frontend_nl.json index ed9565696..73eafcb22 100644 --- a/backend/i18n/frontend_nl.json +++ b/backend/i18n/frontend_nl.json @@ -320,6 +320,8 @@ "common.openAPI": "Open API", "common.or": "of", "common.pagerInfo": "{itemFirst} - {itemLast} van {numberOfItems}", + "common.pagerInfoNoTotal": "{itemFirst}-{itemLast} of total?", + "common.pagerReload": "Click to reload view and get total number of items", "common.password": "Wachtwoord", "common.passwordConfirm": "Bevestig wachtwoord", "common.pattern": "Patroon", diff --git a/backend/i18n/source/frontend_en.json b/backend/i18n/source/frontend_en.json index c4ec2974f..5666528c8 100644 --- a/backend/i18n/source/frontend_en.json +++ b/backend/i18n/source/frontend_en.json @@ -320,6 +320,8 @@ "common.openAPI": "Open API", "common.or": "or", "common.pagerInfo": "{itemFirst}-{itemLast} of {numberOfItems}", + "common.pagerInfoNoTotal": "{itemFirst}-{itemLast} of total?", + "common.pagerReload": "Click to reload view and get total number of items", "common.password": "Password", "common.passwordConfirm": "Confirm Password", "common.pattern": "Pattern", diff --git a/backend/src/Migrations/MigrationPath.cs b/backend/src/Migrations/MigrationPath.cs index 2784c5ba8..acd61401e 100644 --- a/backend/src/Migrations/MigrationPath.cs +++ b/backend/src/Migrations/MigrationPath.cs @@ -15,7 +15,7 @@ namespace Migrations { public sealed class MigrationPath : IMigrationPath { - private const int CurrentVersion = 26; + private const int CurrentVersion = 25; private readonly IServiceProvider serviceProvider; public MigrationPath(IServiceProvider serviceProvider) diff --git a/backend/src/Migrations/Migrations/CreateAssetSlugs.cs b/backend/src/Migrations/Migrations/CreateAssetSlugs.cs index 4eba02826..91469b946 100644 --- a/backend/src/Migrations/Migrations/CreateAssetSlugs.cs +++ b/backend/src/Migrations/Migrations/CreateAssetSlugs.cs @@ -7,7 +7,6 @@ using Squidex.Domain.Apps.Entities.Assets; using Squidex.Domain.Apps.Entities.Assets.DomainObject; -using Squidex.Infrastructure; using Squidex.Infrastructure.Migrations; using Squidex.Infrastructure.States; @@ -25,13 +24,13 @@ namespace Migrations.Migrations public async Task UpdateAsync( CancellationToken ct) { - await foreach (var (state, version) in stateForAssets.ReadAllAsync(ct)) + await foreach (var (key, state, version, _) in stateForAssets.ReadAllAsync(ct)) { state.Slug = state.FileName.ToAssetSlug(); - var key = DomainId.Combine(state.AppId.Id, state.Id); + var job = new SnapshotWriteJob(key, state, version); - await stateForAssets.WriteAsync(key, state, version, version, ct); + await stateForAssets.WriteAsync(job, ct); } } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs index e005d0b5c..f3e174205 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/GeoJsonValue.cs @@ -8,7 +8,6 @@ using GeoJSON.Net; using GeoJSON.Net.Geometry; using Squidex.Infrastructure; -using Squidex.Infrastructure.Collections; using Squidex.Infrastructure.Json; using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.ObjectPool; diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/DefaultValues/DefaultValueExtensions.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/DefaultValues/DefaultValueExtensions.cs index 4bba3dad5..669ae76b5 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/DefaultValues/DefaultValueExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/DefaultValues/DefaultValueExtensions.cs @@ -9,7 +9,6 @@ using NodaTime; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Core.Schemas; using Squidex.Infrastructure; -using Squidex.Infrastructure.Json.Objects; namespace Squidex.Domain.Apps.Core.DefaultValues { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/ExtractReferenceIds/ValueReferencesConverter.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/ExtractReferenceIds/ValueReferencesConverter.cs index b4d2e29b4..f9358a415 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/ExtractReferenceIds/ValueReferencesConverter.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/ExtractReferenceIds/ValueReferencesConverter.cs @@ -7,7 +7,6 @@ using Squidex.Domain.Apps.Core.ConvertContent; using Squidex.Infrastructure; -using Squidex.Infrastructure.Json.Objects; namespace Squidex.Domain.Apps.Core.ExtractReferenceIds { diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs index 446ba6d9a..156b9c37a 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Scripting/ContentWrapper/JsonMapper.cs @@ -10,7 +10,6 @@ using Jint; using Jint.Native; using Jint.Native.Object; using Squidex.Infrastructure; -using Squidex.Infrastructure.Collections; using Squidex.Infrastructure.Json.Objects; namespace Squidex.Domain.Apps.Core.Scripting.ContentWrapper diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetEntity.cs index 1a60960a9..04797ca5d 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetEntity.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetEntity.cs @@ -9,8 +9,11 @@ using MongoDB.Bson.Serialization.Attributes; using NodaTime; using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; +using Squidex.Infrastructure.Reflection; +using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { @@ -114,5 +117,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { get => DocumentId; } + + public AssetDomainObject.State ToState() + { + return SimpleMapper.Map(this, new AssetDomainObject.State()); + } + + public static MongoAssetEntity Create(SnapshotWriteJob job) + { + var entity = SimpleMapper.Map(job.Value, new MongoAssetEntity()); + + entity.DocumentId = job.Key; + entity.IndexedAppId = job.Value.AppId.Id; + + return entity; + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderEntity.cs index 3dc79a7e3..eb6ae6d49 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderEntity.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderEntity.cs @@ -8,8 +8,11 @@ using MongoDB.Bson.Serialization.Attributes; using NodaTime; using Squidex.Domain.Apps.Entities.Assets; +using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; +using Squidex.Infrastructure.Reflection; +using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { @@ -67,5 +70,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { get => DocumentId; } + + public AssetFolderDomainObject.State ToState() + { + return SimpleMapper.Map(this, new AssetFolderDomainObject.State()); + } + + public static MongoAssetFolderEntity Create(SnapshotWriteJob job) + { + var entity = SimpleMapper.Map(job.Value, new MongoAssetFolderEntity()); + + entity.DocumentId = job.Key; + entity.IndexedAppId = job.Value.AppId.Id; + + return entity; + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs index 0ba263993..fb25f4bc8 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs @@ -11,7 +11,6 @@ using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; -using Squidex.Infrastructure.Reflection; using Squidex.Infrastructure.States; #pragma warning disable MA0048 // File name must match type name @@ -26,13 +25,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets return Collection.DeleteManyAsync(Filter.Eq(x => x.IndexedAppId, app.Id), ct); } - IAsyncEnumerable<(AssetFolderDomainObject.State State, long Version)> ISnapshotStore.ReadAllAsync( + IAsyncEnumerable> ISnapshotStore.ReadAllAsync( CancellationToken ct) { - return Collection.Find(new BsonDocument(), Batching.Options).ToAsyncEnumerable(ct).Select(x => (Map(x), x.Version)); + return Collection.Find(new BsonDocument(), Batching.Options).ToAsyncEnumerable(ct) + .Select(x => new SnapshotResult(x.DocumentId, x.ToState(), x.Version, true)); } - async Task<(AssetFolderDomainObject.State Value, bool Valid, long Version)> ISnapshotStore.ReadAsync(DomainId key, + async Task> ISnapshotStore.ReadAsync(DomainId key, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/ReadAsync")) @@ -43,30 +43,30 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets if (existing != null) { - return (Map(existing), true, existing.Version); + return new SnapshotResult(existing.DocumentId, existing.ToState(), existing.Version); } - return (null!, true, EtagVersion.Empty); + return new SnapshotResult(default, null!, EtagVersion.Empty); } } - async Task ISnapshotStore.WriteAsync(DomainId key, AssetFolderDomainObject.State value, long oldVersion, long newVersion, + async Task ISnapshotStore.WriteAsync(SnapshotWriteJob job, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/WriteAsync")) { - var entity = Map(value); + var entity = MongoAssetFolderEntity.Create(job); - await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, entity, ct); + await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct); } } - async Task ISnapshotStore.WriteManyAsync(IEnumerable<(DomainId Key, AssetFolderDomainObject.State Value, long Version)> snapshots, + async Task ISnapshotStore.WriteManyAsync(IEnumerable> jobs, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/WriteManyAsync")) { - var updates = snapshots.Select(Map).Select(x => + var updates = jobs.Select(MongoAssetFolderEntity.Create).Select(x => new ReplaceOneModel( Filter.Eq(y => y.DocumentId, x.DocumentId), x) @@ -91,28 +91,5 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets await Collection.DeleteOneAsync(x => x.DocumentId == key, ct); } } - - private static MongoAssetFolderEntity Map(AssetFolderDomainObject.State value) - { - var entity = SimpleMapper.Map(value, new MongoAssetFolderEntity()); - - entity.IndexedAppId = value.AppId.Id; - - return entity; - } - - private static MongoAssetFolderEntity Map((DomainId Key, AssetFolderDomainObject.State Value, long Version) snapshot) - { - var entity = Map(snapshot.Value); - - entity.DocumentId = snapshot.Key; - - return entity; - } - - private static AssetFolderDomainObject.State Map(MongoAssetFolderEntity existing) - { - return SimpleMapper.Map(existing, new AssetFolderDomainObject.State()); - } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index 9f73e65d8..847af8338 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -19,9 +19,12 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { public sealed partial class MongoAssetRepository : MongoRepositoryBase, IAssetRepository { + private readonly MongoCountCollection countCollection; + public MongoAssetRepository(IMongoDatabase database) : base(database) { + countCollection = new MongoCountCollection(database, CollectionName()); } public IMongoCollection GetInternalCollection() @@ -50,15 +53,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets new CreateIndexModel( Index .Ascending(x => x.IndexedAppId) - .Ascending(x => x.IsDeleted) .Ascending(x => x.Slug)), new CreateIndexModel( Index .Ascending(x => x.IndexedAppId) - .Ascending(x => x.IsDeleted) - .Ascending(x => x.FileHash) - .Ascending(x => x.FileName) - .Ascending(x => x.FileSize)), + .Ascending(x => x.FileHash)), new CreateIndexModel( Index .Ascending(x => x.Id) @@ -101,13 +100,16 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets .ToListAsync(ct); long assetTotal = assetEntities.Count; - if (q.NoTotal) + if (assetEntities.Count >= q.Query.Take || q.Query.Skip > 0) { - assetTotal = -1; - } - else if (assetEntities.Count >= q.Query.Take || q.Query.Skip > 0) - { - assetTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + if (q.NoTotal) + { + assetTotal = -1; + } + else + { + assetTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + } } return ResultList.Create(assetTotal, assetEntities.OfType()); @@ -116,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { var query = q.Query.AdjustToModel(appId); - var filter = query.BuildFilter(appId, parentId); + var (filter, isDefault) = query.BuildFilter(appId, parentId); var assetEntities = await Collection.Find(filter) @@ -126,13 +128,25 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets .ToListAsync(ct); long assetTotal = assetEntities.Count; - if (q.NoTotal) + if (assetEntities.Count >= q.Query.Take || q.Query.Skip > 0) { - assetTotal = -1; - } - else if (assetEntities.Count >= q.Query.Take || q.Query.Skip > 0) - { - assetTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + var isDefaultQuery = q.Query.Filter == null; + + if (q.NoTotal || (q.NoSlowTotal && !isDefaultQuery)) + { + assetTotal = -1; + } + else if (isDefaultQuery) + { + // Cache total count by app and folder. + var totalKey = parentId.HasValue ? DomainId.Combine(appId, parentId.Value) : appId; + + assetTotal = await countCollection.GetOrAddAsync(totalKey, ct => Collection.Find(filter).CountDocumentsAsync(ct), ct); + } + else + { + assetTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + } } return ResultList.Create(assetTotal, assetEntities); @@ -166,7 +180,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets using (Telemetry.Activities.StartActivity("MongoAssetRepository/QueryChildIdsAsync")) { var assetEntities = - await Collection.Find(x => x.IndexedAppId == appId && !x.IsDeleted && x.ParentId == parentId).Only(x => x.Id) + await Collection.Find(BuildFilter(appId, parentId)).Only(x => x.Id) .ToListAsync(ct); var field = Field.Of(x => nameof(x.Id)); @@ -181,7 +195,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets using (Telemetry.Activities.StartActivity("MongoAssetRepository/FindAssetByHashAsync")) { var assetEntity = - await Collection.Find(x => x.IndexedAppId == appId && !x.IsDeleted && x.FileHash == hash && x.FileName == fileName && x.FileSize == fileSize) + await Collection.Find(x => x.IndexedAppId == appId && x.FileHash == hash && !x.IsDeleted && x.FileSize == fileSize && x.FileName == fileName) .FirstOrDefaultAsync(ct); return assetEntity; @@ -194,7 +208,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets using (Telemetry.Activities.StartActivity("MongoAssetRepository/FindAssetBySlugAsync")) { var assetEntity = - await Collection.Find(x => x.IndexedAppId == appId && !x.IsDeleted && x.Slug == slug) + await Collection.Find(x => x.IndexedAppId == appId && x.Slug == slug && !x.IsDeleted) .FirstOrDefaultAsync(ct); return assetEntity; @@ -237,5 +251,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets Filter.In(x => x.DocumentId, documentIds), Filter.Ne(x => x.IsDeleted, true)); } + + private static FilterDefinition BuildFilter(DomainId appId, DomainId parentId) + { + return Filter.And( + Filter.Gt(x => x.LastModified, default), + Filter.Gt(x => x.Id, DomainId.Create(string.Empty)), + Filter.Gt(x => x.IndexedAppId, appId), + Filter.Ne(x => x.IsDeleted, true), + Filter.Ne(x => x.ParentId, parentId)); + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs index 6042de55c..44f355836 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs @@ -11,7 +11,6 @@ using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; -using Squidex.Infrastructure.Reflection; using Squidex.Infrastructure.States; #pragma warning disable MA0048 // File name must match type name @@ -26,13 +25,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets return Collection.DeleteManyAsync(Filter.Eq(x => x.IndexedAppId, app.Id), ct); } - IAsyncEnumerable<(AssetDomainObject.State State, long Version)> ISnapshotStore.ReadAllAsync( + IAsyncEnumerable> ISnapshotStore.ReadAllAsync( CancellationToken ct) { - return Collection.Find(new BsonDocument(), Batching.Options).ToAsyncEnumerable(ct).Select(x => (Map(x), x.Version)); + return Collection.Find(new BsonDocument(), Batching.Options).ToAsyncEnumerable(ct) + .Select(x => new SnapshotResult(x.DocumentId, x.ToState(), x.Version)); } - async Task<(AssetDomainObject.State Value, bool Valid, long Version)> ISnapshotStore.ReadAsync(DomainId key, + async Task> ISnapshotStore.ReadAsync(DomainId key, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetRepository/ReadAsync")) @@ -43,30 +43,30 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets if (existing != null) { - return (Map(existing), true, existing.Version); + return new SnapshotResult(existing.DocumentId, existing.ToState(), existing.Version); } - return (null!, true, EtagVersion.Empty); + return new SnapshotResult(default, null!, EtagVersion.Empty); } } - async Task ISnapshotStore.WriteAsync(DomainId key, AssetDomainObject.State value, long oldVersion, long newVersion, + async Task ISnapshotStore.WriteAsync(SnapshotWriteJob job, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetRepository/WriteAsync")) { - var entity = Map(value); + var entity = MongoAssetEntity.Create(job); - await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, entity, ct); + await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct); } } - async Task ISnapshotStore.WriteManyAsync(IEnumerable<(DomainId Key, AssetDomainObject.State Value, long Version)> snapshots, + async Task ISnapshotStore.WriteManyAsync(IEnumerable> jobs, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoAssetRepository/WriteManyAsync")) { - var updates = snapshots.Select(Map).Select(x => + var updates = jobs.Select(MongoAssetEntity.Create).Select(x => new ReplaceOneModel( Filter.Eq(y => y.DocumentId, x.DocumentId), x) @@ -88,31 +88,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { using (Telemetry.Activities.StartActivity("MongoAssetRepository/RemoveAsync")) { - await Collection.DeleteOneAsync(x => x.DocumentId == key, ct); + await Collection.DeleteOneAsync(x => x.DocumentId == key, null, ct); } } - - private static MongoAssetEntity Map(AssetDomainObject.State value) - { - var entity = SimpleMapper.Map(value, new MongoAssetEntity()); - - entity.IndexedAppId = value.AppId.Id; - - return entity; - } - - private static MongoAssetEntity Map((DomainId Key, AssetDomainObject.State Value, long Version) snapshot) - { - var entity = Map(snapshot.Value); - - entity.DocumentId = snapshot.Key; - - return entity; - } - - private static AssetDomainObject.State Map(MongoAssetEntity existing) - { - return SimpleMapper.Map(existing, new AssetDomainObject.State()); - } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs index 7cfb20942..f506ca20b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/Visitors/FindExtensions.cs @@ -36,18 +36,22 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets.Visitors return query; } - public static FilterDefinition BuildFilter(this ClrQuery query, DomainId appId, DomainId? parentId) + public static (FilterDefinition, bool) BuildFilter(this ClrQuery query, DomainId appId, DomainId? parentId) { var filters = new List> { - Filter.Exists(x => x.LastModified), - Filter.Exists(x => x.Id), + Filter.Ne(x => x.LastModified, default), + Filter.Ne(x => x.Id, default), Filter.Eq(x => x.IndexedAppId, appId) }; + var isDefault = false; + if (!query.HasFilterField("IsDeleted")) { filters.Add(Filter.Eq(x => x.IsDeleted, false)); + + isDefault = true; } if (parentId != null) @@ -63,12 +67,16 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets.Visitors { filters.Add(Filter.Eq(x => x.ParentId, parentId.Value)); } + + isDefault = false; } var (filter, last) = query.BuildFilter(false); if (filter != null) { + isDefault = false; + if (last) { filters.Add(filter); @@ -79,7 +87,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets.Visitors } } - return Filter.And(filters); + return (Filter.And(filters), isDefault); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index 778caa228..8a79b6781 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using MongoDB.Bson; using MongoDB.Driver; using NodaTime; using Squidex.Domain.Apps.Core.Contents; @@ -27,8 +28,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents private readonly QueryReferences queryReferences; private readonly QueryReferrers queryReferrers; private readonly QueryScheduled queryScheduled; - private readonly string name; private readonly ReadPreference readPreference; + private readonly string name; public MongoContentCollection(string name, IMongoDatabase database, IAppProvider appProvider, ReadPreference readPreference) : base(database) @@ -38,7 +39,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents queryAsStream = new QueryAsStream(); queryBdId = new QueryById(); queryByIds = new QueryByIds(); - queryByQuery = new QueryByQuery(appProvider); + queryByQuery = new QueryByQuery(appProvider, new MongoCountCollection(database, name)); queryReferences = new QueryReferences(queryByIds); queryReferrers = new QueryReferrers(); queryScheduled = new QueryScheduled(); @@ -202,42 +203,34 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } - public async Task FindVersionAsync(DomainId documentId, + public Task FindAsync(DomainId documentId, CancellationToken ct = default) { - var result = await Collection.Find(x => x.DocumentId == documentId).Only(x => x.Version).FirstOrDefaultAsync(ct); + return Collection.Find(x => x.DocumentId == documentId).FirstOrDefaultAsync(ct); + } - return result?["vs"].AsInt64 ?? EtagVersion.Empty; + public IAsyncEnumerable StreamAll( + CancellationToken ct) + { + return Collection.Find(new BsonDocument()).ToAsyncEnumerable(ct); } - public Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity entity, + public Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity value, CancellationToken ct = default) { - return Collection.UpsertVersionedAsync(documentId, oldVersion, entity.Version, entity, ct); + return Collection.UpsertVersionedAsync(documentId, oldVersion, value.Version, value, ct); } - public Task RemoveAsync(DomainId documentId, + public Task RemoveAsync(DomainId key, CancellationToken ct = default) { - return Collection.DeleteOneAsync(x => x.DocumentId == documentId, ct); + return Collection.DeleteOneAsync(x => x.DocumentId == key, null, ct); } - public Task InsertManyAsync(IReadOnlyList entities, + public Task InsertManyAsync(IReadOnlyList snapshots, CancellationToken ct = default) { - if (entities.Count == 0) - { - return Task.CompletedTask; - } - - var writes = entities.Select(x => new ReplaceOneModel( - Filter.Eq(y => y.DocumentId, x.DocumentId), - x) - { - IsUpsert = true - }).ToList(); - - return Collection.BulkWriteAsync(writes, BulkUnordered, ct); + return Collection.InsertManyAsync(snapshots, InsertUnordered, ct); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentEntity.cs index 78e244bf6..5bd38e06a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentEntity.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentEntity.cs @@ -8,9 +8,13 @@ using MongoDB.Bson.Serialization.Attributes; using NodaTime; using Squidex.Domain.Apps.Core.Contents; +using Squidex.Domain.Apps.Core.ExtractReferenceIds; using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.DomainObject; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; +using Squidex.Infrastructure.Reflection; +using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { @@ -58,6 +62,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents [BsonJson] public ContentData Data { get; set; } + [BsonIgnoreIfNull] + [BsonElement("dd")] + [BsonJson] + public ContentData? DraftData { get; set; } + [BsonIgnoreIfNull] [BsonElement("sa")] public Instant? ScheduledAt { get; set; } @@ -78,6 +87,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents [BsonElement("dl")] public bool IsDeleted { get; set; } + [BsonIgnoreIfDefault] + [BsonElement("is")] + public bool IsSnapshot { get; set; } + [BsonIgnoreIfDefault] [BsonElement("sj")] public ScheduleJob? ScheduleJob { get; set; } @@ -94,5 +107,73 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { get => DocumentId; } + + public ContentDomainObject.State ToState() + { + var state = SimpleMapper.Map(this, new ContentDomainObject.State()); + + if (DraftData != null && NewStatus.HasValue) + { + state.NewVersion = new ContentVersion(NewStatus.Value, Data); + state.CurrentVersion = new ContentVersion(Status, DraftData); + } + else + { + state.NewVersion = null; + state.CurrentVersion = new ContentVersion(Status, Data); + } + + return state; + } + + public static async Task CreatePublishedAsync(SnapshotWriteJob job, IAppProvider appProvider) + { + var entity = await CreateContentAsync(job.Value.CurrentVersion.Data, job, appProvider); + + entity.ScheduledAt = null; + entity.ScheduleJob = null; + entity.NewStatus = null; + + return entity; + } + + public static async Task CreateDraftAsync(SnapshotWriteJob job, IAppProvider appProvider) + { + var entity = await CreateContentAsync(job.Value.Data, job, appProvider); + + entity.ScheduledAt = job.Value.ScheduleJob?.DueTime; + entity.ScheduleJob = job.Value.ScheduleJob; + entity.NewStatus = job.Value.NewStatus; + entity.DraftData = job.Value.NewVersion != null ? job.Value.CurrentVersion.Data : null; + entity.IsSnapshot = true; + + return entity; + } + + private static async Task CreateContentAsync(ContentData data, SnapshotWriteJob job, IAppProvider appProvider) + { + var entity = SimpleMapper.Map(job.Value, new MongoContentEntity()); + + entity.Data = data; + entity.DocumentId = job.Value.UniqueId; + entity.IndexedAppId = job.Value.AppId.Id; + entity.IndexedSchemaId = job.Value.SchemaId.Id; + entity.ReferencedIds ??= new HashSet(); + entity.Version = job.NewVersion; + + if (data.CanHaveReference()) + { + var schema = await appProvider.GetSchemaAsync(job.Value.AppId.Id, job.Value.SchemaId.Id, true); + + if (schema != null) + { + var components = await appProvider.GetComponentsAsync(schema); + + entity.Data.AddReferencedIds(schema.SchemaDef, entity.ReferencedIds, components); + } + } + + return entity; + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs index 3de39a9a0..d9319a31b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs @@ -6,11 +6,9 @@ // ========================================================================== using Squidex.Domain.Apps.Core.Contents; -using Squidex.Domain.Apps.Core.ExtractReferenceIds; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Contents.DomainObject; using Squidex.Infrastructure; -using Squidex.Infrastructure.Reflection; using Squidex.Infrastructure.States; #pragma warning disable MA0048 // File name must match type name @@ -19,20 +17,27 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { public partial class MongoContentRepository : ISnapshotStore, IDeleter { - IAsyncEnumerable<(ContentDomainObject.State State, long Version)> ISnapshotStore.ReadAllAsync( + IAsyncEnumerable> ISnapshotStore.ReadAllAsync( CancellationToken ct) { - return AsyncEnumerable.Empty<(ContentDomainObject.State State, long Version)>(); + return collectionAll.StreamAll(ct) + .Select(x => new SnapshotResult(x.DocumentId, x.ToState(), x.Version, true)); } - async Task<(ContentDomainObject.State Value, bool Valid, long Version)> ISnapshotStore.ReadAsync(DomainId key, + async Task> ISnapshotStore.ReadAsync(DomainId key, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoContentRepository/ReadAsync")) { - var version = await collectionAll.FindVersionAsync(key, ct); + var existing = + await collectionAll.FindAsync(key, ct); - return (null!, false, version); + if (existing?.IsSnapshot == true) + { + return new SnapshotResult(existing.DocumentId, existing.ToState(), existing.Version); + } + + return new SnapshotResult(default, null!, EtagVersion.Empty); } } @@ -66,23 +71,23 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } - async Task ISnapshotStore.WriteAsync(DomainId key, ContentDomainObject.State value, long oldVersion, long newVersion, + async Task ISnapshotStore.WriteAsync(SnapshotWriteJob job, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteAsync")) { - if (value.SchemaId.Id == DomainId.Empty) + if (job.Value.SchemaId.Id == DomainId.Empty) { return; } await Task.WhenAll( - UpsertDraftContentAsync(value, oldVersion, newVersion, ct), - UpsertOrDeletePublishedAsync(value, oldVersion, newVersion, ct)); + UpsertDraftContentAsync(job, ct), + UpsertOrDeletePublishedAsync(job, ct)); } } - async Task ISnapshotStore.WriteManyAsync(IEnumerable<(DomainId Key, ContentDomainObject.State Value, long Version)> snapshots, + async Task ISnapshotStore.WriteManyAsync(IEnumerable> jobs, CancellationToken ct) { using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteManyAsync")) @@ -90,20 +95,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents var entitiesPublished = new List(); var entitiesAll = new List(); - foreach (var (_, value, version) in snapshots) + foreach (var job in jobs.Where(IsValid)) { - // Some data is corrupt and might throw an exception during migration if we do not skip them. - if (value.AppId == null || value.CurrentVersion == null) - { - continue; - } - - if (ShouldWritePublished(value)) + if (ShouldWritePublished(job.Value)) { - entitiesPublished.Add(await CreatePublishedContentAsync(value, version)); + entitiesPublished.Add(await MongoContentEntity.CreatePublishedAsync(job, appProvider)); } - entitiesAll.Add(await CreateDraftContentAsync(value, version)); + entitiesAll.Add(await MongoContentEntity.CreateDraftAsync(job, appProvider)); } await Task.WhenAll( @@ -112,16 +111,16 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } - private async Task UpsertOrDeletePublishedAsync(ContentDomainObject.State value, long oldVersion, long newVersion, + private async Task UpsertOrDeletePublishedAsync(SnapshotWriteJob job, CancellationToken ct = default) { - if (ShouldWritePublished(value)) + if (ShouldWritePublished(job.Value)) { - await UpsertPublishedContentAsync(value, oldVersion, newVersion, ct); + await UpsertPublishedContentAsync(job, ct); } else { - await DeletePublishedContentAsync(value.AppId.Id, value.Id, ct); + await DeletePublishedContentAsync(job.Value.AppId.Id, job.Value.Id, ct); } } @@ -133,73 +132,32 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return collectionPublished.RemoveAsync(documentId, ct); } - private async Task UpsertDraftContentAsync(ContentDomainObject.State value, long oldVersion, long newVersion, + private async Task UpsertDraftContentAsync(SnapshotWriteJob job, CancellationToken ct = default) { - var entity = await CreateDraftContentAsync(value, newVersion); + var entity = await MongoContentEntity.CreateDraftAsync(job, appProvider); - await collectionAll.UpsertVersionedAsync(entity.DocumentId, oldVersion, entity, ct); + await collectionAll.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct); } - private async Task UpsertPublishedContentAsync(ContentDomainObject.State value, long oldVersion, long newVersion, + private async Task UpsertPublishedContentAsync(SnapshotWriteJob job, CancellationToken ct = default) { - var entity = await CreatePublishedContentAsync(value, newVersion); + var entity = await MongoContentEntity.CreatePublishedAsync(job, appProvider); - await collectionPublished.UpsertVersionedAsync(entity.DocumentId, oldVersion, entity, ct); + await collectionPublished.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct); } - private async Task CreatePublishedContentAsync(ContentDomainObject.State value, long newVersion) - { - var entity = await CreateContentAsync(value, value.CurrentVersion.Data, newVersion); - - entity.ScheduledAt = null; - entity.ScheduleJob = null; - entity.NewStatus = null; - - return entity; - } - - private async Task CreateDraftContentAsync(ContentDomainObject.State value, long newVersion) - { - var entity = await CreateContentAsync(value, value.Data, newVersion); - - entity.ScheduledAt = value.ScheduleJob?.DueTime; - entity.ScheduleJob = value.ScheduleJob; - entity.NewStatus = value.NewStatus; - - return entity; - } - - private async Task CreateContentAsync(ContentDomainObject.State value, ContentData data, long newVersion) + private static bool ShouldWritePublished(ContentDomainObject.State value) { - var entity = SimpleMapper.Map(value, new MongoContentEntity()); - - entity.Data = data; - entity.DocumentId = value.UniqueId; - entity.IndexedAppId = value.AppId.Id; - entity.IndexedSchemaId = value.SchemaId.Id; - entity.ReferencedIds ??= new HashSet(); - entity.Version = newVersion; - - if (data.CanHaveReference()) - { - var schema = await appProvider.GetSchemaAsync(value.AppId.Id, value.SchemaId.Id, true); - - if (schema != null) - { - var components = await appProvider.GetComponentsAsync(schema); - - entity.Data.AddReferencedIds(schema.SchemaDef, entity.ReferencedIds, components); - } - } - - return entity; + // Only published content is written to the published collection. + return value.Status == Status.Published && !value.IsDeleted; } - private static bool ShouldWritePublished(ContentDomainObject.State value) + private static bool IsValid(SnapshotWriteJob job) { - return value.Status == Status.Published && !value.IsDeleted; + // Some data is corrupt and might throw an exception during migration if we do not skip them. + return job.Value.AppId != null || job.Value.CurrentVersion != null; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByIds.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByIds.cs index e48c8a11f..2d17f8c88 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByIds.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByIds.cs @@ -47,13 +47,16 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations var contentEntities = await FindContentsAsync(q.Query, filter); var contentTotal = (long)contentEntities.Count; - if (q.NoTotal) + if (contentTotal >= q.Query.Take || q.Query.Skip > 0) { - contentTotal = -1; - } - else if (contentTotal >= q.Query.Take || q.Query.Skip > 0) - { - contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + if (q.NoTotal) + { + contentTotal = -1; + } + else + { + contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + } } return ResultList.Create(contentTotal, contentEntities); diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs index db95850e8..61467ae99 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryByQuery.cs @@ -21,6 +21,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations internal sealed class QueryByQuery : OperationBase { private readonly IAppProvider appProvider; + private readonly MongoCountCollection countCollection; [BsonIgnoreExtraElements] internal sealed class IdOnly @@ -32,9 +33,10 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations public MongoContentEntity[] Joined { get; set; } } - public QueryByQuery(IAppProvider appProvider) + public QueryByQuery(IAppProvider appProvider, MongoCountCollection countCollection) { this.appProvider = appProvider; + this.countCollection = countCollection; } public override IEnumerable> CreateIndexes() @@ -93,18 +95,25 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations { var query = q.Query.AdjustToModel(app.Id); - var filter = CreateFilter(app.Id, schemas.Select(x => x.Id), query, q.Reference, q.CreatedBy); + var (filter, isDefault) = CreateFilter(app.Id, schemas.Select(x => x.Id), query, q.Reference, q.CreatedBy); var contentEntities = await FindContentsAsync(query, filter, ct); var contentTotal = (long)contentEntities.Count; - if (q.NoTotal) + if (contentTotal >= q.Query.Take || q.Query.Skip > 0) { - contentTotal = -1; - } - else if (contentTotal >= q.Query.Take || q.Query.Skip > 0) - { - contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + if (q.NoTotal || (q.NoSlowTotal && q.Query.Filter != null)) + { + contentTotal = -1; + } + else if (IsSatisfiedByIndex(query)) + { + contentTotal = await Collection.Find(filter).QuerySort(query).CountDocumentsAsync(ct); + } + else + { + contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + } } return ResultList.Create(contentTotal, contentEntities); @@ -130,18 +139,32 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations { var query = q.Query.AdjustToModel(app.Id); - var filter = CreateFilter(schema.AppId.Id, Enumerable.Repeat(schema.Id, 1), query, q.Reference, q.CreatedBy); + var (filter, isDefault) = CreateFilter(schema.AppId.Id, Enumerable.Repeat(schema.Id, 1), query, q.Reference, q.CreatedBy); var contentEntities = await FindContentsAsync(query, filter, ct); var contentTotal = (long)contentEntities.Count; - if (q.NoTotal) - { - contentTotal = -1; - } - else if (contentTotal >= q.Query.Take || q.Query.Skip > 0) + if (contentTotal >= q.Query.Take || q.Query.Skip > 0) { - contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + if (q.NoTotal || (q.NoSlowTotal && q.Query.Filter != null)) + { + contentTotal = -1; + } + else if (isDefault) + { + // Cache total count by app and schema. + var totalKey = DomainId.Combine(app.Id, schema.Id); + + contentTotal = await countCollection.GetOrAddAsync(totalKey, ct => Collection.Find(filter).CountDocumentsAsync(ct), ct); + } + else if (IsSatisfiedByIndex(query)) + { + contentTotal = await Collection.Find(filter).QuerySort(query).CountDocumentsAsync(ct); + } + else + { + contentTotal = await Collection.Find(filter).CountDocumentsAsync(ct); + } } return ResultList.Create(contentTotal, contentEntities); @@ -224,38 +247,48 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations return Filter.And(filters); } - private static FilterDefinition CreateFilter(DomainId appId, IEnumerable schemaIds, ClrQuery? query, + private static (FilterDefinition, bool) CreateFilter(DomainId appId, IEnumerable schemaIds, ClrQuery? query, DomainId referenced, RefToken? createdBy) { var filters = new List> { - Filter.Exists(x => x.LastModified), - Filter.Exists(x => x.Id), + Filter.Gt(x => x.LastModified, default), + Filter.Gt(x => x.Id, default), Filter.Eq(x => x.IndexedAppId, appId), Filter.In(x => x.IndexedSchemaId, schemaIds) }; + var isDefault = false; + if (query?.HasFilterField("dl") != true) { filters.Add(Filter.Ne(x => x.IsDeleted, true)); + + isDefault = true; } if (query?.Filter != null) { filters.Add(query.Filter.BuildFilter()); + + isDefault = false; } if (referenced != default) { filters.Add(Filter.AnyEq(x => x.ReferencedIds, referenced)); + + isDefault = false; } if (createdBy != null) { filters.Add(Filter.Eq(x => x.CreatedBy, createdBy)); + + isDefault = false; } - return Filter.And(filters); + return (Filter.And(filters), isDefault); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountCollection.cs new file mode 100644 index 000000000..baf04f66e --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountCollection.cs @@ -0,0 +1,93 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Driver; +using NodaTime; +using Squidex.Infrastructure; +using Squidex.Infrastructure.MongoDb; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.MongoDb +{ + internal sealed class MongoCountCollection : MongoRepositoryBase + { + private readonly string name; + + public MongoCountCollection(IMongoDatabase database, string name) + : base(database) + { + this.name = $"{name}_Count"; + + InitializeAsync(default).Wait(); + } + + protected override string CollectionName() + { + return name; + } + + public Task GetOrAddAsync(DomainId key, Func> provider, + CancellationToken ct) + { + return GetOrAddAsync(key.ToString(), provider, ct); + } + + public async Task GetOrAddAsync(string key, Func> provider, + CancellationToken ct) + { + var (cachedTotal, isOutdated) = await CountAsync(key, ct); + + if (cachedTotal < 5_000) + { + return await RefreshTotalAsync(key, cachedTotal, provider, ct); + } + + if (isOutdated) + { + // If we have a loot of items, the query might be slow and therefore we execute it in the background. + RefreshTotalAsync(key, cachedTotal, provider, ct).Forget(); + } + + return cachedTotal; + } + + private async Task RefreshTotalAsync(string key, long cachedCount, Func> provider, + CancellationToken ct) + { + var actualCount = await provider(ct); + + if (actualCount != cachedCount) + { + var now = SystemClock.Instance.GetCurrentInstant(); + + await Collection.UpdateOneAsync(x => x.Key == key, + Update + .Set(x => x.Key, key) + .SetOnInsert(x => x.Count, actualCount) + .SetOnInsert(x => x.Created, now), + Upsert, ct); + } + + return actualCount; + } + + private async Task<(long, bool)> CountAsync(string key, + CancellationToken ct) + { + var entity = await Collection.Find(x => x.Key == key).FirstOrDefaultAsync(ct); + + if (entity != null) + { + var now = SystemClock.Instance.GetCurrentInstant(); + + return (entity.Count, now - entity.Created > Duration.FromSeconds(10)); + } + + return (0, true); + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountEntity.cs new file mode 100644 index 000000000..5e5b993e5 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/MongoCountEntity.cs @@ -0,0 +1,25 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Bson.Serialization.Attributes; +using NodaTime; + +namespace Squidex.Domain.Apps.Entities.MongoDb +{ + internal sealed class MongoCountEntity + { + [BsonId] + [BsonRequired] + public string Key { get; set; } + + [BsonElement] + public long Count { get; set; } + + [BsonElement] + public Instant Created { get; set; } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs index 919cb203d..c91257fca 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetQueryParser.cs @@ -57,6 +57,10 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries { q = q.WithoutTotal(); } + else if (context.ShouldSkipSlowTotal()) + { + q = q.WithoutSlowTotal(); + } return q; } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Transformations.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Transformations.cs index 187e3c8ee..6735c5dfc 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Transformations.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Transformations.cs @@ -77,7 +77,8 @@ namespace Squidex.Domain.Apps.Entities.Assets public static async Task GetBlurHashAsync(this AssetRef asset, BlurOptions options, IAssetFileStore assetFileStore, - IAssetThumbnailGenerator thumbnailGenerator, CancellationToken ct = default) + IAssetThumbnailGenerator thumbnailGenerator, + CancellationToken ct = default) { using (var stream = DefaultPools.MemoryStream.GetStream()) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentOptions.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentOptions.cs index f80cb8523..581f5f74b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentOptions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentOptions.cs @@ -11,6 +11,8 @@ namespace Squidex.Domain.Apps.Entities.Contents { public bool CanCache { get; set; } + public bool OptimizeTotal { get; set; } = true; + public int DefaultPageSize { get; set; } = 200; public int MaxResults { get; set; } = 200; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentsSearchSource.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentsSearchSource.cs index 70493da5a..3ccef20f6 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentsSearchSource.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentsSearchSource.cs @@ -64,7 +64,7 @@ namespace Squidex.Domain.Apps.Entities.Contents var appId = context.App.NamedId(); - var contents = await contentQuery.QueryAsync(context, Q.Empty.WithIds(ids), ct); + var contents = await contentQuery.QueryAsync(context, Q.Empty.WithIds(ids).WithoutTotal(), ct); foreach (var content in contents) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs index 182f605cb..8282d5329 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/ContentQueryParser.cs @@ -64,6 +64,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries { q = q.WithoutTotal(); } + else if (context.ShouldSkipSlowTotal()) + { + q = q.WithoutSlowTotal(); + } return q; } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/Steps/ResolveReferences.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/Steps/ResolveReferences.cs index 264c8522e..cd1ed5a9a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/Steps/ResolveReferences.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Queries/Steps/ResolveReferences.cs @@ -162,7 +162,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Queries.Steps .WithoutContentEnrichment(true) .WithoutTotal()); - var references = await ContentQuery.QueryAsync(queryContext, Q.Empty.WithIds(ids), ct); + var references = await ContentQuery.QueryAsync(queryContext, Q.Empty.WithIds(ids).WithoutTotal(), ct); return references.ToLookup(x => x.Id); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/ContextExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities/ContextExtensions.cs index de2dd9a72..d93719b2b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/ContextExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/ContextExtensions.cs @@ -10,6 +10,7 @@ namespace Squidex.Domain.Apps.Entities public static class ContextExtensions { private const string HeaderNoTotal = "X-NoTotal"; + private const string HeaderNoSlowTotal = "X-NoSlowTotal"; public static bool ShouldSkipTotal(this Context context) { @@ -21,6 +22,16 @@ namespace Squidex.Domain.Apps.Entities return builder.WithBoolean(HeaderNoTotal, value); } + public static bool ShouldSkipSlowTotal(this Context context) + { + return context.Headers.ContainsKey(HeaderNoSlowTotal); + } + + public static ICloneBuilder WithoutSlowTotal(this ICloneBuilder builder, bool value = true) + { + return builder.WithBoolean(HeaderNoSlowTotal, value); + } + public static ICloneBuilder WithBoolean(this ICloneBuilder builder, string key, bool value) { if (value) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Q.cs b/backend/src/Squidex.Domain.Apps.Entities/Q.cs index 8f9318ca6..52f12cd66 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Q.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Q.cs @@ -38,6 +38,8 @@ namespace Squidex.Domain.Apps.Entities public bool NoTotal { get; init; } + public bool NoSlowTotal { get; init; } + private Q() { } @@ -54,6 +56,11 @@ namespace Squidex.Domain.Apps.Entities return this with { NoTotal = value }; } + public Q WithoutSlowTotal(bool value = true) + { + return this with { NoSlowTotal = value }; + } + public Q WithODataQuery(string? query) { return this with { QueryAsOdata = query }; diff --git a/backend/src/Squidex.Domain.Users/DefaultKeyStore.cs b/backend/src/Squidex.Domain.Users/DefaultKeyStore.cs index 100c20329..b2fb0e4f1 100644 --- a/backend/src/Squidex.Domain.Users/DefaultKeyStore.cs +++ b/backend/src/Squidex.Domain.Users/DefaultKeyStore.cs @@ -47,7 +47,7 @@ namespace Squidex.Domain.Users private async Task GetOrCreateKeyAsync() { - var (state, _, _) = await store.ReadAsync(default); + var (_, state, _, _) = await store.ReadAsync(default); RsaSecurityKey securityKey; @@ -75,13 +75,13 @@ namespace Squidex.Domain.Users try { - await store.WriteAsync(default, state, 0, 0); + await store.WriteAsync(new SnapshotWriteJob(default, state, 0)); return securityKey; } catch (InconsistentStateException) { - (state, _, _) = await store.ReadAsync(default); + (_, state, _, _) = await store.ReadAsync(default); } } diff --git a/backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs b/backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs index 554ce6f9a..52fec9604 100644 --- a/backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs +++ b/backend/src/Squidex.Domain.Users/DefaultXmlRepository.cs @@ -50,12 +50,12 @@ namespace Squidex.Domain.Users { var state = new State(element); - store.WriteAsync(DomainId.Create(friendlyName), state, EtagVersion.Any, 0); + store.WriteAsync(new SnapshotWriteJob(DomainId.Create(friendlyName), state, 0)); } private async Task> GetAllElementsAsync() { - return await store.ReadAllAsync().Select(x => x.State.ToXml()).ToListAsync(); + return await store.ReadAllAsync().Select(x => x.Value.ToXml()).ToListAsync(); } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/InstantSerializer.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/InstantSerializer.cs index faecef85e..92792cde5 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/InstantSerializer.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/InstantSerializer.cs @@ -9,10 +9,11 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Serializers; using NodaTime; +using NodaTime.Text; namespace Squidex.Infrastructure.MongoDb { - public sealed class InstantSerializer : SerializerBase, IBsonPolymorphicSerializer + public sealed class InstantSerializer : SerializerBase, IBsonPolymorphicSerializer, IRepresentationConfigurable { public static void Register() { @@ -31,16 +32,61 @@ namespace Squidex.Infrastructure.MongoDb get => true; } + public BsonType Representation { get; } + + public InstantSerializer(BsonType representation = BsonType.DateTime) + { + if (representation != BsonType.DateTime && representation != BsonType.Int64 && representation != BsonType.String) + { + throw new ArgumentException("Unsupported representation.", nameof(representation)); + } + + Representation = representation; + } + public override Instant Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) { - var value = context.Reader.ReadDateTime(); + var reader = context.Reader; + + switch (reader.CurrentBsonType) + { + case BsonType.DateTime: + return Instant.FromUnixTimeMilliseconds(context.Reader.ReadDateTime()); + case BsonType.Int64: + return Instant.FromUnixTimeMilliseconds(context.Reader.ReadInt64()); + case BsonType.String: + return InstantPattern.ExtendedIso.Parse(context.Reader.ReadString()).Value; + } - return Instant.FromUnixTimeMilliseconds(value); + throw new NotSupportedException("Unsupported Representation."); } public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, Instant value) { - context.Writer.WriteDateTime(value.ToUnixTimeMilliseconds()); + switch (Representation) + { + case BsonType.DateTime: + context.Writer.WriteDateTime(value.ToUnixTimeMilliseconds()); + return; + case BsonType.Int64: + context.Writer.WriteInt64(value.ToUnixTimeMilliseconds()); + return; + case BsonType.String: + context.Writer.WriteString(InstantPattern.ExtendedIso.Format(value)); + return; + } + + throw new NotSupportedException("Unsupported Representation."); + } + + public InstantSerializer WithRepresentation(BsonType representation) + { + return Representation == representation ? this : new InstantSerializer(representation); + } + + IBsonSerializer IRepresentationConfigurable.WithRepresentation(BsonType representation) + { + return WithRepresentation(representation); } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs index 26cd1ad27..eef6bf902 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs @@ -104,7 +104,22 @@ namespace Squidex.Infrastructure.MongoDb return find.Project(Builders.Projection.Exclude(exclude1).Exclude(exclude2)); } - public static async Task UpsertVersionedAsync(this IMongoCollection collection, TKey key, long oldVersion, long newVersion, T document, + public static long ToLong(this BsonValue value) + { + switch (value.BsonType) + { + case BsonType.Int32: + return value.AsInt32; + case BsonType.Int64: + return value.AsInt64; + case BsonType.Double: + return (long)value.AsDouble; + default: + throw new InvalidCastException($"Cannot cast from {value.BsonType} to long."); + } + } + + public static async Task UpsertVersionedAsync(this IMongoCollection collection, TKey key, long oldVersion, long newVersion, T document, CancellationToken ct = default) where T : IVersionedEntity where TKey : notnull { @@ -113,14 +128,14 @@ namespace Squidex.Infrastructure.MongoDb document.DocumentId = key; document.Version = newVersion; - if (oldVersion > EtagVersion.Any) - { - await collection.ReplaceOneAsync(x => x.DocumentId.Equals(key) && x.Version == oldVersion, document, UpsertReplace, ct); - } - else - { - await collection.ReplaceOneAsync(x => x.DocumentId.Equals(key), document, UpsertReplace, ct); - } + Expression> filter = + oldVersion > EtagVersion.Any ? + x => x.DocumentId.Equals(key) && x.Version == oldVersion : + x => x.DocumentId.Equals(key); + + var result = await collection.ReplaceOneAsync(filter, document, UpsertReplace, ct); + + return result.IsAcknowledged && result.ModifiedCount == 1; } catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) { diff --git a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs index 2a214a0aa..b0f50f652 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs @@ -38,7 +38,7 @@ namespace Squidex.Infrastructure.States return $"States_{name}"; } - public async Task<(T Value, bool Valid, long Version)> ReadAsync(DomainId key, + public async Task> ReadAsync(DomainId key, CancellationToken ct = default) { using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/ReadAsync")) @@ -49,31 +49,31 @@ namespace Squidex.Infrastructure.States if (existing != null) { - return (existing.Document, true, existing.Version); + return new SnapshotResult(existing.DocumentId, existing.Document, existing.Version); } - return (default!, true, EtagVersion.Empty); + return new SnapshotResult(default, default!, EtagVersion.Empty); } } - public async Task WriteAsync(DomainId key, T value, long oldVersion, long newVersion, + public async Task WriteAsync(SnapshotWriteJob job, CancellationToken ct = default) { using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/WriteAsync")) { - var document = CreateDocument(key, value, newVersion); + var document = CreateDocument(job.Key, job.Value, job.OldVersion); - await Collection.UpsertVersionedAsync(key, oldVersion, newVersion, document, ct); + await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, document, ct); } } - public async Task WriteManyAsync(IEnumerable<(DomainId Key, T Value, long Version)> snapshots, + public async Task WriteManyAsync(IEnumerable> jobs, CancellationToken ct = default) { using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/WriteManyAsync")) { - var writes = snapshots.Select(x => - new ReplaceOneModel(Filter.Eq(y => y.DocumentId, x.Key), CreateDocument(x.Key, x.Value, x.Version)) + var writes = jobs.Select(x => + new ReplaceOneModel(Filter.Eq(y => y.DocumentId, x.Key), CreateDocument(x.Key, x.Value, x.NewVersion)) { IsUpsert = true }).ToList(); @@ -96,7 +96,7 @@ namespace Squidex.Infrastructure.States } } - public async IAsyncEnumerable<(T State, long Version)> ReadAllAsync( + public async IAsyncEnumerable> ReadAllAsync( [EnumeratorCancellation] CancellationToken ct = default) { using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/ReadAllAsync")) @@ -105,7 +105,7 @@ namespace Squidex.Infrastructure.States await foreach (var document in find.ToAsyncEnumerable(ct)) { - yield return (document.Document, document.Version); + yield return new SnapshotResult(document.DocumentId, document.Document, document.Version, true); } } } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs b/backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs index ed44c7715..c871c5178 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/EnvelopeExtensions.cs @@ -8,7 +8,6 @@ using System.Globalization; using NodaTime; using NodaTime.Text; -using Squidex.Infrastructure.Json.Objects; namespace Squidex.Infrastructure.EventSourcing { diff --git a/backend/src/Squidex.Infrastructure/InstantExtensions.cs b/backend/src/Squidex.Infrastructure/InstantExtensions.cs index 3b9e37d8b..087af3ffc 100644 --- a/backend/src/Squidex.Infrastructure/InstantExtensions.cs +++ b/backend/src/Squidex.Infrastructure/InstantExtensions.cs @@ -15,5 +15,10 @@ namespace Squidex.Infrastructure { return Instant.FromUnixTimeSeconds(value.ToUnixTimeSeconds()); } + + public static Instant WithoutNs(this Instant value) + { + return Instant.FromUnixTimeMilliseconds(value.ToUnixTimeMilliseconds()); + } } } diff --git a/backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs b/backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs index 26f7ea2d6..32bd9703f 100644 --- a/backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs +++ b/backend/src/Squidex.Infrastructure/Json/Newtonsoft/JsonValueConverter.cs @@ -6,7 +6,6 @@ // ========================================================================== using Newtonsoft.Json; -using Squidex.Infrastructure.Collections; using Squidex.Infrastructure.Json.Objects; #pragma warning disable RECS0018 // Comparison of floating point numbers with equality operator diff --git a/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs b/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs index be9b5305a..18f1c0fa6 100644 --- a/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs +++ b/backend/src/Squidex.Infrastructure/Json/Objects/JsonObject.cs @@ -5,8 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using Squidex.Infrastructure.Collections; - namespace Squidex.Infrastructure.Json.Objects { public class JsonObject : Dictionary, IEquatable diff --git a/backend/src/Squidex.Infrastructure/Orleans/GrainState.cs b/backend/src/Squidex.Infrastructure/Orleans/GrainState.cs index 4ac42a7c0..db356451a 100644 --- a/backend/src/Squidex.Infrastructure/Orleans/GrainState.cs +++ b/backend/src/Squidex.Infrastructure/Orleans/GrainState.cs @@ -57,7 +57,7 @@ namespace Squidex.Infrastructure.Orleans persistence = factory.WithSnapshots(GetType(), key, ApplyState); - return persistence.ReadAsync(); + return persistence.ReadAsync(ct: ct); } private void ApplyState(T value, long version) diff --git a/backend/src/Squidex.Infrastructure/States/BatchContext.cs b/backend/src/Squidex.Infrastructure/States/BatchContext.cs index 367787bd9..c5724e987 100644 --- a/backend/src/Squidex.Infrastructure/States/BatchContext.cs +++ b/backend/src/Squidex.Infrastructure/States/BatchContext.cs @@ -20,7 +20,7 @@ namespace Squidex.Infrastructure.States private readonly IEventDataFormatter eventDataFormatter; private readonly IStreamNameResolver streamNameResolver; private readonly Dictionary>)> @events = new Dictionary>)>(); - private Dictionary? snapshots; + private Dictionary>? snapshots; internal BatchContext( Type owner, @@ -38,11 +38,11 @@ namespace Squidex.Infrastructure.States internal void Add(DomainId key, T snapshot, long version) { - snapshots ??= new Dictionary(); + snapshots ??= new (); - if (!snapshots.TryGetValue(key, out var existing) || existing.Version < version) + if (!snapshots.TryGetValue(key, out var existing) || existing.NewVersion < version) { - snapshots[key] = (snapshot, version); + snapshots[key] = new SnapshotWriteJob(key, snapshot, version); } } @@ -86,9 +86,7 @@ namespace Squidex.Infrastructure.States return Task.CompletedTask; } - var list = current.Select(x => (x.Key, x.Value.Snapshot, x.Value.Version)); - - return snapshotStore.WriteManyAsync(list); + return snapshotStore.WriteManyAsync(current.Values); } public IPersistence WithEventSourcing(Type owner, DomainId key, HandleEvent? applyEvent) diff --git a/backend/src/Squidex.Infrastructure/States/BatchPersistence.cs b/backend/src/Squidex.Infrastructure/States/BatchPersistence.cs index e50ce56dd..067b11c4e 100644 --- a/backend/src/Squidex.Infrastructure/States/BatchPersistence.cs +++ b/backend/src/Squidex.Infrastructure/States/BatchPersistence.cs @@ -31,17 +31,20 @@ namespace Squidex.Infrastructure.States Version = version; } - public Task DeleteAsync() + public Task DeleteAsync( + CancellationToken ct = default) { throw new NotSupportedException(); } - public Task WriteEventsAsync(IReadOnlyList> events) + public Task WriteEventsAsync(IReadOnlyList> events, + CancellationToken ct = default) { throw new NotSupportedException(); } - public Task ReadAsync(long expectedVersion = -2) + public Task ReadAsync(long expectedVersion = -2, + CancellationToken ct = default) { if (applyEvent != null) { @@ -69,7 +72,8 @@ namespace Squidex.Infrastructure.States return Task.CompletedTask; } - public Task WriteSnapshotAsync(T state) + public Task WriteSnapshotAsync(T state, + CancellationToken ct = default) { context.Add(ownerKey, state, Version); diff --git a/backend/src/Squidex.Infrastructure/States/IPersistence.cs b/backend/src/Squidex.Infrastructure/States/IPersistence.cs index a22ca67ce..093c7148e 100644 --- a/backend/src/Squidex.Infrastructure/States/IPersistence.cs +++ b/backend/src/Squidex.Infrastructure/States/IPersistence.cs @@ -15,12 +15,16 @@ namespace Squidex.Infrastructure.States bool IsSnapshotStale { get; } - Task DeleteAsync(); + Task DeleteAsync( + CancellationToken ct = default); - Task WriteEventsAsync(IReadOnlyList> events); + Task WriteEventsAsync(IReadOnlyList> events, + CancellationToken ct = default); - Task WriteSnapshotAsync(TState state); + Task WriteSnapshotAsync(TState state, + CancellationToken ct = default); - Task ReadAsync(long expectedVersion = EtagVersion.Any); + Task ReadAsync(long expectedVersion = EtagVersion.Any, + CancellationToken ct = default); } } diff --git a/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs b/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs index a751e28c3..6e6462292 100644 --- a/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs +++ b/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs @@ -5,17 +5,20 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +#pragma warning disable MA0048 // File name must match type name +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + namespace Squidex.Infrastructure.States { public interface ISnapshotStore { - Task WriteAsync(DomainId key, T value, long oldVersion, long newVersion, + Task WriteAsync(SnapshotWriteJob job, CancellationToken ct = default); - Task WriteManyAsync(IEnumerable<(DomainId Key, T Value, long Version)> snapshots, + Task WriteManyAsync(IEnumerable> jobs, CancellationToken ct = default); - Task<(T Value, bool Valid, long Version)> ReadAsync(DomainId key, + Task> ReadAsync(DomainId key, CancellationToken ct = default); Task ClearAsync( @@ -24,7 +27,12 @@ namespace Squidex.Infrastructure.States Task RemoveAsync(DomainId key, CancellationToken ct = default); - IAsyncEnumerable<(T State, long Version)> ReadAllAsync( + IAsyncEnumerable> ReadAllAsync( CancellationToken ct = default); } + + public record struct SnapshotResult(DomainId Key, T Value, long Version, + bool IsValid = true); + + public record struct SnapshotWriteJob(DomainId Key, T Value, long NewVersion, long OldVersion = EtagVersion.Any); } diff --git a/backend/src/Squidex.Infrastructure/States/Persistence.cs b/backend/src/Squidex.Infrastructure/States/Persistence.cs index 10004f82d..ed6cc4ba2 100644 --- a/backend/src/Squidex.Infrastructure/States/Persistence.cs +++ b/backend/src/Squidex.Infrastructure/States/Persistence.cs @@ -65,13 +65,14 @@ namespace Squidex.Infrastructure.States streamName = new Lazy(() => streamNameResolver.GetStreamName(ownerType, ownerKey.ToString()!)); } - public async Task DeleteAsync() + public async Task DeleteAsync( + CancellationToken ct = default) { if (UseSnapshots) { using (Telemetry.Activities.StartActivity("Persistence/ReadState")) { - await snapshotStore.RemoveAsync(ownerKey); + await snapshotStore.RemoveAsync(ownerKey, ct); } } @@ -79,24 +80,25 @@ namespace Squidex.Infrastructure.States { using (Telemetry.Activities.StartActivity("Persistence/ReadEvents")) { - await eventStore.DeleteStreamAsync(streamName.Value); + await eventStore.DeleteStreamAsync(streamName.Value, ct); } } } - public async Task ReadAsync(long expectedVersion = EtagVersion.Any) + public async Task ReadAsync(long expectedVersion = EtagVersion.Any, + CancellationToken ct = default) { versionSnapshot = EtagVersion.Empty; versionEvents = EtagVersion.Empty; if (UseSnapshots) { - await ReadSnapshotAsync(); + await ReadSnapshotAsync(ct); } if (UseEventSourcing) { - await ReadEventsAsync(); + await ReadEventsAsync(ct); } UpdateVersion(); @@ -114,9 +116,10 @@ namespace Squidex.Infrastructure.States } } - private async Task ReadSnapshotAsync() + private async Task ReadSnapshotAsync( + CancellationToken ct) { - var (state, valid, version) = await snapshotStore.ReadAsync(ownerKey); + var (_, state, version, valid) = await snapshotStore.ReadAsync(ownerKey, ct); version = Math.Max(version, EtagVersion.Empty); versionSnapshot = version; @@ -132,9 +135,10 @@ namespace Squidex.Infrastructure.States } } - private async Task ReadEventsAsync() + private async Task ReadEventsAsync( + CancellationToken ct) { - var events = await eventStore.QueryAsync(streamName.Value, versionEvents + 1); + var events = await eventStore.QueryAsync(streamName.Value, versionEvents + 1, ct); var isStopped = false; @@ -161,7 +165,8 @@ namespace Squidex.Infrastructure.States } } - public async Task WriteSnapshotAsync(T state) + public async Task WriteSnapshotAsync(T state, + CancellationToken ct = default) { var oldVersion = versionSnapshot; @@ -179,7 +184,12 @@ namespace Squidex.Infrastructure.States using (Telemetry.Activities.StartActivity("Persistence/WriteState")) { - await snapshotStore.WriteAsync(ownerKey, state, oldVersion, newVersion); + var job = new SnapshotWriteJob(ownerKey, state, newVersion) + { + OldVersion = oldVersion + }; + + await snapshotStore.WriteAsync(job, ct); } versionSnapshot = newVersion; @@ -187,7 +197,8 @@ namespace Squidex.Infrastructure.States UpdateVersion(); } - public async Task WriteEventsAsync(IReadOnlyList> events) + public async Task WriteEventsAsync(IReadOnlyList> events, + CancellationToken ct = default) { Guard.NotEmpty(events); @@ -205,7 +216,7 @@ namespace Squidex.Infrastructure.States { using (Telemetry.Activities.StartActivity("Persistence/WriteEvents")) { - await eventStore.AppendAsync(eventCommitId, streamName.Value, oldVersion, eventData); + await eventStore.AppendAsync(eventCommitId, streamName.Value, oldVersion, eventData, ct); } } catch (WrongEventVersionException ex) diff --git a/backend/src/Squidex/Config/Domain/AssetServices.cs b/backend/src/Squidex/Config/Domain/AssetServices.cs index 48feaff64..91007e2fb 100644 --- a/backend/src/Squidex/Config/Domain/AssetServices.cs +++ b/backend/src/Squidex/Config/Domain/AssetServices.cs @@ -6,7 +6,6 @@ // ========================================================================== using FluentFTP; -using MongoDB.Driver; using MongoDB.Driver.GridFS; using Squidex.Assets; using Squidex.Domain.Apps.Entities; @@ -16,7 +15,6 @@ using Squidex.Domain.Apps.Entities.Assets.Queries; using Squidex.Domain.Apps.Entities.History; using Squidex.Domain.Apps.Entities.Search; using Squidex.Hosting; -using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Orleans; using tusdotnet.Interfaces; diff --git a/backend/src/Squidex/Config/Domain/EventSourcingServices.cs b/backend/src/Squidex/Config/Domain/EventSourcingServices.cs index fceed69e3..2c6e86ad5 100644 --- a/backend/src/Squidex/Config/Domain/EventSourcingServices.cs +++ b/backend/src/Squidex/Config/Domain/EventSourcingServices.cs @@ -7,7 +7,6 @@ using EventStore.Client; using MongoDB.Driver; -using Squidex.Infrastructure; using Squidex.Infrastructure.Commands; using Squidex.Infrastructure.Diagnostics; using Squidex.Infrastructure.EventSourcing; diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetMappingTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetMappingTests.cs new file mode 100644 index 000000000..f5c7a5feb --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetMappingTests.cs @@ -0,0 +1,89 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using FluentAssertions; +using NodaTime; +using Squidex.Domain.Apps.Core.Assets; +using Squidex.Domain.Apps.Entities.Assets.DomainObject; +using Squidex.Domain.Apps.Entities.MongoDb.Assets; +using Squidex.Infrastructure; +using Squidex.Infrastructure.States; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Assets.MongoDb +{ + public class AssetMappingTests + { + [Fact] + public void Should_map_asset() + { + var user = new RefToken(RefTokenType.Subject, "1"); + + var time = SystemClock.Instance.GetCurrentInstant(); + + var source = new AssetDomainObject.State + { + Id = DomainId.NewGuid(), + AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), + Created = time, + CreatedBy = user, + FileHash = "my-hash", + FileName = "my-image.png", + FileSize = 1024, + FileVersion = 13, + IsDeleted = true, + IsProtected = true, + LastModified = time, + LastModifiedBy = user, + Metadata = new AssetMetadata().SetPixelHeight(600), + MimeType = "image/png", + ParentId = DomainId.NewGuid(), + Slug = "my-image", + Tags = new HashSet { "image" }, + TotalSize = 1024 * 2, + Type = AssetType.Image, + Version = 42, + }; + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = MongoAssetEntity.Create(snapshotJob); + + var mapped = snapshot.ToState(); + + mapped.Should().BeEquivalentTo(source); + } + + [Fact] + public void Should_map_asset_folder() + { + var user = new RefToken(RefTokenType.Subject, "1"); + + var time = SystemClock.Instance.GetCurrentInstant(); + + var source = new AssetFolderDomainObject.State + { + Id = DomainId.NewGuid(), + AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), + Created = time, + CreatedBy = user, + FolderName = "my-folder", + IsDeleted = true, + LastModified = time, + LastModifiedBy = user, + ParentId = DomainId.NewGuid(), + Version = 42, + }; + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = MongoAssetFolderEntity.Create(snapshotJob); + + var mapped = snapshot.ToState(); + + mapped.Should().BeEquivalentTo(source); + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs index 226df13ea..c564d4925 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/MongoDb/AssetsQueryFixture.cs @@ -36,11 +36,11 @@ namespace Squidex.Domain.Apps.Entities.Assets.MongoDb public AssetsQueryFixture() { + SetupJson(); + mongoClient = new MongoClient(TestConfig.Configuration["mongodb:configuration"]); mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]); - SetupJson(); - var assetRepository = new MongoAssetRepository(mongoDatabase); Task.Run(async () => diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentMappingTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentMappingTests.cs new file mode 100644 index 000000000..52ac6247f --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/ContentMappingTests.cs @@ -0,0 +1,159 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using FakeItEasy; +using FluentAssertions; +using NodaTime; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Domain.Apps.Entities.Contents.DomainObject; +using Squidex.Domain.Apps.Entities.MongoDb.Contents; +using Squidex.Infrastructure; +using Squidex.Infrastructure.States; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Contents.MongoDb +{ + public class ContentMappingTests + { + private readonly IAppProvider appProvider = A.Fake(); + + [Fact] + public async Task Should_map_content_without_new_version_to_draft() + { + var source = CreateContentWithoutNewVersion(); + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = await MongoContentEntity.CreateDraftAsync(snapshotJob, appProvider); + + Assert.Equal(source.CurrentVersion.Data, snapshot.Data); + Assert.Null(snapshot.DraftData); + Assert.Null(snapshot.NewStatus); + Assert.NotNull(snapshot.ScheduleJob); + Assert.True(snapshot.IsSnapshot); + + var mapped = snapshot.ToState(); + + mapped.Should().BeEquivalentTo(source); + } + + [Fact] + public async Task Should_map_content_without_new_version_to_published() + { + var source = CreateContentWithoutNewVersion(); + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = await MongoContentEntity.CreatePublishedAsync(snapshotJob, appProvider); + + Assert.Equal(source.CurrentVersion.Data, snapshot.Data); + Assert.Null(snapshot.DraftData); + Assert.Null(snapshot.NewStatus); + Assert.Null(snapshot.ScheduleJob); + Assert.False(snapshot.IsSnapshot); + } + + [Fact] + public async Task Should_map_content_with_new_version_to_draft() + { + var source = CreateContentWithNewVersion(); + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = await MongoContentEntity.CreateDraftAsync(snapshotJob, appProvider); + + Assert.Equal(source.NewVersion?.Data, snapshot.Data); + Assert.Equal(source.CurrentVersion.Data, snapshot.DraftData); + Assert.NotNull(snapshot.NewStatus); + Assert.NotNull(snapshot.ScheduleJob); + Assert.True(snapshot.IsSnapshot); + + var mapped = snapshot.ToState(); + + mapped.Should().BeEquivalentTo(source); + } + + [Fact] + public async Task Should_map_content_with_new_version_to_published() + { + var source = CreateContentWithNewVersion(); + + var snapshotJob = new SnapshotWriteJob(source.UniqueId, source, source.Version); + var snapshot = await MongoContentEntity.CreatePublishedAsync(snapshotJob, appProvider); + + Assert.Equal(source.CurrentVersion?.Data, snapshot.Data); + Assert.Null(snapshot.DraftData); + Assert.Null(snapshot.NewStatus); + Assert.Null(snapshot.ScheduleJob); + Assert.False(snapshot.IsSnapshot); + } + + private static ContentDomainObject.State CreateContentWithoutNewVersion() + { + var user = new RefToken(RefTokenType.Subject, "1"); + + var data = + new ContentData() + .AddField("my-field", + new ContentFieldData() + .AddInvariant(42)); + + var time = SystemClock.Instance.GetCurrentInstant(); + + var state = new ContentDomainObject.State + { + Id = DomainId.NewGuid(), + AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), + Created = time, + CreatedBy = user, + CurrentVersion = new ContentVersion(Status.Archived, data), + IsDeleted = true, + LastModified = time, + LastModifiedBy = user, + ScheduleJob = new ScheduleJob(DomainId.NewGuid(), Status.Published, user, time), + SchemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"), + Version = 42, + }; + + return state; + } + + private static ContentDomainObject.State CreateContentWithNewVersion() + { + var user = new RefToken(RefTokenType.Subject, "1"); + + var data = + new ContentData() + .AddField("my-field", + new ContentFieldData() + .AddInvariant(42)); + + var newData = + new ContentData() + .AddField("my-field", + new ContentFieldData() + .AddInvariant(13)); + + var time = SystemClock.Instance.GetCurrentInstant(); + + var state = new ContentDomainObject.State + { + Id = DomainId.NewGuid(), + AppId = NamedId.Of(DomainId.NewGuid(), "my-app"), + Created = time, + CreatedBy = user, + CurrentVersion = new ContentVersion(Status.Archived, data), + IsDeleted = true, + LastModified = time, + LastModifiedBy = user, + NewVersion = new ContentVersion(Status.Published, newData), + ScheduleJob = new ScheduleJob(DomainId.NewGuid(), Status.Published, user, time), + SchemaId = NamedId.Of(DomainId.NewGuid(), "my-schema"), + Version = 42, + }; + + return state; + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Properties/Resources.Designer.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Properties/Resources.Designer.cs index 458ced345..e01839883 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Properties/Resources.Designer.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Properties/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Properties { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs index d397d107d..13a2dfda0 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/TestHelpers/HandlerTestBase.cs @@ -64,10 +64,10 @@ namespace Squidex.Domain.Apps.Entities.TestHelpers A.CallTo(() => persistenceFactory.WithEventSourcing(A._, Id, A._)) .Returns(persistence); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) - .Invokes((IReadOnlyList> events) => LastEvents = events); + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) + .Invokes((IReadOnlyList> events, CancellationToken _) => LastEvents = events); - A.CallTo(() => persistence.DeleteAsync()) + A.CallTo(() => persistence.DeleteAsync(default)) .Invokes(() => LastEvents = Enumerable.Empty>()); #pragma warning restore MA0056 // Do not call overridable members in constructor } diff --git a/backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs b/backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs index 1f78407f4..4e0e51834 100644 --- a/backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs +++ b/backend/tests/Squidex.Domain.Users.Tests/DefaultKeyStoreTests.cs @@ -30,7 +30,7 @@ namespace Squidex.Domain.Users public void Should_configure_new_keys() { A.CallTo(() => store.ReadAsync(A._, default)) - .Returns((null!, true, 0)); + .Returns(new SnapshotResult(default, null!, 0)); var options = new OpenIddictServerOptions(); @@ -39,7 +39,7 @@ namespace Squidex.Domain.Users Assert.NotEmpty(options.SigningCredentials); Assert.NotEmpty(options.EncryptionCredentials); - A.CallTo(() => store.WriteAsync(A._, A._, 0, 0, default)) + A.CallTo(() => store.WriteAsync(A>._, default)) .MustHaveHappenedOnceExactly(); } @@ -47,7 +47,7 @@ namespace Squidex.Domain.Users public void Should_configure_existing_keys() { A.CallTo(() => store.ReadAsync(A._, default)) - .Returns((ExistingKey(), true, 0)); + .Returns(new SnapshotResult(default, ExistingKey(), 0)); var options = new OpenIddictServerOptions(); @@ -56,7 +56,7 @@ namespace Squidex.Domain.Users Assert.NotEmpty(options.SigningCredentials); Assert.NotEmpty(options.EncryptionCredentials); - A.CallTo(() => store.WriteAsync(A._, A._, 0, 0, default)) + A.CallTo(() => store.WriteAsync(A>._, default)) .MustNotHaveHappened(); } @@ -64,11 +64,11 @@ namespace Squidex.Domain.Users public void Should_configure_existing_keys_when_initial_setup_failed() { A.CallTo(() => store.ReadAsync(A._, default)) - .Returns((null!, true, 0)).Once() + .Returns(new SnapshotResult(default, null!, 0)).Once() .Then - .Returns((ExistingKey(), true, 0)); + .Returns(new SnapshotResult(default, ExistingKey(), 0)); - A.CallTo(() => store.WriteAsync(A._, A._, 0, 0, default)) + A.CallTo(() => store.WriteAsync(A>._, default)) .Throws(new InconsistentStateException(0, 0)); var options = new OpenIddictServerOptions(); @@ -78,7 +78,7 @@ namespace Squidex.Domain.Users Assert.NotEmpty(options.SigningCredentials); Assert.NotEmpty(options.EncryptionCredentials); - A.CallTo(() => store.WriteAsync(A._, A._, 0, 0, default)) + A.CallTo(() => store.WriteAsync(A>._, default)) .MustHaveHappened(); } diff --git a/backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs b/backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs index c6834706a..5da46e34b 100644 --- a/backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs +++ b/backend/tests/Squidex.Domain.Users.Tests/DefaultXmlRepositoryTests.cs @@ -29,11 +29,11 @@ namespace Squidex.Domain.Users A.CallTo(() => store.ReadAllAsync(default)) .Returns(new[] { - (new DefaultXmlRepository.State + new SnapshotResult(default, new DefaultXmlRepository.State { Xml = new XElement("xml").ToString() }, 0L), - (new DefaultXmlRepository.State + new SnapshotResult(default, new DefaultXmlRepository.State { Xml = new XElement("xml").ToString() }, 0L) @@ -51,7 +51,7 @@ namespace Squidex.Domain.Users sut.StoreElement(xml, "name"); - A.CallTo(() => store.WriteAsync(DomainId.Create("name"), A._, A._, 0, default)) + A.CallTo(() => store.WriteAsync(A>.That.Matches(x => x.Key == DomainId.Create("name")), default)) .MustHaveHappened(); } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs index e22dbef43..4918e613b 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Commands/DomainObjectTests.cs @@ -42,7 +42,7 @@ namespace Squidex.Infrastructure.Commands await sut.EnsureLoadedAsync(); - A.CallTo(() => persistence.WriteSnapshotAsync(A._)) + A.CallTo(() => persistence.WriteSnapshotAsync(A._, default)) .MustHaveHappened(); } @@ -56,7 +56,7 @@ namespace Squidex.Infrastructure.Commands await sut.EnsureLoadedAsync(); - A.CallTo(() => persistence.WriteSnapshotAsync(A._)) + A.CallTo(() => persistence.WriteSnapshotAsync(A._, default)) .MustNotHaveHappened(); } @@ -67,11 +67,11 @@ namespace Squidex.Infrastructure.Commands var result = await sut.ExecuteAsync(new CreateAuto { Value = 4 }); - A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 4))) + A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 4), default)) .MustHaveHappened(); - A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1))) + A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1), default)) .MustHaveHappened(); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustNotHaveHappened(); Assert.Equal(CommandResult.Empty(id, 0, EtagVersion.Empty), result); @@ -139,14 +139,14 @@ namespace Squidex.Infrastructure.Commands SetupCreated(2); SetupDeleted(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Throws(new InconsistentStateException(2, -1)).Once(); var result = await sut.ExecuteAsync(new CreateAuto { Value = 4 }); - A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1))) + A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1), default)) .MustHaveHappenedANumberOfTimesMatching(x => x == 3); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustHaveHappened(); Assert.Equal(CommandResult.Empty(id, 2, 1), result); @@ -165,7 +165,7 @@ namespace Squidex.Infrastructure.Commands SetupCreated(2); SetupDeleted(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Throws(new InconsistentStateException(2, -1)).Once(); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto { Value = 4 })); @@ -180,14 +180,14 @@ namespace Squidex.Infrastructure.Commands SetupCreated(2); SetupDeleted(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Throws(new InconsistentStateException(2, -1)).Once(); var result = await sut.ExecuteAsync(new Upsert { Value = 4 }); - A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1))) + A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1), default)) .MustHaveHappenedANumberOfTimesMatching(x => x == 3); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustHaveHappened(); Assert.Equal(CommandResult.Empty(id, 2, 1), result); @@ -206,7 +206,7 @@ namespace Squidex.Infrastructure.Commands SetupCreated(2); SetupDeleted(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Throws(new InconsistentStateException(2, -1)).Once(); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new Upsert { Value = 4 })); @@ -221,11 +221,11 @@ namespace Squidex.Infrastructure.Commands var result = await sut.ExecuteAsync(new UpdateAuto { Value = 8, ExpectedVersion = 0 }); - A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 8))) + A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 8), default)) .MustHaveHappened(); - A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1))) + A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1), default)) .MustHaveHappened(); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustNotHaveHappened(); Assert.Equal(CommandResult.Empty(id, 1, 0), result); @@ -243,11 +243,11 @@ namespace Squidex.Infrastructure.Commands var result = await sut.ExecuteAsync(new UpdateAuto { Value = 8, ExpectedVersion = 0 }); - A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 8))) + A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 8), default)) .MustHaveHappened(); - A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1))) + A.CallTo(() => persistence.WriteEventsAsync(A>>.That.Matches(x => x.Count == 1), default)) .MustHaveHappened(); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustHaveHappenedOnceExactly(); Assert.Equal(CommandResult.Empty(id, 1, 0), result); @@ -265,7 +265,7 @@ namespace Squidex.Infrastructure.Commands await sut.ExecuteAsync(new CreateAuto()); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustNotHaveHappened(); } @@ -277,7 +277,7 @@ namespace Squidex.Infrastructure.Commands await sut.ExecuteAsync(new UpdateAuto { Value = 8, ExpectedVersion = 0 }); await sut.ExecuteAsync(new UpdateAuto { Value = 9, ExpectedVersion = 1 }); - A.CallTo(() => persistence.ReadAsync(A._)) + A.CallTo(() => persistence.ReadAsync(A._, default)) .MustHaveHappenedOnceExactly(); Assert.Empty(sut.GetUncomittedEvents()); @@ -291,9 +291,9 @@ namespace Squidex.Infrastructure.Commands await sut.RebuildStateAsync(); - A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 4))) + A.CallTo(() => persistence.WriteSnapshotAsync(A.That.Matches(x => x.Value == 4), default)) .MustHaveHappened(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .MustNotHaveHappened(); } @@ -310,7 +310,7 @@ namespace Squidex.Infrastructure.Commands { SetupEmpty(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Throws(new InconsistentStateException(4, EtagVersion.Empty)); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); @@ -396,7 +396,7 @@ namespace Squidex.Infrastructure.Commands { SetupEmpty(); - A.CallTo(() => persistence.WriteSnapshotAsync(A._)) + A.CallTo(() => persistence.WriteSnapshotAsync(A._, default)) .Throws(new InvalidOperationException()); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new CreateAuto())); @@ -410,7 +410,7 @@ namespace Squidex.Infrastructure.Commands { SetupCreated(4); - A.CallTo(() => persistence.WriteSnapshotAsync(A._)) + A.CallTo(() => persistence.WriteSnapshotAsync(A._, default)) .Throws(new InvalidOperationException()); await Assert.ThrowsAsync(() => sut.ExecuteAsync(new UpdateAuto())); @@ -434,13 +434,13 @@ namespace Squidex.Infrastructure.Commands AssertSnapshot(sut.Snapshot, 0, EtagVersion.Empty, false); - A.CallTo(() => persistence.DeleteAsync()) + A.CallTo(() => persistence.DeleteAsync(default)) .MustHaveHappened(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .MustHaveHappenedOnceExactly(); - A.CallTo(() => deleteStream.WriteEventsAsync(A>>._)) + A.CallTo(() => deleteStream.WriteEventsAsync(A>>._, default)) .MustHaveHappened(); } @@ -518,7 +518,7 @@ namespace Squidex.Infrastructure.Commands var version = -1; - A.CallTo(() => persistence.ReadAsync(-2)) + A.CallTo(() => persistence.ReadAsync(-2, default)) .Invokes(() => { version++; @@ -548,7 +548,7 @@ namespace Squidex.Infrastructure.Commands var @events = new List>(); - A.CallTo(() => persistence.WriteEventsAsync(A>>._)) + A.CallTo(() => persistence.WriteEventsAsync(A>>._, default)) .Invokes(args => { @events.AddRange(args.GetArgument>>(0)!); @@ -563,7 +563,7 @@ namespace Squidex.Infrastructure.Commands }) .Returns(eventsPersistence); - A.CallTo(() => eventsPersistence.ReadAsync(EtagVersion.Any)) + A.CallTo(() => eventsPersistence.ReadAsync(EtagVersion.Any, default)) .Invokes(_ => { foreach (var @event in events) diff --git a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/Entities.cs b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/Entities.cs new file mode 100644 index 000000000..9e3b40c8f --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/Entities.cs @@ -0,0 +1,50 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Squidex.Infrastructure.MongoDb +{ + public static class Entities + { + public sealed class DateTimeEntity + { + [BsonRepresentation(BsonType.DateTime)] + public T Value { get; set; } + } + + public sealed class Int64Entity + { + [BsonRepresentation(BsonType.Int64)] + public T Value { get; set; } + } + + public sealed class Int32Entity + { + [BsonRepresentation(BsonType.Int32)] + public T Value { get; set; } + } + + public sealed class StringEntity + { + [BsonRepresentation(BsonType.String)] + public T Value { get; set; } + } + + public sealed class BinaryEntity + { + [BsonRepresentation(BsonType.Binary)] + public T Value { get; set; } + } + + public sealed class DefaultEntity + { + public T Value { get; set; } + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/InstantSerializerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/InstantSerializerTests.cs new file mode 100644 index 000000000..65e3e4146 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/InstantSerializerTests.cs @@ -0,0 +1,93 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Bson.IO; +using MongoDB.Bson.Serialization; +using NodaTime; +using Xunit; + +namespace Squidex.Infrastructure.MongoDb +{ + public class InstantSerializerTests + { + public InstantSerializerTests() + { + InstantSerializer.Register(); + } + + [Fact] + public void Should_serialize_as_default() + { + var source = new Entities.DefaultEntity { Value = GetTime() }; + + var result1 = SerializeAndDeserializeBson(source); + + Assert.Equal(source.Value, result1.Value); + } + + [Fact] + public void Should_serialize_as_string() + { + var source = new Entities.StringEntity { Value = GetTime() }; + + var result1 = SerializeAndDeserializeBson(source); + + Assert.Equal(source.Value, result1.Value); + } + + [Fact] + public void Should_serialize_as_int64() + { + var source = new Entities.Int64Entity { Value = GetTime() }; + + var result1 = SerializeAndDeserializeBson(source); + + Assert.Equal(source.Value, result1.Value); + } + + [Fact] + public void Should_serialize_as_datetime() + { + var source = new Entities.DateTimeEntity { Value = GetTime() }; + + var result1 = SerializeAndDeserializeBson(source); + + Assert.Equal(source.Value, result1.Value); + } + + private static Instant GetTime() + { + return SystemClock.Instance.GetCurrentInstant().WithoutNs(); + } + + private static T SerializeAndDeserializeBson(T source) + { + return SerializeAndDeserializeBson(source); + } + + private static TOut SerializeAndDeserializeBson(TIn source) + { + var stream = new MemoryStream(); + + using (var writer = new BsonBinaryWriter(stream)) + { + BsonSerializer.Serialize(writer, source); + + writer.Flush(); + } + + stream.Position = 0; + + using (var reader = new BsonBinaryReader(stream)) + { + var target = BsonSerializer.Deserialize(reader); + + return target; + } + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs index 4718a29cc..cb3d005e3 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceBatchTests.cs @@ -111,16 +111,16 @@ namespace Squidex.Infrastructure.States await persistence1.WriteSnapshotAsync(12); await persistence2.WriteSnapshotAsync(12); - A.CallTo(() => snapshotStore.WriteAsync(A._, A._, A._, A._, A._)) + A.CallTo(() => snapshotStore.WriteAsync(A>._, A._)) .MustNotHaveHappened(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>._, A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>._, A._)) .MustNotHaveHappened(); await bulk.CommitAsync(); await bulk.DisposeAsync(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>.That.Matches(x => x.Count() == 2), A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>.That.Matches(x => x.Count() == 2), A._)) .MustHaveHappenedOnceExactly(); } @@ -143,16 +143,16 @@ namespace Squidex.Infrastructure.States await persistence1_1.WriteSnapshotAsync(12); await persistence1_2.WriteSnapshotAsync(12); - A.CallTo(() => snapshotStore.WriteAsync(A._, A._, A._, A._, A._)) + A.CallTo(() => snapshotStore.WriteAsync(A>._, A._)) .MustNotHaveHappened(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>._, A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>._, A._)) .MustNotHaveHappened(); await bulk.CommitAsync(); await bulk.DisposeAsync(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>.That.Matches(x => x.Count() == 1), A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>.That.Matches(x => x.Count() == 1), A._)) .MustHaveHappenedOnceExactly(); } @@ -172,16 +172,16 @@ namespace Squidex.Infrastructure.States await persistence1.WriteSnapshotAsync(12); await persistence1.WriteSnapshotAsync(13); - A.CallTo(() => snapshotStore.WriteAsync(A._, A._, A._, A._, A._)) + A.CallTo(() => snapshotStore.WriteAsync(A>._, A._)) .MustNotHaveHappened(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>._, A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>._, A._)) .MustNotHaveHappened(); await bulk.CommitAsync(); await bulk.DisposeAsync(); - A.CallTo(() => snapshotStore.WriteManyAsync(A>.That.Matches(x => x.Count() == 1), A._)) + A.CallTo(() => snapshotStore.WriteManyAsync(A>>.That.Matches(x => x.Count() == 1), A._)) .MustHaveHappenedOnceExactly(); } diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs index 0b6ef4838..56d09c982 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceEventSourcingTests.cs @@ -86,7 +86,7 @@ namespace Squidex.Infrastructure.States public async Task Should_read_read_from_snapshot_store() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 2L)); + .Returns(new SnapshotResult(key, "2", 2)); SetupEventStore(3, 2); @@ -106,7 +106,7 @@ namespace Squidex.Infrastructure.States public async Task Should_mark_as_stale_if_snapshot_old_than_events() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 1L)); + .Returns(new SnapshotResult(key, "2", 1)); SetupEventStore(3, 2, 2); @@ -126,7 +126,7 @@ namespace Squidex.Infrastructure.States public async Task Should_throw_exception_if_events_are_older_than_snapshot() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 2L)); + .Returns(new SnapshotResult(key, "2", 2)); SetupEventStore(3, 0, 3); @@ -141,7 +141,7 @@ namespace Squidex.Infrastructure.States public async Task Should_throw_exception_if_events_have_gaps_to_snapshot() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 2L)); + .Returns(new SnapshotResult(key, "2", 2)); SetupEventStore(3, 4, 3); @@ -178,7 +178,7 @@ namespace Squidex.Infrastructure.States public async Task Should_throw_exception_if_other_version_found_from_snapshot() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 2L)); + .Returns(new SnapshotResult(key, "2", 2)); SetupEventStore(0); @@ -219,7 +219,7 @@ namespace Squidex.Infrastructure.States A.CallTo(() => eventStore.AppendAsync(A._, key.ToString(), 3, A>.That.Matches(x => x.Count == 1), A._)) .MustHaveHappened(); - A.CallTo(() => snapshotStore.WriteAsync(A._, A._, A._, A._, A._)) + A.CallTo(() => snapshotStore.WriteAsync(A>._, A._)) .MustNotHaveHappened(); } @@ -238,7 +238,7 @@ namespace Squidex.Infrastructure.States public async Task Should_write_snapshot_to_store() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("2", true, 2L)); + .Returns(new SnapshotResult(key, "2", 2)); SetupEventStore(3); @@ -254,9 +254,9 @@ namespace Squidex.Infrastructure.States await persistence.WriteEventAsync(Envelope.Create(new MyEvent())); await persistence.WriteSnapshotAsync("5"); - A.CallTo(() => snapshotStore.WriteAsync(key, "4", 2, 3, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, "4", 3, 2), A._)) .MustHaveHappened(); - A.CallTo(() => snapshotStore.WriteAsync(key, "5", 3, 4, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, "5", 4, 3), A._)) .MustHaveHappened(); } @@ -264,7 +264,7 @@ namespace Squidex.Infrastructure.States public async Task Should_write_snapshot_to_store_if_not_read_before() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((null!, true, EtagVersion.Empty)); + .Returns(new SnapshotResult(key, null!, EtagVersion.Empty)); SetupEventStore(3); @@ -280,9 +280,9 @@ namespace Squidex.Infrastructure.States await persistence.WriteEventAsync(Envelope.Create(new MyEvent())); await persistence.WriteSnapshotAsync("5"); - A.CallTo(() => snapshotStore.WriteAsync(key, "4", 2, 3, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, "4", 3, 2), A._)) .MustHaveHappened(); - A.CallTo(() => snapshotStore.WriteAsync(key, "5", 3, 4, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, "5", 4, 3), A._)) .MustHaveHappened(); } @@ -290,7 +290,7 @@ namespace Squidex.Infrastructure.States public async Task Should_not_write_snapshot_to_store_if_not_changed() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns(("0", true, 2)); + .Returns(new SnapshotResult(key, "0", 2)); SetupEventStore(3); @@ -302,7 +302,7 @@ namespace Squidex.Infrastructure.States await persistence.WriteSnapshotAsync("4"); - A.CallTo(() => snapshotStore.WriteAsync(key, A._, A._, A._, A._)) + A.CallTo(() => snapshotStore.WriteAsync(A>._, A._)) .MustNotHaveHappened(); } diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs index 2d958f8a6..d5d2fb507 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/States/PersistenceSnapshotTests.cs @@ -29,7 +29,7 @@ namespace Squidex.Infrastructure.States public async Task Should_read_from_store() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, true, 10)); + .Returns(new SnapshotResult(key, 20, 10)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -44,7 +44,7 @@ namespace Squidex.Infrastructure.States public async Task Should_not_read_from_store_if_not_valid() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, false, 10)); + .Returns(new SnapshotResult(key, 20, 10, false)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -59,7 +59,7 @@ namespace Squidex.Infrastructure.States public async Task Should_return_empty_version_if_version_negative() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, true, -10)); + .Returns(new SnapshotResult(key, 20, -10)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -73,7 +73,7 @@ namespace Squidex.Infrastructure.States public async Task Should_set_to_empty_if_store_returns_not_found() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, true, EtagVersion.Empty)); + .Returns(new SnapshotResult(key, 20, EtagVersion.Empty)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -88,7 +88,7 @@ namespace Squidex.Infrastructure.States public async Task Should_throw_exception_if_not_found_and_version_expected() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((123, true, EtagVersion.Empty)); + .Returns(new SnapshotResult(key, 42, EtagVersion.Empty)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -100,7 +100,7 @@ namespace Squidex.Infrastructure.States public async Task Should_throw_exception_if_other_version_found() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((123, true, 2)); + .Returns(new SnapshotResult(key, 42, 2)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -112,7 +112,7 @@ namespace Squidex.Infrastructure.States public async Task Should_write_to_store_with_previous_version() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, true, 10)); + .Returns(new SnapshotResult(key, 20, 10)); var persistedState = Save.Snapshot(0); var persistence = sut.WithSnapshots(None.Type, key, persistedState.Write); @@ -124,7 +124,7 @@ namespace Squidex.Infrastructure.States await persistence.WriteSnapshotAsync(100); - A.CallTo(() => snapshotStore.WriteAsync(key, 100, 10, 11, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, 100, 11, 10), A._)) .MustHaveHappened(); } @@ -135,7 +135,7 @@ namespace Squidex.Infrastructure.States await persistence.WriteSnapshotAsync(100); - A.CallTo(() => snapshotStore.WriteAsync(key, 100, EtagVersion.Empty, 0, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, 100, 0, -1), A._)) .MustHaveHappened(); } @@ -143,9 +143,9 @@ namespace Squidex.Infrastructure.States public async Task Should_not_wrap_exception_if_writing_to_store_with_previous_version() { A.CallTo(() => snapshotStore.ReadAsync(key, A._)) - .Returns((20, true, 10)); + .Returns(new SnapshotResult(key, 42, 10)); - A.CallTo(() => snapshotStore.WriteAsync(key, 100, 10, 11, A._)) + A.CallTo(() => snapshotStore.WriteAsync(new SnapshotWriteJob(key, 100, 11, 10), A._)) .Throws(new InconsistentStateException(1, 1, new InvalidOperationException())); var persistedState = Save.Snapshot(0); diff --git a/frontend/app/framework/angular/pager.component.html b/frontend/app/framework/angular/pager.component.html new file mode 100644 index 000000000..4b108be72 --- /dev/null +++ b/frontend/app/framework/angular/pager.component.html @@ -0,0 +1,28 @@ +
+ +
\ No newline at end of file diff --git a/frontend/src/app/features/administration/state/users.state.ts b/frontend/src/app/features/administration/state/users.state.ts index 17e181e52..e45e79ec6 100644 --- a/frontend/src/app/features/administration/state/users.state.ts +++ b/frontend/src/app/features/administration/state/users.state.ts @@ -187,7 +187,7 @@ export class UsersState extends State { } public search(query: string) { - if (!this.next({ query, page: 0 }, 'Loading Search')) { + if (!this.next({ query, page: 0, total: 0 }, 'Loading Search')) { return EMPTY; } diff --git a/frontend/src/app/features/assets/pages/assets-page.component.html b/frontend/src/app/features/assets/pages/assets-page.component.html index 59ae6a942..237d0e4ec 100644 --- a/frontend/src/app/features/assets/pages/assets-page.component.html +++ b/frontend/src/app/features/assets/pages/assets-page.component.html @@ -59,7 +59,7 @@ - + diff --git a/frontend/src/app/features/assets/pages/assets-page.component.ts b/frontend/src/app/features/assets/pages/assets-page.component.ts index 66cd8a4ef..189ab56ca 100644 --- a/frontend/src/app/features/assets/pages/assets-page.component.ts +++ b/frontend/src/app/features/assets/pages/assets-page.component.ts @@ -45,7 +45,7 @@ export class AssetsPageComponent extends ResourceOwner implements OnInit { .withSynchronizer(QueryFullTextSynchronizer.INSTANCE) .getInitial(); - this.assetsState.load(false, initial); + this.assetsState.load(false, true, initial); this.assetsRoute.listen(); } @@ -53,6 +53,10 @@ export class AssetsPageComponent extends ResourceOwner implements OnInit { this.assetsState.load(true); } + public reloadTotal() { + this.assetsState.load(true, false); + } + public search(query: Query) { this.assetsState.search(query); } diff --git a/frontend/src/app/features/content/pages/contents/contents-page.component.html b/frontend/src/app/features/content/pages/contents/contents-page.component.html index d226449d2..0b087154b 100644 --- a/frontend/src/app/features/content/pages/contents/contents-page.component.html +++ b/frontend/src/app/features/content/pages/contents/contents-page.component.html @@ -134,7 +134,7 @@ - + diff --git a/frontend/src/app/features/content/pages/contents/contents-page.component.ts b/frontend/src/app/features/content/pages/contents/contents-page.component.ts index ac8845b35..88189a72c 100644 --- a/frontend/src/app/features/content/pages/contents/contents-page.component.ts +++ b/frontend/src/app/features/content/pages/contents/contents-page.component.ts @@ -102,7 +102,7 @@ export class ContentsPageComponent extends ResourceOwner implements OnInit { .withSynchronizer(QuerySynchronizer.INSTANCE) .getInitial(); - this.contentsState.load(false, initial); + this.contentsState.load(false, true, initial); this.contentsRoute.listen(); const languageKey = this.localStore.get(this.languageKey()); @@ -124,6 +124,10 @@ export class ContentsPageComponent extends ResourceOwner implements OnInit { this.contentsState.load(true); } + public reloadTotal() { + this.contentsState.load(true, false); + } + public delete(content: ContentDto) { this.contentsState.deleteMany([content]); } diff --git a/frontend/src/app/framework/angular/pager.component.html b/frontend/src/app/framework/angular/pager.component.html index 0b2fd4cbc..37b92cfb4 100644 --- a/frontend/src/app/framework/angular/pager.component.html +++ b/frontend/src/app/framework/angular/pager.component.html @@ -5,7 +5,17 @@ - {{ 'common.pagerInfo' | sqxTranslate: translationInfo }} + + + + + + +