From b5b68de4ebeaec16b5d75c3ee644ba8c4e36262b Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 29 Jul 2022 18:35:47 +0200 Subject: [PATCH] Consistency improvements (#906) * Improve consistency for content repositories. * Temp * Improved consistency for assets. * Improve tests * Fixes to app plan management * Remove referrer * Fixes cancellation token. * Fix method naming. * Remove test loop. * Another test to improve tests * Small improvement for tests. * Perhaps the potential to get better error messages. * Revert batch change. * Test logging. * Fix JSON * Better polling subscription. * Store recent events. * Test output. * Test * Another test * Fix asset deleter. * Build fix. * Revert "Store recent events." This reverts commit 127491d61dbfa371dfd8e9d93f344f9b4c9b6456. --- .../Tags/ITagService.cs | 7 +- .../Tags/Tag.cs | 4 +- .../Tags/TagNormalizer.cs | 151 --------- .../Tags/TagsExport.cs | 23 +- ...ongoAssetFolderRepository_SnapshotStore.cs | 4 +- .../Assets/MongoAssetRepository.cs | 2 +- .../MongoAssetRepository_SnapshotStore.cs | 4 +- .../Contents/MongoContentCollection.cs | 33 +- .../Contents/MongoContentRepository.cs | 11 + .../MongoContentRepository_SnapshotStore.cs | 80 +++-- .../Operations/QueryInDedicatedCollection.cs | 23 +- .../Apps/AppUISettings.cs | 16 +- .../Apps/Commands/ChangePlan.cs | 2 - .../Apps/DomainObject/AppDomainObject.cs | 105 +++--- .../Apps/Plans/IAppPlanBillingManager.cs | 8 +- .../Apps/Plans/NoopAppPlanBillingManager.cs | 20 +- .../Apps/Plans/PlanChangeAsyncResult.cs | 13 - .../Apps/Plans/PlanChangedResult.cs | 4 +- .../Apps/Plans/RedirectToCheckoutResult.cs | 23 -- .../Assets/AssetUsageTracker.cs | 22 +- .../Assets/AssetUsageTracker_EventHandling.cs | 160 ++++++++- .../Assets/BackupAssets.cs | 29 +- .../DomainObject/AssetCommandMiddleware.cs | 5 - .../Assets/DomainObject/AssetDomainObject.cs | 8 +- .../DomainObject/Guards/TagsExtensions.cs | 19 +- .../Assets/Queries/AssetEnricher.cs | 2 +- .../Comments/WatchingService.cs | 4 +- .../Contents/Counter/CounterService.cs | 8 +- .../OperationContextBase.cs | 16 +- .../Tags/TagService.cs | 254 +++++++++------ .../Assets/AssetDeleted.cs | 2 + .../MongoDb/MongoBase.cs | 3 - .../MongoDb/MongoExtensions.cs | 49 ++- .../States/MongoSnapshotStoreBase.cs | 14 +- .../UsageTracking/MongoUsageRepository.cs | 9 + .../CollectionExtensions.cs | 5 - .../Consume/EventConsumerManager.cs | 8 +- .../Consume/EventConsumerProcessor.cs | 17 +- .../EventSourcing/IEventConsumer.cs | 2 + .../States/IOnRead.cs} | 5 +- .../States/ISnapshotStore.cs | 8 +- .../States/NameReservationState.cs | 10 +- .../States/SimpleState.cs | 66 +--- .../UsageTracking/BackgroundUsageTracker.cs | 13 +- .../UsageTracking/CachingUsageTracker.cs | 13 +- .../UsageTracking/IUsageRepository.cs | 3 + .../UsageTracking/IUsageTracker.cs | 5 + .../Controllers/Plans/AppPlansController.cs | 6 +- .../Controllers/Plans/Models/ChangePlanDto.cs | 4 +- .../Squidex/Config/Domain/LoggingServices.cs | 4 +- .../Operations/Tags/TagNormalizerTests.cs | 136 -------- .../Apps/DomainObject/AppDomainObjectTests.cs | 63 ++-- .../Plans/NoopAppPlanBillingManagerTests.cs | 22 +- .../Assets/AssetUsageTrackerTests.cs | 307 +++++++++++++++++- .../Assets/BackupAssetsTests.cs | 25 +- .../DomainObject/AssetDomainObjectTests.cs | 6 +- .../Assets/Queries/AssetEnricherTests.cs | 4 +- .../Tags/TagServiceTests.cs | 209 ++++++++---- .../CollectionExtensionsTests.cs | 20 +- .../Consume/EventConsumerManagerTests.cs | 62 ++-- .../Consume/EventConsumerProcessorTests.cs | 23 ++ .../EventSourcing/EventStoreTests.cs | 31 +- .../EventSourcing/GetEventStoreTests.cs | 2 - .../MongoEventStoreTests_Direct.cs | 2 - .../MongoEventStoreTests_ReplicaSet.cs | 2 - .../EventSourcing/PollingSubscriptionTests.cs | 116 +++++-- .../States/SimpleStateTests.cs | 208 ++++++++++++ .../TestHelpers/TestState.cs | 37 ++- .../BackgroundUsageTrackerTests.cs | 15 + .../UsageTracking/CachingUsageTrackerTests.cs | 18 + .../TestSuite.ApiTests/AssetTests.cs | 67 +++- .../TestSuite.Shared/ClientExtensions.cs | 25 ++ 72 files changed, 1825 insertions(+), 881 deletions(-) delete mode 100644 backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs delete mode 100644 backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangeAsyncResult.cs delete mode 100644 backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/RedirectToCheckoutResult.cs rename backend/src/{Squidex.Domain.Apps.Entities/Apps/Plans/IChangePlanResult.cs => Squidex.Infrastructure/States/IOnRead.cs} (79%) delete mode 100644 backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs index f8a64343c..5a6a93049 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs @@ -14,10 +14,10 @@ namespace Squidex.Domain.Apps.Core.Tags Task> GetTagIdsAsync(DomainId id, string group, HashSet names, CancellationToken ct = default); - Task> NormalizeTagsAsync(DomainId id, string group, HashSet? names, HashSet? ids, + Task> GetTagNamesAsync(DomainId id, string group, HashSet ids, CancellationToken ct = default); - Task> DenormalizeTagsAsync(DomainId id, string group, HashSet ids, + Task UpdateAsync(DomainId id, string group, Dictionary updates, CancellationToken ct = default); Task GetTagsAsync(DomainId id, string group, @@ -34,5 +34,8 @@ namespace Squidex.Domain.Apps.Core.Tags Task ClearAsync(DomainId id, string group, CancellationToken ct = default); + + Task ClearAsync( + CancellationToken ct = default); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/Tag.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/Tag.cs index 1bff58c70..ff47a46c1 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/Tag.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/Tag.cs @@ -7,10 +7,10 @@ namespace Squidex.Domain.Apps.Core.Tags { - public sealed class Tag + public sealed record Tag { public string Name { get; set; } - public int Count { get; set; } = 1; + public int Count { get; set; } } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs deleted file mode 100644 index fb6bf7dc3..000000000 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs +++ /dev/null @@ -1,151 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using Squidex.Domain.Apps.Core.Contents; -using Squidex.Domain.Apps.Core.Schemas; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Json.Objects; - -namespace Squidex.Domain.Apps.Core.Tags -{ - public static class TagNormalizer - { - public static async Task NormalizeAsync(this ITagService tagService, DomainId appId, DomainId schemaId, Schema schema, ContentData newData, ContentData? oldData) - { - Guard.NotNull(tagService); - Guard.NotNull(schema); - Guard.NotNull(newData); - - var newValues = new HashSet(); - var newArrays = new List(); - - var oldValues = new HashSet(); - var oldArrays = new List(); - - GetValues(schema, newValues, newArrays, newData); - - if (oldData != null) - { - GetValues(schema, oldValues, oldArrays, oldData); - } - - if (newValues.Count > 0) - { - var normalized = await tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), newValues, oldValues); - - foreach (var source in newArrays) - { - var array = source.AsArray; - - for (var i = 0; i < array.Count; i++) - { - if (normalized.TryGetValue(array[i].ToString(), out var result)) - { - array[i] = result; - } - } - } - } - } - - public static async Task DenormalizeAsync(this ITagService tagService, DomainId appId, DomainId schemaId, Schema schema, params ContentData[] datas) - { - Guard.NotNull(tagService); - Guard.NotNull(schema); - - var tagsValues = new HashSet(); - var tagsArrays = new List(); - - GetValues(schema, tagsValues, tagsArrays, datas); - - if (tagsValues.Count > 0) - { - var denormalized = await tagService.DenormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), tagsValues); - - foreach (var source in tagsArrays) - { - var array = source.AsArray; - - for (var i = 0; i < array.Count; i++) - { - if (denormalized.TryGetValue(array[i].ToString(), out var result)) - { - array[i] = result; - } - } - } - } - } - - private static void GetValues(Schema schema, HashSet values, List arrays, params ContentData[] datas) - { - foreach (var field in schema.Fields) - { - if (field is IField tags && tags.Properties.Normalization == TagsFieldNormalization.Schema) - { - foreach (var data in datas) - { - if (data.TryGetValue(field.Name, out var fieldData) && fieldData != null) - { - foreach (var partition in fieldData) - { - ExtractTags(partition.Value, values, arrays); - } - } - } - } - else if (field is IArrayField arrayField) - { - foreach (var nestedField in arrayField.Fields) - { - if (nestedField is IField nestedTags && nestedTags.Properties.Normalization == TagsFieldNormalization.Schema) - { - foreach (var data in datas) - { - if (data.TryGetValue(field.Name, out var fieldData) && fieldData != null) - { - foreach (var partition in fieldData) - { - if (partition.Value.Value is JsonArray a) - { - foreach (var value in a) - { - if (value.Value is JsonObject o) - { - if (o.TryGetValue(nestedField.Name, out var nestedValue)) - { - ExtractTags(nestedValue, values, arrays); - } - } - } - } - } - } - } - } - } - } - } - } - - private static void ExtractTags(JsonValue value, ISet values, ICollection arrays) - { - if (value.Value is JsonArray a) - { - foreach (var item in a) - { - if (item.Value is string s) - { - values.Add(s); - } - } - - arrays.Add(value); - } - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs index 872fdee05..b7e0c81b2 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs @@ -9,27 +9,8 @@ namespace Squidex.Domain.Apps.Core.Tags { public class TagsExport { - public Dictionary? Tags { get; set; } + public Dictionary Tags { get; set; } = new Dictionary(); - public Dictionary? Alias { get; set; } - - public TagsExport Clone() - { - var alias = (Dictionary?)null; - - if (Alias != null) - { - alias = new Dictionary(Alias); - } - - var tags = (Dictionary?)null; - - if (Tags != null) - { - tags = new Dictionary(Tags); - } - - return new TagsExport { Alias = alias, Tags = tags }; - } + public Dictionary Alias { get; set; } = new Dictionary(); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs index 9319377f8..23eca83ed 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs @@ -55,9 +55,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/WriteAsync")) { - var entity = MongoAssetFolderEntity.Create(job); + var entityJob = job.As(MongoAssetFolderEntity.Create(job)); - await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct); + await Collection.UpsertVersionedAsync(entityJob, ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index 6145576bc..018757f8c 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -260,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets Filter.Gt(x => x.Id, DomainId.Create(string.Empty)), Filter.Eq(x => x.IndexedAppId, appId), Filter.Ne(x => x.IsDeleted, true), - Filter.Ne(x => x.ParentId, parentId)); + Filter.Eq(x => x.ParentId, parentId)); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs index 0127ed81b..5623a8ebe 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs @@ -55,9 +55,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { using (Telemetry.Activities.StartActivity("MongoAssetRepository/WriteAsync")) { - var entity = MongoAssetEntity.Create(job); + var entityJob = job.As(MongoAssetEntity.Create(job)); - await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct); + await Collection.UpsertVersionedAsync(entityJob, ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index 4b6d2b551..51fe183c6 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -16,6 +16,7 @@ using Squidex.Domain.Apps.Entities.Schemas; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.Queries; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.Translations; #pragma warning disable IDE0060 // Remove unused parameter @@ -251,25 +252,47 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents return Collection.Find(FindAll).ToAsyncEnumerable(ct); } - public async Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity value, + public async Task UpsertAsync(SnapshotWriteJob job, CancellationToken ct = default) { if (queryInDedicatedCollection != null) { - await queryInDedicatedCollection.UpsertVersionedAsync(documentId, oldVersion, value, default); + await queryInDedicatedCollection.UpsertAsync(job, ct); } - await Collection.UpsertVersionedAsync(documentId, oldVersion, value.Version, value, default); + await Collection.ReplaceOneAsync(Filter.Eq(x => x.DocumentId, job.Key), job.Value, UpsertReplace, ct); + } + + public async Task UpsertVersionedAsync(IClientSessionHandle session, SnapshotWriteJob job, + CancellationToken ct = default) + { + if (queryInDedicatedCollection != null) + { + await queryInDedicatedCollection.UpsertVersionedAsync(session, job, ct); + } + + await Collection.UpsertVersionedAsync(session, job, ct); } public async Task RemoveAsync(DomainId key, CancellationToken ct = default) { - var previous = await Collection.FindOneAndDeleteAsync(x => x.DocumentId == key, null, default); + var previous = await Collection.FindOneAndDeleteAsync(x => x.DocumentId == key, null, ct); + + if (queryInDedicatedCollection != null && previous != null) + { + await queryInDedicatedCollection.RemoveAsync(previous, ct); + } + } + + public async Task RemoveAsync(IClientSessionHandle session, DomainId key, + CancellationToken ct = default) + { + var previous = await Collection.FindOneAndDeleteAsync(session, x => x.DocumentId == key, null, ct); if (queryInDedicatedCollection != null && previous != null) { - await queryInDedicatedCollection.RemoveAsync(previous, default); + await queryInDedicatedCollection.RemoveAsync(session, previous, ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 810b8f6a3..65c9ffd1b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Options; using MongoDB.Driver; +using MongoDB.Driver.Core.Clusters; using NodaTime; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Apps; @@ -24,8 +25,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { private readonly MongoContentCollection collectionComplete; private readonly MongoContentCollection collectionPublished; + private readonly IMongoDatabase database; private readonly IAppProvider appProvider; + public bool CanUseTransactions { get; private set; } + static MongoContentRepository() { BsonStringSerializer.Register(); @@ -34,6 +38,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider, IOptions options) { + this.database = database; + collectionComplete = new MongoContentCollection("States_Contents_All3", database, ReadPreference.Primary, options.Value.OptimizeForSelfHosting); @@ -50,6 +56,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { await collectionComplete.InitializeAsync(ct); await collectionPublished.InitializeAsync(ct); + + var clusterVersion = await database.GetMajorVersionAsync(ct); + var clusteredAsReplica = database.Client.Cluster.Description.Type == ClusterType.ReplicaSet; + + CanUseTransactions = clusteredAsReplica && clusterVersion >= 4; } public IAsyncEnumerable StreamAll(DomainId appId, HashSet? schemaIds, diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs index c6ba30681..a5f5f65f3 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs @@ -33,6 +33,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents var existing = await collectionComplete.FindAsync(key, ct); + // Support for all versions, where we do not have full snapshots in the collection. if (existing?.IsSnapshot == true) { return new SnapshotResult(existing.DocumentId, existing.ToState(), existing.Version); @@ -67,6 +68,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { using (Telemetry.Activities.StartActivity("MongoContentRepository/RemoveAsync")) { + // Some data is corrupt and might throw an exception if we do not ignore it. if (key == DomainId.Empty) { return; @@ -83,14 +85,33 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteAsync")) { + // Some data is corrupt and might throw an exception if we do not ignore it. if (!IsValid(job.Value)) { return; } - await Task.WhenAll( - UpsertFrontendAsync(job, ct), - UpsertPublishedAsync(job, ct)); + if (!CanUseTransactions) + { + // If transactions are not supported we update the documents without version checks, + // otherwise we would not be able to recover from inconsistencies. + await Task.WhenAll( + UpsertCompleteAsync(job, default), + UpsertPublishedAsync(job, default)); + return; + } + + using (var session = await database.Client.StartSessionAsync(cancellationToken: ct)) + { + // Make an update with full transaction support to be more consistent. + await session.WithTransactionAsync(async (session, ct) => + { + await Task.WhenAll( + UpsertVersionedCompleteAsync(session, job, ct), + UpsertVersionedPublishedAsync(session, job, ct)); + return true; + }, null, ct); + } } } @@ -99,11 +120,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteManyAsync")) { - var updates = new Dictionary, List>(); + var collectionUpdates = new Dictionary, List>(); var add = new Action, MongoContentEntity>((collection, entity) => { - updates.GetOrAddNew(collection).Add(entity); + collectionUpdates.GetOrAddNew(collection).Add(entity); }); foreach (var job in jobs) @@ -123,7 +144,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } } - await Parallel.ForEachAsync(updates, ct, (update, ct) => + var parallelOptions = new ParallelOptions + { + CancellationToken = ct, + // This is just an estimate, but we do not want ot have unlimited parallelism. + MaxDegreeOfParallelism = 8 + }; + + // Make one update per collection. + await Parallel.ForEachAsync(collectionUpdates, parallelOptions, (update, ct) => { return new ValueTask(update.Key.InsertManyAsync(update.Value, InsertUnordered, ct)); }); @@ -131,38 +160,49 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents } private async Task UpsertPublishedAsync(SnapshotWriteJob job, - CancellationToken ct = default) + CancellationToken ct) { if (ShouldWritePublished(job.Value)) { - await UpsertPublishedContentAsync(job, ct); + var entityJob = job.As(await MongoContentEntity.CreatePublishedAsync(job, appProvider)); + + await collectionPublished.UpsertAsync(entityJob, ct); } else { - await DeletePublishedContentAsync(job.Value.UniqueId, ct); + await collectionPublished.RemoveAsync(job.Key, ct); } } - private Task DeletePublishedContentAsync(DomainId key, - CancellationToken ct = default) + private async Task UpsertVersionedPublishedAsync(IClientSessionHandle session, SnapshotWriteJob job, + CancellationToken ct) { - return collectionPublished.RemoveAsync(key, ct); + if (ShouldWritePublished(job.Value)) + { + var entityJob = job.As(await MongoContentEntity.CreatePublishedAsync(job, appProvider)); + + await collectionPublished.UpsertVersionedAsync(session, entityJob, ct); + } + else + { + await collectionPublished.RemoveAsync(session, job.Key, ct); + } } - private async Task UpsertFrontendAsync(SnapshotWriteJob job, - CancellationToken ct = default) + private async Task UpsertCompleteAsync(SnapshotWriteJob job, + CancellationToken ct) { - var entity = await MongoContentEntity.CreateAsync(job, appProvider); + var entityJob = job.As(await MongoContentEntity.CreateAsync(job, appProvider)); - await collectionComplete.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct); + await collectionComplete.UpsertAsync(entityJob, ct); } - private async Task UpsertPublishedContentAsync(SnapshotWriteJob job, - CancellationToken ct = default) + private async Task UpsertVersionedCompleteAsync(IClientSessionHandle session, SnapshotWriteJob job, + CancellationToken ct) { - var entity = await MongoContentEntity.CreatePublishedAsync(job, appProvider); + var entityJob = job.As(await MongoContentEntity.CreateAsync(job, appProvider)); - await collectionPublished.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct); + await collectionComplete.UpsertVersionedAsync(session, entityJob, ct); } private static bool ShouldWritePublished(ContentDomainObject.State value) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryInDedicatedCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryInDedicatedCollection.cs index 998b110b2..ca664a1ea 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryInDedicatedCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryInDedicatedCollection.cs @@ -15,6 +15,7 @@ using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.MongoDb.Queries; using Squidex.Infrastructure.Queries; +using Squidex.Infrastructure.States; namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations { @@ -110,12 +111,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations return ResultList.Create(contentTotal, contentEntities); } - public async Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity value, + public async Task UpsertAsync(SnapshotWriteJob job, CancellationToken ct = default) { - var collection = await GetCollectionAsync(value.AppId.Id, value.SchemaId.Id); + var collection = await GetCollectionAsync(job.Value.AppId.Id, job.Value.SchemaId.Id); + + await collection.ReplaceOneAsync(Filter.Eq(x => x.DocumentId, job.Key), job.Value, UpsertReplace, ct); + } + + public async Task UpsertVersionedAsync(IClientSessionHandle session, SnapshotWriteJob job, + CancellationToken ct = default) + { + var collection = await GetCollectionAsync(job.Value.AppId.Id, job.Value.SchemaId.Id); - await collection.UpsertVersionedAsync(documentId, oldVersion, value.Version, value, ct); + await collection.UpsertVersionedAsync(session, job, ct); } public async Task RemoveAsync(MongoContentEntity value, @@ -126,6 +135,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations await collection.DeleteOneAsync(x => x.DocumentId == value.DocumentId, null, ct); } + public async Task RemoveAsync(IClientSessionHandle session, MongoContentEntity value, + CancellationToken ct = default) + { + var collection = await GetCollectionAsync(value.AppId.Id, value.SchemaId.Id); + + await collection.DeleteOneAsync(session, x => x.DocumentId == value.DocumentId, null, ct); + } + private static FilterDefinition BuildFilter(FilterNode? filter) { var filters = new List> diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/AppUISettings.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/AppUISettings.cs index 23211c02d..e75b8f3f2 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/AppUISettings.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/AppUISettings.cs @@ -20,9 +20,17 @@ namespace Squidex.Domain.Apps.Entities.Apps { public JsonObject Settings { get; set; } = new JsonObject(); - public void Set(JsonObject settings) + public bool Set(JsonObject settings) { - Settings = settings; + var isChanged = false; + + if (!Settings.Equals(settings)) + { + Settings = settings; + isChanged = true; + } + + return isChanged; } public bool Set(string path, JsonValue value) @@ -134,7 +142,7 @@ namespace Squidex.Domain.Apps.Entities.Apps { var state = await GetStateAsync(appId, userId, ct); - await state.UpdateIfAsync(s => s.Remove(path), ct: ct); + await state.UpdateAsync(s => s.Remove(path), ct: ct); } public async Task SetAsync(DomainId appId, string? userId, string path, JsonValue value, @@ -142,7 +150,7 @@ namespace Squidex.Domain.Apps.Entities.Apps { var state = await GetStateAsync(appId, userId, ct); - await state.UpdateIfAsync(s => s.Set(path, value), ct: ct); + await state.UpdateAsync(s => s.Set(path, value), ct: ct); } public async Task SetAsync(DomainId appId, string? userId, JsonObject settings, diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/ChangePlan.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/ChangePlan.cs index 9d3424211..28351c602 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/ChangePlan.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/ChangePlan.cs @@ -12,7 +12,5 @@ namespace Squidex.Domain.Apps.Entities.Apps.Commands public bool FromCallback { get; set; } public string PlanId { get; set; } - - public string Referer { get; set; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs index ec6799875..611d97f8e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs @@ -118,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject case AssignContributor assignContributor: return UpdateReturnAsync(assignContributor, async (c, ct) => { - await GuardAppContributors.CanAssign(c, Snapshot, UserResolver(), GetPlan()); + await GuardAppContributors.CanAssign(c, Snapshot, Users(), GetPlan()); AssignContributor(c, !Snapshot.Contributors.ContainsKey(assignContributor.ContributorId)); @@ -255,46 +255,71 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject return Snapshot; }, ct); - case ChangePlan changePlan: - return UpdateReturnAsync(changePlan, async (c, ct) => - { - GuardApp.CanChangePlan(c, Snapshot, AppPlansProvider()); - - if (c.FromCallback) - { - ChangePlan(c); - - return null; - } - else - { - var result = await AppPlanBillingManager().ChangePlanAsync(c.Actor.Identifier, Snapshot.NamedId(), c.PlanId, c.Referer, default); - - switch (result) - { - case PlanChangedResult: - ChangePlan(c); - break; - } - - return result; - } - }, ct); - case DeleteApp delete: return UpdateAsync(delete, async (c, ct) => { - await AppPlanBillingManager().ChangePlanAsync(c.Actor.Identifier, Snapshot.NamedId(), null, null, ct); + await Billing().UnsubscribeAsync(c.Actor.Identifier, Snapshot.NamedId(), default); DeleteApp(c); }, ct); + case ChangePlan changePlan: + return ChangeBillingPlanAsync(changePlan, ct); + default: ThrowHelper.NotSupportedException(); return default!; } } + private async Task ChangeBillingPlanAsync(ChangePlan changePlan, + CancellationToken ct) + { + var userId = changePlan.Actor.Identifier; + + var result = await UpdateReturnAsync(changePlan, async (c, ct) => + { + GuardApp.CanChangePlan(c, Snapshot, Plans()); + + if (string.Equals(GetFreePlan()?.Id, c.PlanId, StringComparison.Ordinal)) + { + ResetPlan(c); + + return new PlanChangedResult(c.PlanId, true, null); + } + + if (!c.FromCallback) + { + var redirectUri = await Billing().MustRedirectToPortalAsync(userId, Snapshot.NamedId(), c.PlanId, ct); + + if (redirectUri != null) + { + return new PlanChangedResult(c.PlanId, false, redirectUri); + } + } + + ChangePlan(c); + + return new PlanChangedResult(c.PlanId); + }, ct); + + if (changePlan.FromCallback) + { + return result; + } + + if (result.Payload is PlanChangedResult { Unsubscribed: true, RedirectUri: null }) + { + await Billing().UnsubscribeAsync(userId, Snapshot.NamedId(), default); + } + else if (result.Payload is PlanChangedResult { RedirectUri: null }) + { + await Billing().SubscribeAsync(userId, Snapshot.NamedId(), changePlan.PlanId, default); + } + + return result; + } + private void Create(CreateApp command) { var appId = NamedId.Of(command.AppId, command.Name); @@ -321,14 +346,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject private void ChangePlan(ChangePlan command) { - if (string.Equals(GetFreePlan()?.Id, command.PlanId, StringComparison.Ordinal)) - { - Raise(command, new AppPlanReset()); - } - else - { - Raise(command, new AppPlanChanged()); - } + Raise(command, new AppPlanChanged()); + } + + private void ResetPlan(ChangePlan command) + { + Raise(command, new AppPlanReset()); } private void Update(UpdateApp command) @@ -455,29 +478,29 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject return new AppSettingsUpdated { Settings = serviceProvider.GetRequiredService().Settings }; } - private IAppPlansProvider AppPlansProvider() + private IAppPlansProvider Plans() { return serviceProvider.GetRequiredService(); } - private IAppPlanBillingManager AppPlanBillingManager() + private IAppPlanBillingManager Billing() { return serviceProvider.GetRequiredService(); } - private IUserResolver UserResolver() + private IUserResolver Users() { return serviceProvider.GetRequiredService(); } private IAppLimitsPlan GetFreePlan() { - return AppPlansProvider().GetFreePlan(); + return Plans().GetFreePlan(); } private IAppLimitsPlan GetPlan() { - return AppPlansProvider().GetPlanForApp(Snapshot).Plan; + return Plans().GetPlanForApp(Snapshot).Plan; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IAppPlanBillingManager.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IAppPlanBillingManager.cs index 9f9fb5571..963f935a5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IAppPlanBillingManager.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IAppPlanBillingManager.cs @@ -13,7 +13,13 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans { bool HasPortal { get; } - Task ChangePlanAsync(string userId, NamedId appId, string? planId, string? referer, + Task MustRedirectToPortalAsync(string userId, NamedId appId, string? planId, + CancellationToken ct = default); + + Task SubscribeAsync(string userId, NamedId appId, string planId, + CancellationToken ct = default); + + Task UnsubscribeAsync(string userId, NamedId appId, CancellationToken ct = default); Task GetPortalLinkAsync(string userId, diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/NoopAppPlanBillingManager.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/NoopAppPlanBillingManager.cs index ccd571397..2a3d22307 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/NoopAppPlanBillingManager.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/NoopAppPlanBillingManager.cs @@ -16,16 +16,28 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans get => false; } - public Task ChangePlanAsync(string userId, NamedId appId, string? planId, string? referer, + public Task GetPortalLinkAsync(string userId, CancellationToken ct = default) { - return Task.FromResult(new PlanChangedResult()); + return Task.FromResult(string.Empty); } - public Task GetPortalLinkAsync(string userId, + public Task MustRedirectToPortalAsync(string userId, NamedId appId, string? planId, CancellationToken ct = default) { - return Task.FromResult(string.Empty); + return Task.FromResult(null); + } + + public Task SubscribeAsync(string userId, NamedId appId, string planId, + CancellationToken ct = default) + { + return Task.CompletedTask; + } + + public Task UnsubscribeAsync(string userId, NamedId appId, + CancellationToken ct = default) + { + return Task.CompletedTask; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangeAsyncResult.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangeAsyncResult.cs deleted file mode 100644 index 6715dc0d5..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangeAsyncResult.cs +++ /dev/null @@ -1,13 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -namespace Squidex.Domain.Apps.Entities.Apps.Plans -{ - public sealed class PlanChangeAsyncResult : IChangePlanResult - { - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangedResult.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangedResult.cs index df5d47d86..bbe78e3d3 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangedResult.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangedResult.cs @@ -5,9 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + namespace Squidex.Domain.Apps.Entities.Apps.Plans { - public sealed class PlanChangedResult : IChangePlanResult + public sealed record PlanChangedResult(string PlanId, bool Unsubscribed = false, Uri? RedirectUri = null) { } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/RedirectToCheckoutResult.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/RedirectToCheckoutResult.cs deleted file mode 100644 index c6b84bf60..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/RedirectToCheckoutResult.cs +++ /dev/null @@ -1,23 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities.Apps.Plans -{ - public sealed class RedirectToCheckoutResult : IChangePlanResult - { - public Uri Url { get; } - - public RedirectToCheckoutResult(Uri url) - { - Guard.NotNull(url); - - Url = url; - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs index 2fc6df82f..f08afe66e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs @@ -5,8 +5,10 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Infrastructure; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; #pragma warning disable CS0649 @@ -18,11 +20,26 @@ namespace Squidex.Domain.Apps.Entities.Assets private const string CounterTotalCount = "TotalAssets"; private const string CounterTotalSize = "TotalSize"; private static readonly DateTime SummaryDate; + private readonly IAssetLoader assetLoader; + private readonly ISnapshotStore store; + private readonly ITagService tagService; private readonly IUsageTracker usageTracker; - public AssetUsageTracker(IUsageTracker usageTracker) + [CollectionName("Index_TagHistory")] + public sealed class State { + public HashSet? Tags { get; set; } + } + + public AssetUsageTracker(IUsageTracker usageTracker, IAssetLoader assetLoader, ITagService tagService, + ISnapshotStore store) + { + this.assetLoader = assetLoader; + this.tagService = tagService; + this.store = store; this.usageTracker = usageTracker; + + ClearCache(); } Task IDeleter.DeleteAppAsync(IAppEntity app, @@ -48,12 +65,13 @@ namespace Squidex.Domain.Apps.Entities.Assets var usages = await usageTracker.QueryAsync(GetKey(appId), fromDate, toDate); - if (usages.TryGetValue("*", out var byCategory1)) + if (usages.TryGetValue(usageTracker.FallbackCategory, out var byCategory1)) { AddCounters(enriched, byCategory1); } else if (usages.TryGetValue("Default", out var byCategory2)) { + // Fallback for older versions where default was uses as tracking category. AddCounters(enriched, byCategory2); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs index cbd821ead..c3367537b 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs @@ -5,9 +5,13 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; +using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; #pragma warning disable MA0048 // File name must match type name @@ -16,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Assets { public partial class AssetUsageTracker : IEventConsumer { + private IMemoryCache memoryCache; + public int BatchSize { get => 1000; @@ -36,18 +42,158 @@ namespace Squidex.Domain.Apps.Entities.Assets get => "^asset-"; } - public Task On(Envelope @event) + private void ClearCache() + { + memoryCache?.Dispose(); + memoryCache = new MemoryCache(Options.Create(new MemoryCacheOptions())); + } + + public async Task ClearAsync() + { + // Will not remove data, but reset alls counts to zero. + await tagService.ClearAsync(); + + // Also clear the store and cache, because otherwise we would use data from the future when querying old tags. + ClearCache(); + + await store.ClearAsync(); + + // Use a well defined prefix query for the deletion to improve performance. + await usageTracker.DeleteByKeyPatternAsync("^([a-zA-Z0-9]+)_Assets"); + } + + public async Task On(IEnumerable> events) + { + foreach (var @event in events) + { + // Usage tracking is done in the backgroud, therefore we do no use any batching. + await TrackUsageAsync(@event); + } + + // Event consumers should only do one task, but too many consumers also hurt performance. + await AddTagsAsync(events); + } + + private async Task AddTagsAsync(IEnumerable> events) + { + var tagsPerApp = new Dictionary>(); + var tagsPerAsset = new Dictionary(); + + void AddTagsToStore(DomainId appId, HashSet? tagIds, int count) + { + if (tagIds != null) + { + var perApp = tagsPerApp.GetOrAddNew(appId); + + foreach (var tag in tagIds) + { + perApp[tag] = perApp.GetOrDefault(tag) + count; + } + } + } + + void AddTagsToCache(DomainId key, HashSet? tags, long version) + { + var state = new State { Tags = tags }; + + // Write tags to a buffer so that we can write them to a store in batches. + tagsPerAsset[key] = state; + + // Write to the cache immediately, to be available for the next event. Use a relatively long cache time for live updates. + memoryCache.Set(key, state, TimeSpan.FromHours(1)); + } + + foreach (var @event in events) + { + var typedEvent = (AssetEvent)@event.Payload; + + var appId = typedEvent.AppId.Id; + var assetId = typedEvent.AssetId; + var assetKey = @event.Headers.AggregateId(); + var version = @event.Headers.EventStreamNumber(); + + switch (typedEvent) + { + case AssetCreated assetCreated: + { + AddTagsToStore(appId, assetCreated.Tags, 1); + AddTagsToCache(assetKey, assetCreated.Tags, version); + break; + } + + case AssetAnnotated assetAnnotated when assetAnnotated.Tags != null: + { + var oldTags = await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default); + + AddTagsToStore(appId, assetAnnotated.Tags, 1); + AddTagsToStore(appId, oldTags, -1); + AddTagsToCache(assetKey, assetAnnotated.Tags, version); + break; + } + + case AssetDeleted assetDeleted: + { + // We need the old tags here for permanent deletions. + var oldTags = + assetDeleted.OldTags ?? + await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default); + + AddTagsToStore(appId, oldTags, -1); + break; + } + } + } + + // There is no good solution for batching anyway, so there is no need to build a method for that. + foreach (var (appId, updates) in tagsPerApp) + { + await tagService.UpdateAsync(appId, TagGroups.Assets, updates); + } + + await store.WriteManyAsync(tagsPerAsset.Select(x => new SnapshotWriteJob(x.Key, x.Value, 0))); + } + + private async Task?> GetAndUpdateOldTagsAsync(DomainId appId, DomainId assetId, DomainId key, long version, + CancellationToken ct) + { + // Store the latest tags in memory for fast access. + if (memoryCache.TryGetValue(key, out var state)) + { + return state.Tags; + } + + var stored = await store.ReadAsync(key, ct); + + // Stored state can be null, if not serialized yet. + if (stored.Value != null) + { + return stored.Value.Tags; + } + + // Some deleted events (like permanent deletion) have version of zero, but there is not previous event. + if (version == 0) + { + return null; + } + + // This will replay a lot of events, so it is the slowest alternative. + var previousAsset = await assetLoader.GetAsync(appId, assetId, version - 1, ct); + + return previousAsset?.Tags; + } + + private Task TrackUsageAsync(Envelope @event) { switch (@event.Payload) { - case AssetCreated e: - return UpdateSizeAsync(e.AppId.Id, GetDate(@event), e.FileSize, 1); + case AssetCreated assetCreated: + return UpdateSizeAsync(assetCreated.AppId.Id, GetDate(@event), assetCreated.FileSize, 1); - case AssetUpdated e: - return UpdateSizeAsync(e.AppId.Id, GetDate(@event), e.FileSize, 0); + case AssetUpdated assetUpdated: + return UpdateSizeAsync(assetUpdated.AppId.Id, GetDate(@event), assetUpdated.FileSize, 0); - case AssetDeleted e: - return UpdateSizeAsync(e.AppId.Id, GetDate(@event), -e.DeletedSize, -1); + case AssetDeleted assetDeleted: + return UpdateSizeAsync(assetDeleted.AppId.Id, GetDate(@event), -assetDeleted.DeletedSize, -1); } return Task.CompletedTask; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index 6aa66feaf..986425707 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -9,6 +9,7 @@ using Squidex.Assets; using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Events.Apps; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.Commands; @@ -71,6 +72,10 @@ namespace Squidex.Domain.Apps.Entities.Assets { switch (@event.Payload) { + case AppCreated: + // Restore the tags first so that the processing of consecutive events have the necessary structure. + await RestoreTagsAsync(context, ct); + break; case AssetFolderCreated: assetFolderIds.Add(@event.Headers.AggregateId()); break; @@ -100,8 +105,6 @@ namespace Squidex.Domain.Apps.Entities.Assets public async Task RestoreAsync(RestoreContext context, CancellationToken ct) { - await RestoreTagsAsync(context, ct); - if (assetIds.Count > 0) { await rebuilder.InsertManyAsync(assetIds, BatchSize, ct); @@ -116,26 +119,32 @@ namespace Squidex.Domain.Apps.Entities.Assets private async Task RestoreTagsAsync(RestoreContext context, CancellationToken ct) { - var tags = (Dictionary?)null; + var export = new TagsExport(); if (await context.Reader.HasFileAsync(TagsFile, ct)) { - tags = await context.Reader.ReadJsonAsync>(TagsFile, ct); + export.Tags = await context.Reader.ReadJsonAsync>(TagsFile, ct); } - var alias = (Dictionary?)null; - + // For backwards compabibility we store the tags and the aliases in different locations. if (await context.Reader.HasFileAsync(TagsAliasFile, ct)) { - alias = await context.Reader.ReadJsonAsync>(TagsAliasFile, ct); + export.Alias = await context.Reader.ReadJsonAsync>(TagsAliasFile, ct); } - if (alias == null && tags == null) + if (export.Alias == null && export.Tags == null) { return; } - var export = new TagsExport { Tags = tags, Alias = alias }; + if (export.Tags != null) + { + // Import the tags without count, because they will populated later by the event processor. + foreach (var (_, tag) in export.Tags) + { + tag.Count = 0; + } + } await tagService.RebuildTagsAsync(context.AppId, TagGroups.Assets, export, ct); } @@ -147,11 +156,13 @@ namespace Squidex.Domain.Apps.Entities.Assets if (tags.Tags != null) { + // Export the tags with count, even though we do not need it. But in general it makes the code easier. await context.Writer.WriteJsonAsync(TagsFile, tags.Tags, ct); } if (tags.Alias?.Count > 0) { + // For backwards compabibility we store the tags and the aliases in different locations. await context.Writer.WriteJsonAsync(TagsAliasFile, tags.Alias, ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs index 5a6e9b1f0..c437eb609 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs @@ -99,8 +99,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject finally { await assetFileStore.DeleteAsync(tempFile, ct); - - await command.File.DisposeAsync(); } } @@ -119,8 +117,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject finally { await assetFileStore.DeleteAsync(tempFile, ct); - - await command.File.DisposeAsync(); } } @@ -134,7 +130,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject if (result.IsChanged && context.Command is UploadAssetCommand) { var tempFile = context.ContextId.ToString(); - try { await assetFileStore.CopyAsync(tempFile, asset.AppId.Id, asset.AssetId, asset.FileVersion, null, ct); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs index ddea4d5bd..8b6077663 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs @@ -166,7 +166,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject if (create.Tags != null) { - create.Tags = await operation.NormalizeTags(create.Tags); + create.Tags = await operation.GetTagIdsAsync(create.Tags); } Create(create); @@ -181,7 +181,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject if (annotate.Tags != null) { - annotate.Tags = await operation.NormalizeTags(annotate.Tags); + annotate.Tags = await operation.GetTagIdsAsync(annotate.Tags); } Annotate(annotate); @@ -224,8 +224,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject await operation.ExecuteDeleteScriptAsync(delete); } - await operation.UnsetTags(); - Delete(delete); } @@ -262,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject private void Delete(DeleteAsset command) { - Raise(command, new AssetDeleted { DeletedSize = Snapshot.TotalSize }); + Raise(command, new AssetDeleted { OldTags = Snapshot.Tags, DeletedSize = Snapshot.TotalSize }); } private void Raise(T command, TEvent @event) where T : class where TEvent : AppEvent diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/Guards/TagsExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/Guards/TagsExtensions.cs index 90ff4beda..7ddc2abc2 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/Guards/TagsExtensions.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/Guards/TagsExtensions.cs @@ -6,25 +6,26 @@ // ========================================================================== using Squidex.Domain.Apps.Core.Tags; +using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Assets.DomainObject.Guards { public static class TagsExtensions { - public static async Task> NormalizeTags(this AssetOperation operation, HashSet tags) + public static async Task> GetTagIdsAsync(this AssetOperation operation, HashSet? names) { - var tagService = operation.Resolve(); + var result = new HashSet(names?.Count ?? 0); - var normalized = await tagService.NormalizeTagsAsync(operation.App.Id, TagGroups.Assets, tags, operation.Snapshot.Tags); + if (names != null) + { + var tagService = operation.Resolve(); - return new HashSet(normalized.Values); - } + var normalized = await tagService.GetTagIdsAsync(operation.App.Id, TagGroups.Assets, names); - public static async Task UnsetTags(this AssetOperation operation) - { - var tagService = operation.Resolve(); + result.AddRange(normalized.Values); + } - await tagService.NormalizeTagsAsync(operation.App.Id, TagGroups.Assets, null, operation.Snapshot.Tags); + return result; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetEnricher.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetEnricher.cs index 068ec4526..baf877d30 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetEnricher.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetEnricher.cs @@ -161,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries { var uniqueIds = group.Where(x => x.Tags != null).SelectMany(x => x.Tags).ToHashSet(); - return await tagService.DenormalizeTagsAsync(group.Key, TagGroups.Assets, uniqueIds, ct); + return await tagService.GetTagNamesAsync(group.Key, TagGroups.Assets, uniqueIds, ct); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Comments/WatchingService.cs b/backend/src/Squidex.Domain.Apps.Entities/Comments/WatchingService.cs index d30a43122..451943f16 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Comments/WatchingService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Comments/WatchingService.cs @@ -22,7 +22,7 @@ namespace Squidex.Domain.Apps.Entities.Comments public Dictionary Users { get; } = new Dictionary(); - public string[] Add(string watcherId, IClock clock) + public (bool, string[]) Add(string watcherId, IClock clock) { var now = clock.GetCurrentInstant(); @@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Comments Users[watcherId] = now; - return Users.Keys.ToArray(); + return (true, Users.Keys.ToArray()); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Counter/CounterService.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Counter/CounterService.cs index 16c8ddb3c..d52068c47 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Counter/CounterService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Counter/CounterService.cs @@ -20,14 +20,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.Counter { public Dictionary Counters { get; set; } = new Dictionary(); - public void Increment(string name) + public bool Increment(string name) { Counters[name] = Counters.GetValueOrDefault(name) + 1; + + return true; } - public void Reset(string name, long value) + public bool Reset(string name, long value) { Counters[name] = value; + + return true; } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/OperationContextBase.cs b/backend/src/Squidex.Domain.Apps.Entities/OperationContextBase.cs index 573041171..b89d4e874 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/OperationContextBase.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/OperationContextBase.cs @@ -18,7 +18,8 @@ namespace Squidex.Domain.Apps.Entities { private readonly List errors = new List(); private readonly IServiceProvider serviceProvider; - private readonly Func snapshot; + private readonly Func snapshotProvider; + private readonly TSnapShot snapshotInitial; public RefToken Actor => Command.Actor; @@ -28,17 +29,22 @@ namespace Squidex.Domain.Apps.Entities public TCommand Command { get; init; } - public TSnapShot Snapshot => snapshot(); + public TSnapShot Snapshot => snapshotProvider(); + + public TSnapShot SnapshotInitial => snapshotInitial; public ClaimsPrincipal? User => Command.User; - protected OperationContextBase(IServiceProvider serviceProvider, Func snapshot) + public Dictionary Context { get; } = new Dictionary(); + + protected OperationContextBase(IServiceProvider serviceProvider, Func snapshotProvider) { Guard.NotNull(serviceProvider); - Guard.NotNull(snapshot); + Guard.NotNull(snapshotProvider); this.serviceProvider = serviceProvider; - this.snapshot = snapshot; + this.snapshotProvider = snapshotProvider; + this.snapshotInitial = snapshotProvider(); } public T Resolve() where T : notnull diff --git a/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs b/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs index f29b17219..c62c97eb4 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs @@ -5,9 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Threading.Tasks.Dataflow; using Squidex.Domain.Apps.Core.Tags; using Squidex.Infrastructure; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Tags { @@ -16,174 +18,176 @@ namespace Squidex.Domain.Apps.Entities.Tags private readonly IPersistenceFactory persistenceFactory; [CollectionName("Index_Tags")] - public sealed class State : TagsExport + public sealed class State : TagsExport, IOnRead { - public void Rebuild(TagsExport export) + public ValueTask OnReadAsync() { - Tags = export.Tags; + if (Tags == null) + { + Tags = new Dictionary(); + } + + if (Alias == null) + { + Alias = new Dictionary(); + } - Alias = export.Alias; + return default; } - public void Rename(string name, string newName) + public bool Rebuild(TagsExport export) { - Guard.NotNull(name); - Guard.NotNull(newName); + if (export.Tags != null) + { + Tags = export.Tags; + } - name = NormalizeName(name); + if (export.Alias != null) + { + Alias = export.Alias; + } - var (_, tag) = FindTag(name); + return true; + } + + public bool Clear() + { + var isChanged = false; - if (tag == null) + foreach (var (_, tag) in Tags) { - return; + isChanged = tag.Count > 0; + + tag.Count = 0; + } + + return isChanged; + } + + public bool Rename(string name, string newName) + { + name = NormalizeName(name); + + if (!TryGetTag(name, out var tag)) + { + return false; } newName = NormalizeName(newName); - tag.Name = newName; + if (string.Equals(name, newName, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + tag.Value.Name = newName; - if (Alias != null) + foreach (var alias in Alias.Where(x => x.Value == name).ToList()) { - foreach (var alias in Alias.Where(x => x.Value == name).ToList()) - { - Alias.Remove(alias.Key); + Alias.Remove(alias.Key); - if (alias.Key != newName) - { - Alias[alias.Key] = newName; - } + if (alias.Key != tag.Value.Name) + { + Alias[alias.Key] = tag.Value.Name; } } - Alias ??= new Dictionary(); - Alias[name] = newName; + return true; } - public Dictionary Normalize(HashSet? names, HashSet? ids) + public bool Update(Dictionary updates) { - var result = new Dictionary(); + var isChanged = false; - if (names != null) + foreach (var (id, update) in updates) { - foreach (var tag in names) + if (update != 0 && Tags.TryGetValue(id, out var tag)) { - var name = NormalizeName(tag); + var newCount = Math.Max(0, tag.Count + update); - if (!string.IsNullOrWhiteSpace(name)) + if (newCount != tag.Count) { - result.Add(name, GetId(name, ids)); - } - } - } + tag.Count = newCount; - if (ids != null) - { - foreach (var id in ids) - { - if (!result.ContainsValue(id)) - { - if (Tags != null && Tags.TryGetValue(id, out var tagInfo)) - { - tagInfo.Count--; - - if (tagInfo.Count <= 0) - { - Tags.Remove(id); - } - } + isChanged = true; } } } - return result; + return isChanged; } - public Dictionary GetTagIds(HashSet names) + public (bool, Dictionary) GetIds(HashSet names) { - Guard.NotNull(names); + var tagIds = new Dictionary(); - var result = new Dictionary(); + var isChanged = false; - foreach (var tag in names) + foreach (var name in names.Select(NormalizeName)) { - var name = NormalizeName(tag); + if (TryGetTag(name, out var tag)) + { + tagIds[name] = tag.Key; + } + else + { + var id = Guid.NewGuid().ToString(); - var (id, _) = FindTag(name); + Tags[id] = new Tag { Name = name }; + tagIds[name] = id; - if (!string.IsNullOrWhiteSpace(id)) - { - result.Add(name, id); + isChanged = true; } } - return result; + return (isChanged, tagIds); } - public Dictionary Denormalize(HashSet ids) + public Dictionary GetNames(HashSet ids) { - var result = new Dictionary(); + var tagNames = new Dictionary(); foreach (var id in ids) { - if (Tags?.TryGetValue(id, out var tagInfo) == true) + if (Tags.TryGetValue(id, out var tagInfo)) { - result[id] = tagInfo.Name; + tagNames[id] = tagInfo.Name; } } - return result; + return tagNames; } public TagsSet GetTags(long version) { - var tags = Tags?.Values.ToDictionary(x => x.Name, x => x.Count) ?? new Dictionary(); + var clone = Tags.Values.ToDictionary(x => x.Name, x => x.Count); - return new TagsSet(tags, version); + return new TagsSet(clone, version); } - public TagsExport GetExportableTags() + private static string NormalizeName(string name) { - var clone = Clone(); - - return clone; + return name.Trim().ToLowerInvariant(); } - private string GetId(string name, HashSet? ids) + private bool TryGetTag(string name, out KeyValuePair result) { - var (id, tag) = FindTag(name); + result = default; - if (tag != null) - { - if (ids == null || !ids.Contains(id)) - { - tag.Count++; - } - } - else + if (Alias.TryGetValue(name, out var newName)) { - id = DomainId.NewGuid().ToString(); - - Tags ??= new Dictionary(); - Tags.Add(id, new Tag { Name = name }); + name = newName; } - return id; - } - - private static string NormalizeName(string name) - { - return name.Trim().ToLowerInvariant(); - } + var found = Tags.FirstOrDefault(x => x.Value.Name == name); - private KeyValuePair FindTag(string name) - { - if (Alias?.TryGetValue(name, out var newName) == true) + if (found.Value != null) { - name = newName; + result = new KeyValuePair(found.Key, found.Value); + return true; } - return Tags?.FirstOrDefault(x => x.Value.Name == name) ?? default; + return false; } } @@ -220,25 +224,25 @@ namespace Squidex.Domain.Apps.Entities.Tags var state = await GetStateAsync(id, group, ct); - return await state.UpdateAsync(s => s.GetTagIds(names), ct: ct); + return await state.UpdateAsync(s => s.GetIds(names), ct: ct); } - public async Task> DenormalizeTagsAsync(DomainId id, string group, HashSet ids, + public async Task> GetTagNamesAsync(DomainId id, string group, HashSet ids, CancellationToken ct = default) { Guard.NotNull(ids); var state = await GetStateAsync(id, group, ct); - return await state.UpdateAsync(s => s.Denormalize(ids), ct: ct); + return state.Value.GetNames(ids); } - public async Task> NormalizeTagsAsync(DomainId id, string group, HashSet? names, HashSet? ids, + public async Task UpdateAsync(DomainId id, string group, Dictionary update, CancellationToken ct = default) { var state = await GetStateAsync(id, group, ct); - return await state.UpdateAsync(s => s.Normalize(names, ids), ct: ct); + await state.UpdateAsync(s => s.Update(update), ct: ct); } public async Task GetTagsAsync(DomainId id, string group, @@ -254,7 +258,7 @@ namespace Squidex.Domain.Apps.Entities.Tags { var state = await GetStateAsync(id, group, ct); - return state.Value.GetExportableTags(); + return state.Value; } public async Task ClearAsync(DomainId id, string group, @@ -274,5 +278,53 @@ namespace Squidex.Domain.Apps.Entities.Tags return state; } + + public async Task ClearAsync( + CancellationToken ct = default) + { + var writerBlock = new ActionBlock[]>(async batch => + { + try + { + var isChanged = !batch.All(x => !x.Value.Clear()); + + if (isChanged) + { + var jobs = batch.Select(x => new SnapshotWriteJob(x.Key, x.Value, x.Version)); + + await persistenceFactory.Snapshots.WriteManyAsync(jobs, ct); + } + } + catch (OperationCanceledException ex) + { + // Dataflow swallows operation cancelled exception. + throw new AggregateException(ex); + } + }, + new ExecutionDataflowBlockOptions + { + BoundedCapacity = 2, + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1, + }); + + // Create batches of 500 items to clear the tag count for better performance. + var batchBlock = new BatchBlock>(500, new GroupingDataflowBlockOptions + { + BoundedCapacity = 500 + }); + + batchBlock.BidirectionalLinkTo(writerBlock); + + await foreach (var state in persistenceFactory.Snapshots.ReadAllAsync(ct)) + { + // Uses back-propagation to not query additional items from the database, when queue is full. + await batchBlock.SendAsync(state, ct); + } + + batchBlock.Complete(); + + await writerBlock.Completion; + } } } diff --git a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs index 23f983a7e..aaeb5e786 100644 --- a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs +++ b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs @@ -13,5 +13,7 @@ namespace Squidex.Domain.Apps.Events.Assets public sealed class AssetDeleted : AssetEvent { public long DeletedSize { get; set; } + + public HashSet? OldTags { get; set; } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs index 912525612..ec58c7f48 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs @@ -52,9 +52,6 @@ namespace Squidex.Infrastructure.MongoDb BsonJsonConvention.Register(); BsonJsonValueSerializer.Register(); BsonStringSerializer.Register(); - BsonStringSerializer>.Register(); - BsonStringSerializer>.Register(); - BsonStringSerializer>.Register(); } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs index eef6bf902..0326fdb5c 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs @@ -119,21 +119,60 @@ namespace Squidex.Infrastructure.MongoDb } } - public static async Task UpsertVersionedAsync(this IMongoCollection collection, TKey key, long oldVersion, long newVersion, T document, + public static async Task UpsertVersionedAsync(this IMongoCollection collection, IClientSessionHandle session, SnapshotWriteJob job, CancellationToken ct = default) - where T : IVersionedEntity where TKey : notnull + where T : IVersionedEntity { + var (key, snapshot, newVersion, oldVersion) = job; try { - document.DocumentId = key; - document.Version = newVersion; + snapshot.DocumentId = key; + snapshot.Version = newVersion; Expression> filter = oldVersion > EtagVersion.Any ? x => x.DocumentId.Equals(key) && x.Version == oldVersion : x => x.DocumentId.Equals(key); - var result = await collection.ReplaceOneAsync(filter, document, UpsertReplace, ct); + var result = await collection.ReplaceOneAsync(session, filter, job.Value, UpsertReplace, ct); + + return result.IsAcknowledged && result.ModifiedCount == 1; + } + catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey) + { + var existingVersion = + await collection.Find(session, x => x.DocumentId.Equals(key)).Only(x => x.DocumentId, x => x.Version) + .FirstOrDefaultAsync(ct); + + if (existingVersion != null) + { + var field = Field.Of(x => nameof(x.Version)); + + throw new InconsistentStateException(existingVersion[field].AsInt64, oldVersion); + } + else + { + throw new InconsistentStateException(EtagVersion.Any, oldVersion); + } + } + } + + public static async Task UpsertVersionedAsync(this IMongoCollection collection, SnapshotWriteJob job, + CancellationToken ct = default) + where T : IVersionedEntity + { + var (key, snapshot, newVersion, oldVersion) = job; + try + { + snapshot.DocumentId = key; + snapshot.Version = newVersion; + + Expression> filter = + oldVersion > EtagVersion.Any ? + x => x.DocumentId.Equals(key) && x.Version == oldVersion : + x => x.DocumentId.Equals(key); + + var result = await collection.ReplaceOneAsync(filter, snapshot, UpsertReplace, ct); return result.IsAcknowledged && result.ModifiedCount == 1; } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs index 37c0f3da4..0ed74f32c 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs @@ -39,6 +39,11 @@ namespace Squidex.Infrastructure.States if (existing != null) { + if (existing.Document is IOnRead onRead) + { + await onRead.OnReadAsync(); + } + return new SnapshotResult(existing.DocumentId, existing.Document, existing.Version); } @@ -51,9 +56,9 @@ namespace Squidex.Infrastructure.States { using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/WriteAsync")) { - var document = CreateDocument(job.Key, job.Value, job.OldVersion); + var entityJob = job.As(CreateDocument(job.Key, job.Value, job.OldVersion)); - await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, document, ct); + await Collection.UpsertVersionedAsync(entityJob, ct); } } @@ -95,6 +100,11 @@ namespace Squidex.Infrastructure.States await foreach (var document in find.ToAsyncEnumerable(ct)) { + if (document.Document is IOnRead onRead) + { + await onRead.OnReadAsync(); + } + yield return new SnapshotResult(document.DocumentId, document.Document, document.Version, true); } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs b/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs index 495f89290..1e1b6e986 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using MongoDB.Bson; using MongoDB.Driver; using Squidex.Infrastructure.MongoDb; @@ -42,6 +43,14 @@ namespace Squidex.Infrastructure.UsageTracking return Collection.DeleteManyAsync(x => x.Key == key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return Collection.DeleteManyAsync(Filter.Regex(x => x.Key, new BsonRegularExpression(pattern)), ct); + } + public async Task TrackUsagesAsync(UsageUpdate update, CancellationToken ct = default) { diff --git a/backend/src/Squidex.Infrastructure/CollectionExtensions.cs b/backend/src/Squidex.Infrastructure/CollectionExtensions.cs index 2305f4288..6638ce005 100644 --- a/backend/src/Squidex.Infrastructure/CollectionExtensions.cs +++ b/backend/src/Squidex.Infrastructure/CollectionExtensions.cs @@ -327,11 +327,6 @@ namespace Squidex.Infrastructure return dictionary.GetOrAdd(key, _ => default!); } - public static TValue GetOrNew(this IReadOnlyDictionary dictionary, TKey key) where TKey : notnull where TValue : class, new() - { - return dictionary.GetOrCreate(key, _ => new TValue()); - } - public static TValue GetOrAddNew(this IDictionary dictionary, TKey key) where TKey : notnull where TValue : class, new() { return dictionary.GetOrAdd(key, _ => new TValue()); diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs index 0e613e3f9..c08e1d1ec 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs @@ -14,12 +14,14 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { private readonly IPersistenceFactory persistence; private readonly IMessageBus messaging; + private readonly HashSet activeNames; - public EventConsumerManager(IPersistenceFactory persistence, + public EventConsumerManager(IPersistenceFactory persistence, IEnumerable eventConsumers, IMessageBus messaging) { this.persistence = persistence; this.messaging = messaging; + this.activeNames = eventConsumers.Select(x => x.Name).ToHashSet(); } public async Task> GetConsumersAsync( @@ -27,7 +29,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { var snapshots = await persistence.Snapshots.ReadAllAsync(ct).ToListAsync(ct); - return snapshots.Select(x => x.Value.ToInfo(x.Key.ToString())).ToList(); + return snapshots.Where(x => activeNames.Contains(x.Key.ToString())).Select(x => x.Value.ToInfo(x.Key.ToString())).ToList(); } public async Task ResetAsync(string consumerName, @@ -67,7 +69,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume await state.LoadAsync(ct); - if (state.Version <= EtagVersion.Empty) + if (state.Version <= EtagVersion.Empty || !activeNames.Contains(consumerName)) { throw new DomainObjectNotFoundException(consumerName); } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerProcessor.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerProcessor.cs index 918176c6f..58d6667a5 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerProcessor.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerProcessor.cs @@ -144,6 +144,11 @@ namespace Squidex.Infrastructure.EventSourcing.Consume public virtual async Task ResetAsync() { + if (!eventConsumer.CanClear) + { + return; + } + await UpdateAsync(async () => { Unsubscribe(); @@ -160,7 +165,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { if (events.Count > 0) { - await eventConsumer!.On(events); + await eventConsumer.On(events); } } @@ -197,7 +202,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume } log.LogCritical(ex, "Failed to update consumer {consumer} at position {position} from {caller}.", - eventConsumer!.Name, position, caller); + eventConsumer.Name, position, caller); State = previousState.Stopped(ex); } @@ -214,17 +219,17 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { if (log.IsEnabled(LogLevel.Debug)) { - log.LogDebug("Event consumer {consumer} reset started", eventConsumer!.Name); + log.LogDebug("Event consumer {consumer} reset started", eventConsumer.Name); } var watch = ValueStopwatch.StartNew(); try { - await eventConsumer!.ClearAsync(); + await eventConsumer.ClearAsync(); } finally { - log.LogDebug("Event consumer {consumer} reset completed after {time}ms.", eventConsumer!.Name, watch.Stop()); + log.LogDebug("Event consumer {consumer} reset completed after {time}ms.", eventConsumer.Name, watch.Stop()); } } @@ -264,7 +269,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume protected virtual IEventSubscription CreateSubscription(IEventSubscriber subscriber) { - return eventStore.CreateSubscription(subscriber, eventConsumer!.EventsFilter, State.Position); + return eventStore.CreateSubscription(subscriber, eventConsumer.EventsFilter, State.Position); } } } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs b/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs index 7c4003427..0292f95fe 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs @@ -17,6 +17,8 @@ namespace Squidex.Infrastructure.EventSourcing string EventsFilter => ".*"; + bool CanClear => true; + bool Handles(StoredEvent @event) { return true; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IChangePlanResult.cs b/backend/src/Squidex.Infrastructure/States/IOnRead.cs similarity index 79% rename from backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IChangePlanResult.cs rename to backend/src/Squidex.Infrastructure/States/IOnRead.cs index 3f0db643c..3e3de6b5e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IChangePlanResult.cs +++ b/backend/src/Squidex.Infrastructure/States/IOnRead.cs @@ -5,9 +5,10 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -namespace Squidex.Domain.Apps.Entities.Apps.Plans +namespace Squidex.Infrastructure.States { - public interface IChangePlanResult + public interface IOnRead { + ValueTask OnReadAsync(); } } diff --git a/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs b/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs index 4bb0d24ad..085c1f756 100644 --- a/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs +++ b/backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs @@ -33,5 +33,11 @@ namespace Squidex.Infrastructure.States public record struct SnapshotResult(DomainId Key, T Value, long Version, bool IsValid = true); - public record struct SnapshotWriteJob(DomainId Key, T Value, long NewVersion, long OldVersion = EtagVersion.Any); + public record struct SnapshotWriteJob(DomainId Key, T Value, long NewVersion, long OldVersion = EtagVersion.Any) + { + public readonly SnapshotWriteJob As(TOther snapshot) + { + return new SnapshotWriteJob(Key, snapshot, NewVersion, OldVersion); + } + } } diff --git a/backend/src/Squidex.Infrastructure/States/NameReservationState.cs b/backend/src/Squidex.Infrastructure/States/NameReservationState.cs index 0de24dc1f..48625d697 100644 --- a/backend/src/Squidex.Infrastructure/States/NameReservationState.cs +++ b/backend/src/Squidex.Infrastructure/States/NameReservationState.cs @@ -14,11 +14,12 @@ namespace Squidex.Infrastructure.States { public List Reservations { get; set; } = new List(); - public string? Reserve(DomainId id, string name) + public (bool, string?) Reserve(DomainId id, string name) { string? token = null; var reservation = Reservations.Find(x => x.Name == name); + var reserved = false; if (reservation?.Id == id) { @@ -29,14 +30,15 @@ namespace Squidex.Infrastructure.States token = RandomHash.Simple(); Reservations.Add(new NameReservation(token, name, id)); + reserved = true; } - return token; + return (reserved, token); } - public void Remove(string? token) + public bool Remove(string? token) { - Reservations.RemoveAll(x => x.Token == token); + return Reservations.RemoveAll(x => x.Token == token) > 0; } } diff --git a/backend/src/Squidex.Infrastructure/States/SimpleState.cs b/backend/src/Squidex.Infrastructure/States/SimpleState.cs index 09c633b2d..ad3752e32 100644 --- a/backend/src/Squidex.Infrastructure/States/SimpleState.cs +++ b/backend/src/Squidex.Infrastructure/States/SimpleState.cs @@ -64,66 +64,35 @@ namespace Squidex.Infrastructure.States return persistence.WriteEventAsync(envelope, ct); } - public async Task UpdateIfAsync(Func updater, int retries = 20, + public Task UpdateAsync(Func updater, int retries = 20, CancellationToken ct = default) { - await EnsureLoadedAsync(ct); - - for (var i = 0; i < retries; i++) - { - try - { - if (!updater(Value)) - { - return; - } - - await WriteAsync(ct); - return; - } - catch (InconsistentStateException) when (i < retries) - { - await LoadAsync(ct); - } - } + return UpdateAsync(state => (updater(state), None.Value), retries, ct); } - public async Task UpdateAsync(Action updater, int retries = 20, + public async Task UpdateAsync(Func updater, int retries = 20, CancellationToken ct = default) { - await EnsureLoadedAsync(ct); - - for (var i = 0; i < retries; i++) + if (!isLoaded) { - try - { - updater(Value); - - await WriteAsync(ct); - return; - } - catch (InconsistentStateException) when (i < retries) - { - await LoadAsync(ct); - } + await LoadAsync(ct); } - } - - public async Task UpdateAsync(Func updater, int retries = 5, - CancellationToken ct = default) - { - await EnsureLoadedAsync(ct); for (var i = 0; i < retries; i++) { try { - var result = updater(Value); + var (isChanged, result) = updater(Value); + + if (!isChanged) + { + return result; + } await WriteAsync(ct); return result; } - catch (InconsistentStateException) when (i < retries) + catch (InconsistentStateException) when (i < retries - 1) { await LoadAsync(ct); } @@ -131,16 +100,5 @@ namespace Squidex.Infrastructure.States return default!; } - - private async Task EnsureLoadedAsync( - CancellationToken ct) - { - if (isLoaded) - { - return; - } - - await LoadAsync(ct); - } } } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs index c3560bd8d..e9b770286 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs @@ -14,7 +14,6 @@ namespace Squidex.Infrastructure.UsageTracking public sealed class BackgroundUsageTracker : DisposableObjectBase, IUsageTracker { private const int Intervall = 60 * 1000; - private const string FallbackCategory = "*"; private readonly IUsageRepository usageRepository; private readonly ILogger log; private readonly CompletionTimer timer; @@ -22,6 +21,8 @@ namespace Squidex.Infrastructure.UsageTracking public bool ForceWrite { get; set; } + public string FallbackCategory => "*"; + public BackgroundUsageTracker(IUsageRepository usageRepository, ILogger log) { @@ -96,6 +97,14 @@ namespace Squidex.Infrastructure.UsageTracking return usageRepository.DeleteAsync(key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return usageRepository.DeleteByKeyPatternAsync(pattern, ct); + } + public Task TrackAsync(DateTime date, string key, string? category, Counters counters, CancellationToken ct = default) { @@ -187,7 +196,7 @@ namespace Squidex.Infrastructure.UsageTracking return result; } - private static string GetCategory(string? category) + private string GetCategory(string? category) { return !string.IsNullOrWhiteSpace(category) ? category.Trim() : FallbackCategory; } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs index 6e5b006aa..05692da36 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs @@ -15,11 +15,10 @@ namespace Squidex.Infrastructure.UsageTracking private readonly IUsageTracker inner; private readonly IMemoryCache cache; + public string FallbackCategory => inner.FallbackCategory; + public CachingUsageTracker(IUsageTracker inner, IMemoryCache cache) { - Guard.NotNull(inner); - Guard.NotNull(cache); - this.inner = inner; this.cache = cache; } @@ -32,6 +31,14 @@ namespace Squidex.Infrastructure.UsageTracking return inner.DeleteAsync(key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return inner.DeleteByKeyPatternAsync(pattern, ct); + } + public Task>> QueryAsync(string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default) { diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs index 232754e28..cb1fdd54a 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs @@ -20,5 +20,8 @@ namespace Squidex.Infrastructure.UsageTracking Task DeleteAsync(string key, CancellationToken ct = default); + + Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default); } } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs index 72b4a5f84..9819d9452 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs @@ -9,6 +9,8 @@ namespace Squidex.Infrastructure.UsageTracking { public interface IUsageTracker { + string FallbackCategory { get; } + Task TrackAsync(DateTime date, string key, string? category, Counters counters, CancellationToken ct = default); @@ -23,5 +25,8 @@ namespace Squidex.Infrastructure.UsageTracking Task DeleteAsync(string key, CancellationToken ct = default); + + Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default); } } diff --git a/backend/src/Squidex/Areas/Api/Controllers/Plans/AppPlansController.cs b/backend/src/Squidex/Areas/Api/Controllers/Plans/AppPlansController.cs index c26f3271b..663c0380f 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Plans/AppPlansController.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Plans/AppPlansController.cs @@ -77,13 +77,13 @@ namespace Squidex.Areas.Api.Controllers.Plans [ApiCosts(0)] public async Task PutPlan(string app, [FromBody] ChangePlanDto request) { - var context = await CommandBus.PublishAsync(request.ToCommand(HttpContext), HttpContext.RequestAborted); + var context = await CommandBus.PublishAsync(request.ToCommand(), HttpContext.RequestAborted); string? redirectUri = null; - if (context.PlainResult is RedirectToCheckoutResult result) + if (context.PlainResult is PlanChangedResult result) { - redirectUri = result.Url.ToString(); + redirectUri = result.RedirectUri?.ToString(); } return Ok(new PlanChangedDto { RedirectUri = redirectUri }); diff --git a/backend/src/Squidex/Areas/Api/Controllers/Plans/Models/ChangePlanDto.cs b/backend/src/Squidex/Areas/Api/Controllers/Plans/Models/ChangePlanDto.cs index 66230b9a0..d0138b65e 100644 --- a/backend/src/Squidex/Areas/Api/Controllers/Plans/Models/ChangePlanDto.cs +++ b/backend/src/Squidex/Areas/Api/Controllers/Plans/Models/ChangePlanDto.cs @@ -19,12 +19,10 @@ namespace Squidex.Areas.Api.Controllers.Plans.Models [LocalizedRequired] public string PlanId { get; set; } - public ChangePlan ToCommand(HttpContext httpContext) + public ChangePlan ToCommand() { var result = SimpleMapper.Map(this, new ChangePlan()); - result.Referer = httpContext.Request.Headers["Referer"]; - return result; } } diff --git a/backend/src/Squidex/Config/Domain/LoggingServices.cs b/backend/src/Squidex/Config/Domain/LoggingServices.cs index 72ddd7e6a..16e0e261c 100644 --- a/backend/src/Squidex/Config/Domain/LoggingServices.cs +++ b/backend/src/Squidex/Config/Domain/LoggingServices.cs @@ -18,11 +18,11 @@ namespace Squidex.Config.Domain public static void ConfigureForSquidex(this ILoggingBuilder builder, IConfiguration config) { builder.ClearProviders(); + + // Also adds semantic logging. builder.ConfigureSemanticLog(config); builder.AddConfiguration(config.GetSection("logging")); - - builder.AddSemanticLog(); builder.AddFilters(); builder.Services.AddServices(config); diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs deleted file mode 100644 index bcd325b42..000000000 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs +++ /dev/null @@ -1,136 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using FakeItEasy; -using Squidex.Domain.Apps.Core.Contents; -using Squidex.Domain.Apps.Core.Schemas; -using Squidex.Domain.Apps.Core.Tags; -using Squidex.Domain.Apps.Core.TestHelpers; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Json.Objects; -using Xunit; - -namespace Squidex.Domain.Apps.Core.Operations.Tags -{ - public class TagNormalizerTests - { - private readonly ITagService tagService = A.Fake(); - private readonly DomainId appId = DomainId.NewGuid(); - private readonly DomainId schemaId = DomainId.NewGuid(); - private readonly Schema schema; - - public TagNormalizerTests() - { - schema = - new Schema("my-schema") - .AddTags(1, "tags1", Partitioning.Invariant) - .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema }) - .AddString(3, "string", Partitioning.Invariant) - .AddArray(4, "array", Partitioning.Invariant, f => f - .AddTags(401, "nestedTags1") - .AddTags(402, "nestedTags2", new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema }) - .AddString(403, "string")); - } - - [Fact] - public async Task Should_normalize_tags_with_old_data() - { - var newData = GenerateData("n_raw"); - var oldData = GenerateData("o_raw"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("n_raw2_1", "n_raw2_2", "n_raw4"), - A>.That.Is("o_raw2_1", "o_raw2_2", "o_raw4"), - default)) - .Returns(new Dictionary - { - ["n_raw2_2"] = "id2_2", - ["n_raw2_1"] = "id2_1", - ["n_raw4"] = "id4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, oldData); - - Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData)); - } - - [Fact] - public async Task Should_normalize_tags_without_old_data() - { - var newData = GenerateData("name"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("name2_1", "name2_2", "name4"), - A>.That.IsEmpty(), - default)) - .Returns(new Dictionary - { - ["name2_2"] = "id2_2", - ["name2_1"] = "id2_1", - ["name4"] = "id4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, null); - - Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData)); - } - - [Fact] - public async Task Should_denormalize_tags() - { - var newData = GenerateData("id"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("id2_1", "id2_2", "id4"), - A>.That.IsEmpty(), - default)) - .Returns(new Dictionary - { - ["id2_2"] = "name2_2", - ["id2_1"] = "name2_1", - ["id4"] = "name4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, null); - - Assert.Equal(JsonValue.Array("name2_1", "name2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("name4"), GetNestedTags(newData)); - } - - private static JsonValue GetNestedTags(ContentData newData) - { - var arrayValue = newData["array"]!["iv"].AsArray; - var arrayItem = arrayValue[0].AsObject; - - return arrayItem["nestedTags2"]; - } - - private static ContentData GenerateData(string prefix) - { - return new ContentData() - .AddField("tags1", - new ContentFieldData() - .AddInvariant(JsonValue.Array($"{prefix}1"))) - .AddField("tags2", - new ContentFieldData() - .AddInvariant(JsonValue.Array($"{prefix}2_1", $"{prefix}2_2"))) - .AddField("string", - new ContentFieldData() - .AddInvariant($"{prefix}stringValue")) - .AddField("array", - new ContentFieldData() - .AddInvariant( - JsonValue.Array( - new JsonObject() - .Add("nestedTags1", JsonValue.Array($"{prefix}3")) - .Add("nestedTags2", JsonValue.Array($"{prefix}4")) - .Add("string", $"{prefix}nestedStringValue")))); - } - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs index c6c72c14b..dccd42af9 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs @@ -59,6 +59,9 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject A.CallTo(() => appPlansProvider.GetPlan(planIdPaid)) .Returns(new ConfigAppLimitsPlan { Id = planIdPaid, MaxContributors = 30 }); + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, A._, default)) + .Returns(Task.FromResult(null)); + // Create a non-empty setting, otherwise the event is not raised as it does not change the domain object. initialSettings = new InitialSettings { @@ -217,14 +220,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject { var command = new ChangePlan { PlanId = planIdPaid }; - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default)) - .Returns(new PlanChangedResult()); + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default)) + .Returns(Task.FromResult(null)); await ExecuteCreateAsync(); var result = await PublishIdempotentAsync(command); - Assert.True(result is PlanChangedResult); + result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid)); Assert.Equal(planIdPaid, sut.Snapshot.Plan!.PlanId); @@ -232,6 +235,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject .ShouldHaveSameEvents( CreateEvent(new AppPlanChanged { PlanId = planIdPaid }) ); + + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default)) + .MustHaveHappened(); + + A.CallTo(() => appPlansBillingManager.SubscribeAsync(Actor.Identifier, AppNamedId, planIdPaid, default)) + .MustHaveHappened(); } [Fact] @@ -243,7 +252,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject var result = await PublishIdempotentAsync(command); - result.ShouldBeEquivalent(None.Value); + result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid)); Assert.Equal(planIdPaid, sut.Snapshot.Plan!.PlanId); @@ -252,7 +261,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject CreateEvent(new AppPlanChanged { PlanId = planIdPaid }) ); - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(A._, A>._, A._, A._, A._)) + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(A._, A>._, A._, A._)) + .MustNotHaveHappened(); + + A.CallTo(() => appPlansBillingManager.SubscribeAsync(A._, A>._, A._, A._)) .MustNotHaveHappened(); } @@ -261,15 +273,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject { var command = new ChangePlan { PlanId = planIdFree, FromCallback = true }; - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default)) - .Returns(new PlanChangedResult()); - await ExecuteCreateAsync(); await ExecuteChangePlanAsync(); var result = await PublishIdempotentAsync(command); - result.ShouldBeEquivalent(None.Value); + result.ShouldBeEquivalent(new PlanChangedResult(planIdFree, true)); Assert.Null(sut.Snapshot.Plan); @@ -278,7 +287,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject CreateEvent(new AppPlanReset()) ); - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(A._, A>._, planIdFree, A._, A._)) + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(A._, A>._, A._, A._)) + .MustHaveHappenedOnceExactly(); + + A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(A._, A>._, A._)) .MustNotHaveHappened(); } @@ -287,18 +299,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject { var command = new ChangePlan { PlanId = planIdFree }; - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default)) - .Returns(new PlanChangedResult()); - - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdFree, command.Referer, default)) - .Returns(new PlanChangedResult()); - await ExecuteCreateAsync(); await ExecuteChangePlanAsync(); var result = await PublishIdempotentAsync(command); - Assert.True(result is PlanChangedResult); + result.ShouldBeEquivalent(new PlanChangedResult(planIdFree, true)); Assert.Null(sut.Snapshot.Plan); @@ -306,6 +312,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject .ShouldHaveSameEvents( CreateEvent(new AppPlanReset()) ); + + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default)) + .MustHaveHappenedOnceExactly(); + + A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(A._, A>._, A._)) + .MustHaveHappened(); } [Fact] @@ -313,14 +325,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject { var command = new ChangePlan { PlanId = planIdPaid }; - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default)) - .Returns(new RedirectToCheckoutResult(new Uri("http://squidex.io"))); + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default)) + .Returns(new Uri("http://squidex.io")); await ExecuteCreateAsync(); var result = await PublishIdempotentAsync(command); - result.ShouldBeEquivalent(new RedirectToCheckoutResult(new Uri("http://squidex.io"))); + result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid, false, new Uri("http://squidex.io"))); Assert.Null(sut.Snapshot.Plan); } @@ -334,9 +346,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject var result = await PublishIdempotentAsync(command); - result.ShouldBeEquivalent(None.Value); + result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid)); + + Assert.Equal(planIdPaid, sut.Snapshot.Plan?.PlanId); + + A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, A._)) + .MustNotHaveHappened(); - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, A._, default)) + A.CallTo(() => appPlansBillingManager.SubscribeAsync(Actor.Identifier, AppNamedId, planIdPaid, A._)) .MustNotHaveHappened(); } @@ -651,7 +668,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject CreateEvent(new AppDeleted()) ); - A.CallTo(() => appPlansBillingManager.ChangePlanAsync(command.Actor.Identifier, AppNamedId, null, A._, default)) + A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(command.Actor.Identifier, AppNamedId, default)) .MustHaveHappened(); } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/NoopAppPlanBillingManagerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/NoopAppPlanBillingManagerTests.cs index b9439762f..6601b4218 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/NoopAppPlanBillingManagerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/NoopAppPlanBillingManagerTests.cs @@ -20,15 +20,31 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans } [Fact] - public async Task Should_do_nothing_if_changing_plan() + public async Task Should_do_nothing_if_subscribing() { - await sut.ChangePlanAsync(null!, null!, null, null); + await sut.SubscribeAsync(null!, null!, null!); + } + + [Fact] + public async Task Should_do_nothing_if_unsubscribing() + { + await sut.UnsubscribeAsync(null!, null!); } [Fact] public async Task Should_not_return_portal_link() { - Assert.Equal(string.Empty, await sut.GetPortalLinkAsync(null!)); + var result = await sut.GetPortalLinkAsync(null!); + + Assert.Empty(result); + } + + [Fact] + public async Task Should_do_nothing_if_checking_for_redirect() + { + var result = await sut.MustRedirectToPortalAsync(null!, null!, null); + + Assert.Null(result); } } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs index be9d7dfa4..acc37931f 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs @@ -8,9 +8,11 @@ using FakeItEasy; using FluentAssertions; using NodaTime; +using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; using Xunit; @@ -18,13 +20,23 @@ namespace Squidex.Domain.Apps.Entities.Assets { public class AssetUsageTrackerTests { + private readonly IAssetLoader assetLoader = A.Fake(); + private readonly ISnapshotStore store = A.Fake>(); + private readonly ITagService tagService = A.Fake(); private readonly IUsageTracker usageTracker = A.Fake(); private readonly NamedId appId = NamedId.Of(DomainId.NewGuid(), "my-app"); + private readonly DomainId assetId = DomainId.NewGuid(); + private readonly DomainId assetKey; private readonly AssetUsageTracker sut; public AssetUsageTrackerTests() { - sut = new AssetUsageTracker(usageTracker); + assetKey = DomainId.Combine(appId, assetId); + + A.CallTo(() => usageTracker.FallbackCategory) + .Returns("*"); + + sut = new AssetUsageTracker(usageTracker, assetLoader, tagService, store); } [Fact] @@ -130,7 +142,7 @@ namespace Squidex.Domain.Apps.Entities.Assets @event.AppId = appId; var envelope = - Envelope.Create(@event) + Envelope.Create(@event) .SetTimestamp(Instant.FromDateTimeUtc(date)); Counters? countersSummary = null; @@ -142,7 +154,7 @@ namespace Squidex.Domain.Apps.Entities.Assets A.CallTo(() => usageTracker.TrackAsync(date, $"{appId.Id}_Assets", null, A._, default)) .Invokes(x => countersDate = x.GetArgument(3)); - await sut.On(envelope); + await sut.On(new[] { envelope }); var expected = new Counters { @@ -153,5 +165,294 @@ namespace Squidex.Domain.Apps.Entities.Assets countersSummary.Should().BeEquivalentTo(expected); countersDate.Should().BeEquivalentTo(expected); } + + [Fact] + public async Task Should_write_tags_when_asset_created() + { + var @event = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var envelope = + Envelope.Create(@event) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 1, + ["tag2"] = 1 + }); + } + + [Fact] + public async Task Should_group_tags_by_app() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1, envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 1 + }); + + A.CallTo(() => store.WriteManyAsync(A>>._, default)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_annotate() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetAnnotated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1, envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 0, + ["tag2"] = 1, + ["tag3"] = 1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_annotate_from_other_batch() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetAnnotated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1 }); + await sut.On(new[] { envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = 0, + ["tag3"] = 1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_delete() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { Envelope.Create(@event1), Envelope.Create(@event2) }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 0, + ["tag2"] = 0 + }); + } + + [Fact] + public async Task Should_merge_tags_with_stored_state_if_previous_event_not_in_cached() + { + var state = new AssetUsageTracker.State + { + Tags = new HashSet + { + "tag1", + "tag2" + } + }; + + A.CallTo(() => store.ReadAsync(assetKey, default)) + .Returns(new SnapshotResult(assetKey, state, 0)); + + var @event = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope = + Envelope.Create(@event) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = -1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_asset_if_previous_tags_not_in_store() + { + IAssetEntity asset = new AssetEntity + { + Tags = new HashSet + { + "tag1", + "tag2" + } + }; + + A.CallTo(() => assetLoader.GetAsync(appId.Id, assetId, 41, default)) + .Returns(asset); + + var @event = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope = + Envelope.Create(@event) + .SetEventStreamNumber(42) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = -1 + }); + } } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs index 1b9147a06..a21436783 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs @@ -10,6 +10,7 @@ using Squidex.Assets; using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Events.Apps; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.Commands; @@ -97,13 +98,19 @@ namespace Squidex.Domain.Apps.Entities.Assets var context = CreateRestoreContext(); + var envelope = + new Envelope(new AppCreated + { + AppId = appId + }); + A.CallTo(() => context.Reader.HasFileAsync(A._, ct)) .Returns(true); A.CallTo(() => context.Reader.ReadJsonAsync>(A._, ct)) .Returns(tags); - await sut.RestoreAsync(context, ct); + await sut.RestoreEventAsync(envelope, context, ct); A.CallTo(() => tagService.RebuildTagsAsync(appId.Id, TagGroups.Assets, A.That.Matches(x => x.Tags == tags), ct)) .MustHaveHappened(); @@ -116,13 +123,19 @@ namespace Squidex.Domain.Apps.Entities.Assets var context = CreateRestoreContext(); + var envelope = + new Envelope(new AppCreated + { + AppId = appId + }); + A.CallTo(() => context.Reader.HasFileAsync(A._, ct)) .Returns(false).Once().Then.Returns(true); A.CallTo(() => context.Reader.ReadJsonAsync>(A._, ct)) .Returns(alias); - await sut.RestoreAsync(context, ct); + await sut.RestoreEventAsync(envelope, context, ct); A.CallTo(() => tagService.RebuildTagsAsync(appId.Id, TagGroups.Assets, A.That.Matches(x => x.Alias == alias), ct)) .MustHaveHappened(); @@ -135,13 +148,19 @@ namespace Squidex.Domain.Apps.Entities.Assets var context = CreateRestoreContext(); + var envelope = + new Envelope(new AppCreated + { + AppId = appId + }); + A.CallTo(() => context.Reader.HasFileAsync(A._, ct)) .Returns(false); A.CallTo(() => context.Reader.ReadJsonAsync>(A._, ct)) .Returns(alias); - await sut.RestoreAsync(context, ct); + await sut.RestoreEventAsync(envelope, context, ct); A.CallTo(() => context.Reader.ReadJsonAsync>(A._, A._)) .MustNotHaveHappened(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs index 29e3e3c45..b4033ad89 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs @@ -63,7 +63,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject A.CallTo(() => assetQuery.FindAssetFolderAsync(AppId, parentId, A._)) .Returns(new List { A.Fake() }); - A.CallTo(() => tagService.NormalizeTagsAsync(AppId, TagGroups.Assets, A>._, A>._, default)) + A.CallTo(() => tagService.GetTagIdsAsync(AppId, TagGroups.Assets, A>._, default)) .ReturnsLazily(x => Task.FromResult(x.GetArgument>(2)?.ToDictionary(x => x) ?? new Dictionary())); var log = A.Fake>(); @@ -367,7 +367,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject } [Fact] - public async Task Delete_should_create_events_with_total_file_size_and_update_deleted_flag() + public async Task Delete_should_create_events_with_total_file_size_and_tags_and_update_deleted_flag() { var command = new DeleteAsset(); @@ -382,7 +382,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject LastEvents .ShouldHaveSameEvents( - CreateAssetEvent(new AssetDeleted { DeletedSize = 2048 }) + CreateAssetEvent(new AssetDeleted { DeletedSize = 2048, OldTags = new HashSet() }) ); A.CallTo(() => scriptEngine.ExecuteAsync(A._, "", ScriptOptions(), default)) diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs index 1948119ab..54824a2b7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs @@ -80,7 +80,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries AppId = appId }; - A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2"), ct)) + A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2"), ct)) .Returns(new Dictionary { ["id1"] = "name1", @@ -158,7 +158,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries AppId = appId }; - A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2", "id3"), ct)) + A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2", "id3"), ct)) .Returns(new Dictionary { ["id1"] = "name1", diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs index 6c81b6fb6..3e1979390 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs @@ -39,8 +39,9 @@ namespace Squidex.Domain.Apps.Entities.Tags [Fact] public async Task Should_delete_and_reset_state_if_cleaning() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct); + await sut.ClearAsync(appId, group, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); @@ -52,69 +53,76 @@ namespace Squidex.Domain.Apps.Entities.Tags } [Fact] - public async Task Should_rename_tag() + public async Task Should_unset_count_on_full_clear() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct); + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); - // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new"), null, ct); + // Clear is called by the event consumer to fill the counts again, therefore we do not delete other things. + await sut.ClearAsync(ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["tag1_new"] = 4 + ["tag1"] = 0, + ["tag2"] = 0 }, allTags); + + A.CallTo(() => state.Persistence.DeleteAsync(ct)) + .MustNotHaveHappened(); } [Fact] - public async Task Should_rename_tag_twice() + public async Task Should_rename_tag() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct); + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - // Rename again. - await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1_new2", ct); + await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct); // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new2"), null, ct); + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1_new"), ct); var allTags = await sut.GetTagsAsync(appId, group, ct); - Assert.Equal(new Dictionary - { - ["tag1_new2"] = 5 - }, allTags); + Assert.Equal(ids_0.Values, ids_1.Values); + } + + [Fact] + public async Task Should_rename_tag_twice() + { + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); + + await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct); + await sut.RenameTagAsync(appId, group, "tag2", "tag3", ct); + + // Forward the old name to the new name. + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2"), ct); + var ids_2 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag3"), ct); + + // Assert.Equal(ids_0.Values, ids_1.Values); + Assert.Equal(ids_0.Values, ids_2.Values); } [Fact] public async Task Should_rename_tag_back() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct); + await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct); // Rename back. - await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1", ct); + await sut.RenameTagAsync(appId, group, "tag2", "tag1", ct); // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - - var allTags = await sut.GetTagsAsync(appId, group, ct); + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - Assert.Equal(new Dictionary - { - ["tag1"] = 3 - }, allTags); + Assert.Equal(ids_0.Values, ids_1.Values); } [Fact] @@ -124,10 +132,11 @@ namespace Squidex.Domain.Apps.Entities.Tags { Tags = new Dictionary { - ["id1"] = new Tag { Name = "name1", Count = 1 }, - ["id2"] = new Tag { Name = "name2", Count = 2 }, - ["id3"] = new Tag { Name = "name3", Count = 6 } - } + ["id1"] = new Tag { Name = "tag1", Count = 1 }, + ["id2"] = new Tag { Name = "tag2", Count = 2 }, + ["id3"] = new Tag { Name = "tag3", Count = 6 } + }, + Alias = null! }; await sut.RebuildTagsAsync(appId, group, tags, ct); @@ -136,76 +145,146 @@ namespace Squidex.Domain.Apps.Entities.Tags Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 2, - ["name3"] = 6 + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 6 }, allTags); var export = await sut.GetExportableTagsAsync(appId, group, ct); - export.Should().BeEquivalentTo(tags); + Assert.Equal(tags.Tags, export.Tags); + Assert.Empty(export.Alias); + } + + [Fact] + public async Task Should_rebuild_with_broken_export() + { + var tags = new TagsExport + { + Alias = new Dictionary + { + ["id1"] = "id2" + }, + Tags = null! + }; + + await sut.RebuildTagsAsync(appId, group, tags, ct); + + var export = await sut.GetExportableTagsAsync(appId, group, ct); + + Assert.Equal(tags.Alias, export.Alias); + Assert.Empty(export.Tags); } [Fact] - public async Task Should_add_tags() + public async Task Should_add_tag_but_not_count_tags() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 2, - ["name3"] = 1 + ["tag1"] = 0, + ["tag2"] = 0, + ["tag3"] = 0 }, allTags); } [Fact] - public async Task Should_not_add_tags_if_already_added() + public async Task Should_add_and_increment_tags() { - var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2", "name3"), new HashSet(result1.Values), ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag2"]] = 1, + [ids["tag3"]] = 1 + }, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 1, - ["name3"] = 1 + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 1 }, allTags); } [Fact] - public async Task Should_remove_tags() + public async Task Should_add_and_decrement_tags() { - var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); - // Tags from the first normalization are decreased and removed if count reaches zero. - await sut.NormalizeTagsAsync(appId, group, null, new HashSet(result1.Values), ct); + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag2"]] = -2, + [ids["tag3"]] = -2 + }, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name2"] = 1, - ["name3"] = 1 + ["tag1"] = 1, + ["tag2"] = 0, + ["tag3"] = 0 }, allTags); } + [Fact] + public async Task Should_not_update_non_existing_tags() + { + // We have no names for these IDs so we cannot update it. + await sut.UpdateAsync(appId, group, new Dictionary + { + ["id1"] = 1, + ["id2"] = 1 + }, ct); + + var allTags = await sut.GetTagsAsync(appId, group, ct); + + Assert.Empty(allTags); + } + [Fact] public async Task Should_resolve_tag_names() { // Get IDs from names. - var tagIds = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); + var tagIds = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); // Get names from IDs (reverse operation). - var tagNames = await sut.GetTagIdsAsync(appId, group, HashSet.Of("name1", "name2", "invalid1"), ct); + var tagNames = await sut.GetTagNamesAsync(appId, group, tagIds.Values.ToHashSet(), ct); + + Assert.Equal(tagIds.Keys.ToArray(), tagNames.Values.ToArray()); + } + + [Fact] + public async Task Should_get_exportable_tags() + { + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); - Assert.Equal(tagIds, tagNames); + var allTags = await sut.GetExportableTagsAsync(appId, group, ct); + + allTags.Tags.Should().BeEquivalentTo(new Dictionary + { + [ids["tag1"]] = new Tag { Name = "tag1", Count = 0 }, + [ids["tag2"]] = new Tag { Name = "tag2", Count = 0 }, + }); } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs b/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs index de31dd4e0..eef18c15b 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs @@ -112,7 +112,7 @@ namespace Squidex.Infrastructure { valueDictionary[12] = 34; - Assert.Equal(34, valueDictionary.GetOrAdd(12, x => 34)); + Assert.Equal(34, valueDictionary.GetOrAdd(12, x => 44)); } [Fact] @@ -129,24 +129,6 @@ namespace Squidex.Infrastructure Assert.Equal(24, valueDictionary[12]); } - [Fact] - public void GetOrNew_should_return_value_if_key_exists() - { - var list = new List(); - listDictionary[12] = list; - - Assert.Equal(list, listDictionary.GetOrNew(12)); - } - - [Fact] - public void GetOrNew_should_return_default_but_not_add_it_if_key_not_exists() - { - var list = new List(); - - Assert.Equal(list, listDictionary.GetOrNew(12)); - Assert.False(listDictionary.ContainsKey(12)); - } - [Fact] public void GetOrAddNew_should_return_value_if_key_exists() { diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs index 7cd1f421a..9544d47e9 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs @@ -18,16 +18,26 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { private readonly IPersistenceFactory persistenceFactory = A.Fake>(); private readonly IMessageBus messaging = A.Fake(); - private readonly string consumerName = Guid.NewGuid().ToString(); + private readonly string consumerName1 = Guid.NewGuid().ToString(); + private readonly string consumerName2 = Guid.NewGuid().ToString(); private readonly EventConsumerManager sut; public EventConsumerManagerTests() { - sut = new EventConsumerManager(persistenceFactory, messaging); + var consumer1 = A.Fake(); + var consumer2 = A.Fake(); + + A.CallTo(() => consumer1.Name) + .Returns(consumerName1); + + A.CallTo(() => consumer2.Name) + .Returns(consumerName2); + + sut = new EventConsumerManager(persistenceFactory, new[] { consumer1, consumer2 }, messaging); } [Fact] - public async Task Should_get_states_from_store() + public async Task Should_get_states_from_store_without_old_consumer() { var snapshotStore = A.Fake>(); @@ -37,12 +47,17 @@ namespace Squidex.Infrastructure.EventSourcing.Consume A.CallTo(() => snapshotStore.ReadAllAsync(default)) .Returns(new List> { - new SnapshotResult(DomainId.Create("consumer1"), + new SnapshotResult(DomainId.Create(consumerName1), new EventConsumerState { Position = "1" }, 1), - new SnapshotResult(DomainId.Create("consumer2"), + new SnapshotResult(DomainId.Create(consumerName2), + new EventConsumerState + { + Position = "2" + }, 2), + new SnapshotResult(DomainId.Create("oldConsumer"), new EventConsumerState { Position = "2" @@ -54,25 +69,32 @@ namespace Squidex.Infrastructure.EventSourcing.Consume result.Should().BeEquivalentTo( new List { - new EventConsumerInfo { Name = "consumer1", Position = "1" }, - new EventConsumerInfo { Name = "consumer2", Position = "2" } + new EventConsumerInfo { Name = consumerName1, Position = "1" }, + new EventConsumerInfo { Name = consumerName2, Position = "2" } }); } + [Fact] + public async Task Should_throw_exception_when_calling_old_consumer() + { + await Assert.ThrowsAsync(() => sut.StartAsync("oldConsumer", default)); + } + [Fact] public async Task Should_publish_event_on_start() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(consumerName1, persistenceFactory) { Snapshot = new EventConsumerState { Position = "42" - } + }, + Version = 0 }; - var response = await sut.StartAsync(consumerName, default); + var response = await sut.StartAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); @@ -81,17 +103,18 @@ namespace Squidex.Infrastructure.EventSourcing.Consume [Fact] public async Task Should_publish_event_on_stop() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(consumerName1, persistenceFactory) { Snapshot = new EventConsumerState { Position = "42" - } + }, + Version = 0 }; - var response = await sut.StopAsync(consumerName, default); + var response = await sut.StopAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); @@ -100,17 +123,18 @@ namespace Squidex.Infrastructure.EventSourcing.Consume [Fact] public async Task Should_publish_event_on_reset() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(consumerName1, persistenceFactory) { Snapshot = new EventConsumerState { Position = "42" - } + }, + Version = 0 }; - var response = await sut.ResetAsync(consumerName, default); + var response = await sut.ResetAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorTests.cs index 33b2e54ab..9492b4759 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorTests.cs @@ -71,6 +71,9 @@ namespace Squidex.Infrastructure.EventSourcing.Consume A.CallTo(() => eventConsumer.Name) .Returns(consumerName); + A.CallTo(() => eventConsumer.CanClear) + .Returns(true); + A.CallTo(() => eventConsumer.Handles(A._)) .Returns(true); @@ -205,6 +208,26 @@ namespace Squidex.Infrastructure.EventSourcing.Consume .MustHaveHappenedOnceExactly(); } + [Fact] + public async Task Should_not_reset_consumer_if_not_allowed() + { + A.CallTo(() => eventConsumer.CanClear) + .Returns(false); + + await sut.InitializeAsync(default); + await sut.ActivateAsync(); + + await sut.StopAsync(); + await sut.ResetAsync(); + + await sut.CompleteAsync(); + + AssertGrainState(isStopped: true, position: initialPosition); + + A.CallTo(() => eventConsumer.ClearAsync()) + .MustNotHaveHappened(); + } + [Fact] public async Task Should_invoke_and_update_position_if_event_received() { diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs index 75468c10d..dafdedfd3 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs @@ -50,8 +50,6 @@ namespace Squidex.Infrastructure.EventSourcing get => sut.Value; } - protected abstract int SubscriptionDelayInMs { get; } - protected EventStoreTests() { #pragma warning disable MA0056 // Do not call overridable members in constructor @@ -318,15 +316,12 @@ namespace Squidex.Infrastructure.EventSourcing var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray(); - var takeStep = count / 10; - - for (var take = 0; take < count; take += takeStep) + for (var take = 0; take < count; take += count / 10) { - var expected = allExpected.TakeLast(take).ToArray(); + var eventsExpected = allExpected.TakeLast(take).ToArray(); + var eventsQueried = await Sut.QueryReverseAsync(streamName, take); - var readEvents = await Sut.QueryReverseAsync(streamName, take); - - ShouldBeEquivalentTo(readEvents, expected); + ShouldBeEquivalentTo(eventsQueried, eventsExpected); } } @@ -353,15 +348,12 @@ namespace Squidex.Infrastructure.EventSourcing var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray(); - var takeStep = count / 10; - - for (var take = 0; take < count; take += takeStep) + for (var take = 0; take < count; take += count / 10) { - var expected = allExpected.Reverse().Take(take).ToArray(); + var eventsExpected = allExpected.Reverse().Take(take).ToArray(); + var eventsQueried = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync(); - var readEvents = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync(); - - ShouldBeEquivalentTo(readEvents, expected); + ShouldBeEquivalentTo(eventsQueried, eventsExpected); } } @@ -438,7 +430,12 @@ namespace Squidex.Infrastructure.EventSourcing private static EventData CreateEventData(int i) { - return new EventData($"Type{i}", new EnvelopeHeaders(), i.ToString(CultureInfo.InvariantCulture)); + var headers = new EnvelopeHeaders + { + [CommonHeaders.EventId] = Guid.NewGuid().ToString() + }; + + return new EventData($"Type{i}", headers, i.ToString(CultureInfo.InvariantCulture)); } private async Task?> QueryAllAsync(string? streamFilter = null, string? position = null) diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreTests.cs index 00090dfef..8c2168122 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreTests.cs @@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing { public GetEventStoreFixture _ { get; } - protected override int SubscriptionDelayInMs { get; } = 1000; - public GetEventStoreTests(GetEventStoreFixture fixture) { _ = fixture; diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs index 8dedc674b..b7abd1d72 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs @@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing { public MongoEventStoreFixture _ { get; } - protected override int SubscriptionDelayInMs { get; } = 1000; - public MongoEventStoreTests_Direct(MongoEventStoreDirectFixture fixture) { _ = fixture; diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs index 7250ad7e0..dd481fd92 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs @@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing { public MongoEventStoreFixture _ { get; } - protected override int SubscriptionDelayInMs { get; } = 1000; - public MongoEventStoreTests_ReplicaSet(MongoEventStoreReplicaSetFixture fixture) { _ = fixture; diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs index e68fbb0ce..66e91f4ca 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs @@ -5,7 +5,9 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Globalization; using FakeItEasy; +using FluentAssertions; using Xunit; namespace Squidex.Infrastructure.EventSourcing @@ -15,61 +17,54 @@ namespace Squidex.Infrastructure.EventSourcing private readonly IEventStore eventStore = A.Fake(); private readonly IEventSubscriber eventSubscriber = A.Fake>(); private readonly string position = Guid.NewGuid().ToString(); + private readonly string filter = "^my-stream"; [Fact] public async Task Should_subscribe_on_start() { - var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position); + await SubscribeAsync(false); - await WaitAndStopAsync(sut); - - A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A._, A._)) + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) .MustHaveHappenedOnceExactly(); } [Fact] - public async Task Should_propagate_exception_to_subscriber() + public async Task Should_forward_exception_to_subscriber() { var ex = new InvalidOperationException(); - A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A._, A._)) + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) .Throws(ex); - var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position); - - await WaitAndStopAsync(sut); + var sut = await SubscribeAsync(false); A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex)) .MustHaveHappened(); } [Fact] - public async Task Should_propagate_operation_cancelled_exception_to_subscriber() + public async Task Should_forward_operation_cancelled_exception_to_subscriber() { var ex = new OperationCanceledException(); - A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A._, A._)) + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) .Throws(ex); - var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position); - - await WaitAndStopAsync(sut); + var sut = await SubscribeAsync(false); A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex)) .MustHaveHappened(); } [Fact] - public async Task Should_propagate_aggregate_operation_cancelled_exception_to_subscriber() + public async Task Should_forward_aggregate_operation_cancelled_exception_to_subscriber() { var ex = new AggregateException(new OperationCanceledException()); - A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A._, A._)) + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) .Throws(ex); - var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position); - - await WaitAndStopAsync(sut); + var sut = await SubscribeAsync(false); A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex)) .MustHaveHappened(); @@ -78,21 +73,88 @@ namespace Squidex.Infrastructure.EventSourcing [Fact] public async Task Should_wake_up() { - var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position); + var sut = await SubscribeAsync(true); - sut.WakeUp(); + A.CallTo(() => eventStore.QueryAllAsync(filter, A._, A._, A._)) + .MustHaveHappened(2, Times.Exactly); + } - await WaitAndStopAsync(sut); + [Fact] + public async Task Should_forward_events_to_subscriber() + { + var events = Enumerable.Range(0, 50).Select(CreateEvent).ToArray(); - A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A._, A._)) - .MustHaveHappened(2, Times.Exactly); + var receivedEvents = new List(); + + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) + .Returns(events.ToAsyncEnumerable()); + + A.CallTo(() => eventSubscriber.OnNextAsync(A._, A._)) + .Invokes(x => receivedEvents.Add(x.GetArgument(1)!)); + + await SubscribeAsync(true); + + receivedEvents.Should().BeEquivalentTo(events); } - private static async Task WaitAndStopAsync(IEventSubscription sut) + [Fact] + public async Task Should_continue_on_last_position() { - await Task.Delay(200); + var events1 = Enumerable.Range(10, 10).Select(CreateEvent).ToArray(); + var events2 = Enumerable.Range(20, 10).Select(CreateEvent).ToArray(); + + var lastPosition = events1[^1].EventPosition; + + var receivedEvents = new List(); + + A.CallTo(() => eventStore.QueryAllAsync(filter, position, A._, A._)) + .Returns(events1.ToAsyncEnumerable()); + + A.CallTo(() => eventStore.QueryAllAsync(filter, lastPosition, A._, A._)) + .Returns(events2.ToAsyncEnumerable()); + + A.CallTo(() => eventSubscriber.OnNextAsync(A._, A._)) + .Invokes(x => receivedEvents.Add(x.GetArgument(1)!)); + + await SubscribeAsync(true); - sut.Dispose(); + receivedEvents.Should().BeEquivalentTo(events1.Union(events2)); + } + + private StoredEvent CreateEvent(int position) + { + return new StoredEvent( + "my-stream", + position.ToString(CultureInfo.InvariantCulture)!, + position, + new EventData( + "type", + new EnvelopeHeaders + { + [CommonHeaders.EventId] = Guid.NewGuid().ToString() + }, + "payload")); + } + + private async Task SubscribeAsync(bool wakeup = true) + { + var sut = new PollingSubscription(eventStore, eventSubscriber, filter, position); + + try + { + if (wakeup) + { + sut.WakeUp(); + } + + await Task.Delay(200); + } + finally + { + sut.Dispose(); + } + + return sut; } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs new file mode 100644 index 000000000..380bf48b5 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs @@ -0,0 +1,208 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using FakeItEasy; +using Squidex.Infrastructure.TestHelpers; +using Xunit; + +namespace Squidex.Infrastructure.States +{ + public class SimpleStateTests + { + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private readonly CancellationToken ct; + private readonly TestState testState = new TestState(DomainId.NewGuid()); + private readonly SimpleState sut; + + public SimpleStateTests() + { + ct = cts.Token; + + sut = new SimpleState(testState.PersistenceFactory, GetType(), testState.Id); + } + + [Fact] + public void Should_init_with_base_data() + { + Assert.Equal(-1, sut.Version); + Assert.NotNull(sut.Value); + } + + [Fact] + public async Task Should_get_state_from_persistence_on_load() + { + testState.Version = 42; + testState.Snapshot = new MyDomainState { Value = 50 }; + + await sut.LoadAsync(ct); + + Assert.Equal(42, sut.Version); + Assert.Equal(50, sut.Value.Value); + + A.CallTo(() => testState.Persistence.ReadAsync(-2, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_delete_when_clearing() + { + await sut.ClearAsync(ct); + + A.CallTo(() => testState.Persistence.DeleteAsync(ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_invoke_persistence_when_writing_state() + { + await sut.WriteAsync(ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_load_once_on_update() + { + await sut.UpdateAsync(x => true, ct: ct); + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.ReadAsync(-2, ct)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_write_state_on_update_when_callback_returns_true() + { + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_not_write_state_on_update_when_callback_returns_false() + { + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_write_state_on_update_and_return_when_callback_returns_true() + { + var result = await sut.UpdateAsync(x => (true, 42), ct: ct); + + Assert.Equal(42, result); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_not_write_state_on_update_and_return_when_callback_returns_false() + { + var result = await sut.UpdateAsync(x => (false, 42), ct: ct); + + Assert.Equal(42, result); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustNotHaveHappened(); + } + + [Fact] + public async Task Should_retry_update_when_failed_with_inconsistency_issue() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5); + + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + } + + [Fact] + public async Task Should_give_up_update_after_too_many_inconsistency_issues() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => true, ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + } + + [Fact] + public async Task Should_not_retry_update_with_other_exception() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InvalidOperationException()); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => true, ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + } + + [Fact] + public async Task Should_retry_update_and_return_when_failed_with_inconsistency_issue() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5); + + await sut.UpdateAsync(x => (true, 42), ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + } + + [Fact] + public async Task Should_give_up_update_and_return_after_too_many_inconsistency_issues() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => (true, 42), ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + } + + [Fact] + public async Task Should_not_retry_update_and_return_with_other_exception() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InvalidOperationException()); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => (true, 42), ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs index 9ab205046..6cc935ac8 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs @@ -14,9 +14,12 @@ namespace Squidex.Infrastructure.TestHelpers public sealed class TestState where T : class, new() { private readonly List> events = new List>(); + private readonly ISnapshotStore snapshotStore = A.Fake>(); private HandleSnapshot? handleSnapshot; private HandleEvent? handleEvent; + public DomainId Id { get; } + public IPersistenceFactory PersistenceFactory { get; } public IPersistence Persistence { get; } = A.Fake>(); @@ -30,20 +33,24 @@ namespace Squidex.Infrastructure.TestHelpers { } - public void AddEvent(Envelope @event) - { - events.Add(@event); - } - - public void AddEvent(IEvent @event) - { - events.Add(Envelope.Create(@event)); - } - public TestState(DomainId id, IPersistenceFactory? persistenceFactory = null) { + Id = id; + PersistenceFactory = persistenceFactory ?? A.Fake>(); + A.CallTo(() => PersistenceFactory.Snapshots) + .Returns(snapshotStore); + + A.CallTo(() => Persistence.Version) + .ReturnsLazily(() => Version); + + A.CallTo(() => snapshotStore.ReadAllAsync(A._)) + .ReturnsLazily(() => new List> + { + new SnapshotResult(id, Snapshot, Version, true) + }.ToAsyncEnumerable()); + A.CallTo(() => PersistenceFactory.WithEventSourcing(A._, id, A._)) .Invokes(x => { @@ -98,5 +105,15 @@ namespace Squidex.Infrastructure.TestHelpers Snapshot = new T(); }); } + + public void AddEvent(Envelope @event) + { + events.Add(@event); + } + + public void AddEvent(IEvent @event) + { + events.Add(Envelope.Create(@event)); + } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs index 3b3cd2a0e..50f0e09cd 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs @@ -65,6 +65,21 @@ namespace Squidex.Infrastructure.UsageTracking await Assert.ThrowsAsync(() => sut.GetAsync(key, date, date, null, ct)); } + [Fact] + public void Should_provide_fallback_category() + { + Assert.Equal("*", sut.FallbackCategory); + } + + [Fact] + public async Task Should_forward_delete_prefix_call() + { + await sut.DeleteByKeyPatternAsync("pattern", ct); + + A.CallTo(() => usageStore.DeleteByKeyPatternAsync("pattern", ct)) + .MustHaveHappened(); + } + [Fact] public async Task Should_forward_delete_call() { diff --git a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs index 47474fff5..8f104e3f4 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs @@ -30,6 +30,24 @@ namespace Squidex.Infrastructure.UsageTracking sut = new CachingUsageTracker(inner, cache); } + [Fact] + public void Should_forward_fallback_category() + { + A.CallTo(() => inner.FallbackCategory) + .Returns("*"); + + Assert.Equal("*", sut.FallbackCategory); + } + + [Fact] + public async Task Should_forward_delete_prefix_call() + { + await sut.DeleteByKeyPatternAsync("pattern", ct); + + A.CallTo(() => inner.DeleteByKeyPatternAsync("pattern", ct)) + .MustHaveHappened(); + } + [Fact] public async Task Should_forward_delete_call() { diff --git a/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs b/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs index 9a94f2f0f..0cea61621 100644 --- a/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs +++ b/backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs @@ -319,6 +319,63 @@ namespace TestSuite.ApiTests Assert.Equal(fileNameRequest.FileName, asset_4.FileName); } + [Fact] + public async Task Should_annotate_asset_in_parallel() + { + var tag1 = $"tag_{Guid.NewGuid()}"; + var tag2 = $"tag_{Guid.NewGuid()}"; + + var metadataRequest = new AnnotateAssetDto + { + Tags = new List + { + tag1, + tag2 + } + }; + + + // STEP 1: Create asset + var asset_1 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png"); + + + var numErrors = 0; + var numSuccess = 0; + + // STEP 3: Make parallel upserts. + await Parallel.ForEachAsync(Enumerable.Range(0, 20), async (i, ct) => + { + try + { + await _.Assets.PutAssetAsync(_.AppName, asset_1.Id, metadataRequest); + + Interlocked.Increment(ref numSuccess); + } + catch (SquidexManagementException ex) when (ex.StatusCode is 409 or 412) + { + Interlocked.Increment(ref numErrors); + return; + } + }); + + // At least some errors and success should have happened. + Assert.True(numErrors > 0); + Assert.True(numSuccess > 0); + + + // STEP 3: Make an normal update to ensure nothing is corrupt. + await _.Assets.PutAssetAsync(_.AppName, asset_1.Id, metadataRequest); + + + // STEP 4: Check tags + var tags = await _.Assets.WaitForTagsAsync(_.AppName, tag1, TimeSpan.FromMinutes(2)); + + Assert.Contains(tag1, tags); + Assert.Contains(tag2, tags); + Assert.Equal(1, tags[tag1]); + Assert.Equal(1, tags[tag2]); + } + [Fact] public async Task Should_protect_asset() { @@ -490,10 +547,18 @@ namespace TestSuite.ApiTests var asset_1 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png", null, folder_2.Id); - // STEP 4: Delete folder. + // STEP 4: Create asset outside folder + var asset_2 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png"); + + + // STEP 5: Delete folder. await _.Assets.DeleteAssetFolderAsync(_.AppName, folder_1.Id); + // Ensure that asset in folder is deleted. Assert.True(await _.Assets.WaitForDeletionAsync(_.AppName, asset_1.Id, TimeSpan.FromSeconds(30))); + + // Ensure that other asset is not deleted. + Assert.NotNull(await _.Assets.GetAssetAsync(_.AppName, asset_2.Id)); } [Theory] diff --git a/backend/tools/TestSuite/TestSuite.Shared/ClientExtensions.cs b/backend/tools/TestSuite/TestSuite.Shared/ClientExtensions.cs index 6ddf075fb..7e111b203 100644 --- a/backend/tools/TestSuite/TestSuite.Shared/ClientExtensions.cs +++ b/backend/tools/TestSuite/TestSuite.Shared/ClientExtensions.cs @@ -38,6 +38,31 @@ namespace TestSuite return false; } + public static async Task> WaitForTagsAsync(this IAssetsClient assetsClient, string app, string id, TimeSpan timeout) + { + try + { + using var cts = new CancellationTokenSource(timeout); + + while (!cts.IsCancellationRequested) + { + var tags = await assetsClient.GetTagsAsync(app, cts.Token); + + if (tags.TryGetValue(id, out var count) && count > 0) + { + return tags; + } + + await Task.Delay(200, cts.Token); + } + } + catch (OperationCanceledException) + { + } + + return await assetsClient.GetTagsAsync(app); + } + public static async Task WaitForBackupAsync(this IBackupsClient backupsClient, string app, TimeSpan timeout) { try