From f499c677f53e2b28e893da69be95d305743755b5 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Wed, 27 Jul 2022 22:07:11 +0200 Subject: [PATCH] Improved consistency for assets. --- .../Tags/ITagService.cs | 2 +- .../Tags/TagsExport.cs | 27 +- .../Assets/AssetUsageTracker.cs | 24 +- .../Assets/AssetUsageTracker_EventHandling.cs | 137 ++++++-- .../Assets/BackupAssets.cs | 19 +- .../Assets/DomainObject/AssetDomainObject.cs | 4 +- .../Tags/TagService.cs | 102 ++++-- .../Assets/AssetAnnotated.cs | 2 - .../Assets/AssetDeleted.cs | 2 - .../MongoDb/MongoBase.cs | 3 - .../States/MongoSnapshotStoreBase.cs | 10 + .../UsageTracking/MongoUsageRepository.cs | 9 + .../Consume/EventConsumerManager.cs | 8 +- .../Squidex.Infrastructure/States/IOnRead.cs | 14 + .../States/SimpleState.cs | 40 +-- .../UsageTracking/BackgroundUsageTracker.cs | 13 +- .../UsageTracking/CachingUsageTracker.cs | 13 +- .../UsageTracking/IUsageRepository.cs | 3 + .../UsageTracking/IUsageTracker.cs | 5 + .../Operations/Tags/TagNormalizerTests.cs | 136 -------- .../Assets/AssetUsageTrackerTests.cs | 304 +++++++++++++++++- .../DomainObject/AssetDomainObjectTests.cs | 2 +- .../Assets/Queries/AssetEnricherTests.cs | 4 +- .../Tags/TagServiceTests.cs | 183 +++++++---- .../CollectionExtensionsTests.cs | 18 -- .../Consume/EventConsumerManagerTests.cs | 53 ++- .../States/SimpleStateTests.cs | 208 ++++++++++++ .../TestHelpers/TestState.cs | 17 + .../BackgroundUsageTrackerTests.cs | 15 + .../UsageTracking/CachingUsageTrackerTests.cs | 18 ++ 30 files changed, 1019 insertions(+), 376 deletions(-) create mode 100644 backend/src/Squidex.Infrastructure/States/IOnRead.cs delete mode 100644 backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs index 732bb5bae..5a6a93049 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs @@ -36,6 +36,6 @@ namespace Squidex.Domain.Apps.Core.Tags CancellationToken ct = default); Task ClearAsync( - CancellationToken ct); + CancellationToken ct = default); } } diff --git a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs index 0c484172e..b7e0c81b2 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs @@ -9,31 +9,8 @@ namespace Squidex.Domain.Apps.Core.Tags { public class TagsExport { - private Dictionary tags; - private Dictionary alias; + public Dictionary Tags { get; set; } = new Dictionary(); - public Dictionary Tags - { - get => tags ??= new Dictionary(); - set => tags = value ?? new Dictionary(); - } - - public Dictionary Alias - { - get => alias ??= new Dictionary(); - set => alias = value ?? new Dictionary(); - } - - public TagsExport Clone() - { - var clonedAlias = new Dictionary(Alias); - - var clonedTags = - Tags.ToDictionary( - x => x.Key, - x => x.Value with { }); - - return new TagsExport { Alias = clonedAlias, Tags = clonedTags }; - } + public Dictionary Alias { get; set; } = new Dictionary(); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs index 2acf378c2..f08afe66e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs @@ -8,6 +8,7 @@ using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Infrastructure; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; #pragma warning disable CS0649 @@ -19,14 +20,26 @@ namespace Squidex.Domain.Apps.Entities.Assets private const string CounterTotalCount = "TotalAssets"; private const string CounterTotalSize = "TotalSize"; private static readonly DateTime SummaryDate; - private readonly IUsageTracker usageTracker; + private readonly IAssetLoader assetLoader; + private readonly ISnapshotStore store; private readonly ITagService tagService; + private readonly IUsageTracker usageTracker; - public AssetUsageTracker(IUsageTracker usageTracker, - ITagService tagService) + [CollectionName("Index_TagHistory")] + public sealed class State { - this.usageTracker = usageTracker; + public HashSet? Tags { get; set; } + } + + public AssetUsageTracker(IUsageTracker usageTracker, IAssetLoader assetLoader, ITagService tagService, + ISnapshotStore store) + { + this.assetLoader = assetLoader; this.tagService = tagService; + this.store = store; + this.usageTracker = usageTracker; + + ClearCache(); } Task IDeleter.DeleteAppAsync(IAppEntity app, @@ -52,12 +65,13 @@ namespace Squidex.Domain.Apps.Entities.Assets var usages = await usageTracker.QueryAsync(GetKey(appId), fromDate, toDate); - if (usages.TryGetValue("*", out var byCategory1)) + if (usages.TryGetValue(usageTracker.FallbackCategory, out var byCategory1)) { AddCounters(enriched, byCategory1); } else if (usages.TryGetValue("Default", out var byCategory2)) { + // Fallback for older versions where default was uses as tracking category. AddCounters(enriched, byCategory2); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs index 2292b8ec0..dfc931c6a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs @@ -5,10 +5,13 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Options; using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; #pragma warning disable MA0048 // File name must match type name @@ -17,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Assets { public partial class AssetUsageTracker : IEventConsumer { + private IMemoryCache memoryCache; + public int BatchSize { get => 1000; @@ -37,63 +42,137 @@ namespace Squidex.Domain.Apps.Entities.Assets get => "^asset-"; } - public bool CanClear + private void ClearCache() { - get => false; + memoryCache?.Dispose(); + memoryCache = new MemoryCache(Options.Create(new MemoryCacheOptions())); } - public async Task On(IEnumerable> events) + public async Task ClearAsync() { - var tags = new Dictionary>(); + // Will not remove data, but reset alls counts to zero. + await tagService.ClearAsync(); + + // Also clear the store and cache, because otherwise we would use data from the future when querying old tags. + ClearCache(); + + await store.ClearAsync(); + + // Use a well defined prefix query for the deletion to improve performance. + await usageTracker.DeleteByKeyPatternAsync("^([a-zA-Z0-9]+)_Assets"); + } + public async Task On(IEnumerable> events) + { foreach (var @event in events) { + // Usage tracking is done in the backgroud, therefore we do no use any batching. await TrackUsageAsync(@event); - - AddTags(@event, tags); } - foreach (var (appId, updates) in tags) - { - await tagService.UpdateAsync(appId, TagGroups.Assets, updates); - } + // Event consumers should only do one task, but too many consumers also hurt performance. + await AddTagsAsync(events); } - private static void AddTags(Envelope @event, Dictionary> tags) + private async Task AddTagsAsync(IEnumerable> events) { - if (@event.Headers.Restored()) - { - return; - } + var tagsPerApp = new Dictionary>(); + var tagsPerAsset = new Dictionary(); - void AddTags(DomainId appId, HashSet? tagIds, int count) + void AddTagsToStore(DomainId appId, HashSet? tagIds, int count) { if (tagIds != null) { + var perApp = tagsPerApp.GetOrAddNew(appId); + foreach (var tag in tagIds) { - var perApp = tags.GetOrAddNew(appId); - perApp[tag] = perApp.GetOrAddDefault(tag) + count; } } } - switch (@event.Payload) + void AddTagsToCache(DomainId key, HashSet? tags, long version) { - case AssetCreated assetCreated: - AddTags(assetCreated.AppId.Id, assetCreated.Tags, 1); - break; + // Also cache null tags to keep them in as valid state in cache and store. + var state = new State { Tags = tags }; - case AssetAnnotated assetAnnotated when assetAnnotated.Tags != null && assetAnnotated.OldTags != null: - AddTags(assetAnnotated.AppId.Id, assetAnnotated.Tags, 1); - AddTags(assetAnnotated.AppId.Id, assetAnnotated.OldTags, -1); - break; + // Write tags to a buffer so that we can write them to a store in batches. + tagsPerAsset[key] = state; - case AssetDeleted assetDeleted: - AddTags(assetDeleted.AppId.Id, assetDeleted.OldTags, -1); - break; + // Write to the cache immediately, to be available for the next event. Use a relatively long cache time for live updates. + memoryCache.Set(key, state, TimeSpan.FromHours(1)); + } + + foreach (var @event in events) + { + var typedEvent = (AssetEvent)@event.Payload; + var appId = typedEvent.AppId.Id; + var assetId = typedEvent.AssetId; + var assetKey = @event.Headers.AggregateId(); + var version = @event.Headers.EventStreamNumber(); + + switch (typedEvent) + { + case AssetCreated assetCreated: + { + AddTagsToStore(appId, assetCreated.Tags, 1); + + AddTagsToCache(assetKey, assetCreated.Tags, version); + break; + } + + case AssetAnnotated assetAnnotated when assetAnnotated.Tags != null: + { + var oldTags = await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default); + + AddTagsToStore(appId, assetAnnotated.Tags, 1); + AddTagsToStore(appId, oldTags, -1); + + AddTagsToCache(assetKey, assetAnnotated.Tags, version); + break; + } + + case AssetDeleted assetDeleted: + { + var oldTags = await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default); + + AddTagsToStore(appId, oldTags, -1); + break; + } + } + } + + // There is no good solution for batching anyway, so there is no need to build a method for that. + foreach (var (appId, updates) in tagsPerApp) + { + await tagService.UpdateAsync(appId, TagGroups.Assets, updates); } + + await store.WriteManyAsync(tagsPerAsset.Select(x => new SnapshotWriteJob(x.Key, x.Value, 0))); + } + + private async Task?> GetAndUpdateOldTagsAsync(DomainId appId, DomainId assetId, DomainId key, long version, + CancellationToken ct) + { + // Store the latest tags in memory for fast access. + if (memoryCache.TryGetValue(key, out var state)) + { + return state.Tags; + } + + var stored = await store.ReadAsync(key, ct); + + // Stored state can be null, if not serialized yet. + if (stored.Value != null) + { + return stored.Value.Tags; + } + + // This will replay a lot of events, so it is the slowest alternative. + var previousAsset = await assetLoader.GetAsync(appId, assetId, version - 1, ct); + + return previousAsset?.Tags; } private Task TrackUsageAsync(Envelope @event) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs index 740a18050..409f858e7 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs @@ -9,6 +9,7 @@ using Squidex.Assets; using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Entities.Assets.DomainObject; using Squidex.Domain.Apps.Entities.Backup; +using Squidex.Domain.Apps.Events.Apps; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.Commands; @@ -71,6 +72,10 @@ namespace Squidex.Domain.Apps.Entities.Assets { switch (@event.Payload) { + case AppCreated: + // Restore the tags first so that the processing of consecutive events have the necessary structure. + await RestoreTagsAsync(context, ct); + break; case AssetFolderCreated: assetFolderIds.Add(@event.Headers.AggregateId()); break; @@ -100,8 +105,6 @@ namespace Squidex.Domain.Apps.Entities.Assets public async Task RestoreAsync(RestoreContext context, CancellationToken ct) { - await RestoreTagsAsync(context, ct); - if (assetIds.Count > 0) { await rebuilder.InsertManyAsync(assetIds, BatchSize, ct); @@ -125,6 +128,7 @@ namespace Squidex.Domain.Apps.Entities.Assets var alias = (Dictionary?)null; + // For backwards compabibility we store the tags and the aliases in different locations. if (await context.Reader.HasFileAsync(TagsAliasFile, ct)) { alias = await context.Reader.ReadJsonAsync>(TagsAliasFile, ct); @@ -135,6 +139,15 @@ namespace Squidex.Domain.Apps.Entities.Assets return; } + if (tags != null) + { + // Import the tags without count, because they will populated later by the event processor. + foreach (var (_, tag) in tags) + { + tag.Count = 0; + } + } + var export = new TagsExport { Tags = tags!, Alias = alias! }; await tagService.RebuildTagsAsync(context.AppId, TagGroups.Assets, export, ct); @@ -147,11 +160,13 @@ namespace Squidex.Domain.Apps.Entities.Assets if (tags.Tags != null) { + // Export the tags with count, even though we do not need it. But in general it makes the code easier. await context.Writer.WriteJsonAsync(TagsFile, tags.Tags, ct); } if (tags.Alias?.Count > 0) { + // For backwards compabibility we store the tags and the aliases in different locations. await context.Writer.WriteJsonAsync(TagsAliasFile, tags.Alias, ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs index 50412d991..ea4022cec 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs @@ -250,7 +250,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject private void Annotate(AnnotateAsset command) { - Raise(command, new AssetAnnotated { OldTags = Snapshot.Tags }); + Raise(command, new AssetAnnotated()); } private void Move(MoveAsset command) @@ -260,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject private void Delete(DeleteAsset command) { - Raise(command, new AssetDeleted { OldTags = Snapshot.Tags, DeletedSize = Snapshot.TotalSize }); + Raise(command, new AssetDeleted { DeletedSize = Snapshot.TotalSize }); } private void Raise(T command, TEvent @event) where T : class where TEvent : AppEvent diff --git a/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs b/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs index f2473d98c..e84cc2656 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs @@ -5,9 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Threading.Tasks.Dataflow; using Squidex.Domain.Apps.Core.Tags; using Squidex.Infrastructure; using Squidex.Infrastructure.States; +using Squidex.Infrastructure.Tasks; namespace Squidex.Domain.Apps.Entities.Tags { @@ -16,32 +18,51 @@ namespace Squidex.Domain.Apps.Entities.Tags private readonly IPersistenceFactory persistenceFactory; [CollectionName("Index_Tags")] - public sealed class State : TagsExport + public sealed class State : TagsExport, IOnRead { - public bool Rebuild(TagsExport export) + public ValueTask OnReadAsync() + { + if (Tags == null) + { + Tags = new Dictionary(); + } + + if (Alias == null) + { + Alias = new Dictionary(); + } + + return default; + } + + public bool Clear() { var isChanged = false; - if (!Tags.EqualsDictionary(export.Tags)) + foreach (var (_, tag) in Tags) { - Tags = export.Tags; - isChanged = true; + isChanged = tag.Count > 0; + + tag.Count = 0; } - if (!Alias.EqualsDictionary(export.Alias)) + return isChanged; + } + + public bool Rebuild(TagsExport export) + { + Tags = export.Tags; + + if (Alias.EqualsDictionary(export.Alias)) { Alias = export.Alias; - isChanged = true; } - return isChanged; + return true; } public bool Rename(string name, string newName) { - Guard.NotNull(name); - Guard.NotNull(newName); - name = NormalizeName(name); if (!TryGetTag(name, out var tag)) @@ -94,8 +115,6 @@ namespace Squidex.Domain.Apps.Entities.Tags public (bool, Dictionary) GetIds(HashSet names) { - Guard.NotNull(names); - var tagIds = new Dictionary(); var isChanged = false; @@ -142,13 +161,6 @@ namespace Squidex.Domain.Apps.Entities.Tags return new TagsSet(clone, version); } - public TagsExport GetExportableTags() - { - var clone = Clone(); - - return clone; - } - private static string NormalizeName(string name) { return name.Trim().ToLowerInvariant(); @@ -242,7 +254,7 @@ namespace Squidex.Domain.Apps.Entities.Tags { var state = await GetStateAsync(id, group, ct); - return state.Value.GetExportableTags(); + return state.Value; } public async Task ClearAsync(DomainId id, string group, @@ -263,10 +275,52 @@ namespace Squidex.Domain.Apps.Entities.Tags return state; } - public Task ClearAsync( - CancellationToken ct) + public async Task ClearAsync( + CancellationToken ct = default) { - return persistenceFactory.Snapshots.ClearAsync(ct); + var writerBlock = new ActionBlock[]>(async batch => + { + try + { + var isChanged = !batch.All(x => !x.Value.Clear()); + + if (isChanged) + { + var jobs = batch.Select(x => new SnapshotWriteJob(x.Key, x.Value, x.Version)); + + await persistenceFactory.Snapshots.WriteManyAsync(jobs, ct); + } + } + catch (OperationCanceledException ex) + { + // Dataflow swallows operation cancelled exception. + throw new AggregateException(ex); + } + }, + new ExecutionDataflowBlockOptions + { + BoundedCapacity = 2, + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1, + }); + + // Create batches of 500 items to clear the tag count for better performance. + var batchBlock = new BatchBlock>(500, new GroupingDataflowBlockOptions + { + BoundedCapacity = 500 + }); + + batchBlock.BidirectionalLinkTo(writerBlock); + + await foreach (var state in persistenceFactory.Snapshots.ReadAllAsync(ct)) + { + // Uses back-propagation to not query additional items from the database, when queue is full. + await batchBlock.SendAsync(state, ct); + } + + batchBlock.Complete(); + + await writerBlock.Completion; } } } diff --git a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetAnnotated.cs b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetAnnotated.cs index 878ea728d..1b503677c 100644 --- a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetAnnotated.cs +++ b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetAnnotated.cs @@ -22,7 +22,5 @@ namespace Squidex.Domain.Apps.Events.Assets public AssetMetadata? Metadata { get; set; } public HashSet? Tags { get; set; } - - public HashSet? OldTags { get; set; } } } diff --git a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs index aaeb5e786..23f983a7e 100644 --- a/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs +++ b/backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs @@ -13,7 +13,5 @@ namespace Squidex.Domain.Apps.Events.Assets public sealed class AssetDeleted : AssetEvent { public long DeletedSize { get; set; } - - public HashSet? OldTags { get; set; } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs index 912525612..ec58c7f48 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs @@ -52,9 +52,6 @@ namespace Squidex.Infrastructure.MongoDb BsonJsonConvention.Register(); BsonJsonValueSerializer.Register(); BsonStringSerializer.Register(); - BsonStringSerializer>.Register(); - BsonStringSerializer>.Register(); - BsonStringSerializer>.Register(); } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs index 7c1b7d3d5..0ed74f32c 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs @@ -39,6 +39,11 @@ namespace Squidex.Infrastructure.States if (existing != null) { + if (existing.Document is IOnRead onRead) + { + await onRead.OnReadAsync(); + } + return new SnapshotResult(existing.DocumentId, existing.Document, existing.Version); } @@ -95,6 +100,11 @@ namespace Squidex.Infrastructure.States await foreach (var document in find.ToAsyncEnumerable(ct)) { + if (document.Document is IOnRead onRead) + { + await onRead.OnReadAsync(); + } + yield return new SnapshotResult(document.DocumentId, document.Document, document.Version, true); } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs b/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs index 495f89290..1e1b6e986 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using MongoDB.Bson; using MongoDB.Driver; using Squidex.Infrastructure.MongoDb; @@ -42,6 +43,14 @@ namespace Squidex.Infrastructure.UsageTracking return Collection.DeleteManyAsync(x => x.Key == key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return Collection.DeleteManyAsync(Filter.Regex(x => x.Key, new BsonRegularExpression(pattern)), ct); + } + public async Task TrackUsagesAsync(UsageUpdate update, CancellationToken ct = default) { diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs index 0e613e3f9..c08e1d1ec 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs @@ -14,12 +14,14 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { private readonly IPersistenceFactory persistence; private readonly IMessageBus messaging; + private readonly HashSet activeNames; - public EventConsumerManager(IPersistenceFactory persistence, + public EventConsumerManager(IPersistenceFactory persistence, IEnumerable eventConsumers, IMessageBus messaging) { this.persistence = persistence; this.messaging = messaging; + this.activeNames = eventConsumers.Select(x => x.Name).ToHashSet(); } public async Task> GetConsumersAsync( @@ -27,7 +29,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { var snapshots = await persistence.Snapshots.ReadAllAsync(ct).ToListAsync(ct); - return snapshots.Select(x => x.Value.ToInfo(x.Key.ToString())).ToList(); + return snapshots.Where(x => activeNames.Contains(x.Key.ToString())).Select(x => x.Value.ToInfo(x.Key.ToString())).ToList(); } public async Task ResetAsync(string consumerName, @@ -67,7 +69,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume await state.LoadAsync(ct); - if (state.Version <= EtagVersion.Empty) + if (state.Version <= EtagVersion.Empty || !activeNames.Contains(consumerName)) { throw new DomainObjectNotFoundException(consumerName); } diff --git a/backend/src/Squidex.Infrastructure/States/IOnRead.cs b/backend/src/Squidex.Infrastructure/States/IOnRead.cs new file mode 100644 index 000000000..3e3de6b5e --- /dev/null +++ b/backend/src/Squidex.Infrastructure/States/IOnRead.cs @@ -0,0 +1,14 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Infrastructure.States +{ + public interface IOnRead + { + ValueTask OnReadAsync(); + } +} diff --git a/backend/src/Squidex.Infrastructure/States/SimpleState.cs b/backend/src/Squidex.Infrastructure/States/SimpleState.cs index 0e7e2ae16..ad3752e32 100644 --- a/backend/src/Squidex.Infrastructure/States/SimpleState.cs +++ b/backend/src/Squidex.Infrastructure/States/SimpleState.cs @@ -64,36 +64,19 @@ namespace Squidex.Infrastructure.States return persistence.WriteEventAsync(envelope, ct); } - public async Task UpdateAsync(Func updater, int retries = 20, + public Task UpdateAsync(Func updater, int retries = 20, CancellationToken ct = default) { - await EnsureLoadedAsync(ct); - - for (var i = 0; i < retries; i++) - { - try - { - var isChanged = updater(Value); - - if (!isChanged) - { - return; - } - - await WriteAsync(ct); - return; - } - catch (InconsistentStateException) when (i < retries - 1) - { - await LoadAsync(ct); - } - } + return UpdateAsync(state => (updater(state), None.Value), retries, ct); } public async Task UpdateAsync(Func updater, int retries = 20, CancellationToken ct = default) { - await EnsureLoadedAsync(ct); + if (!isLoaded) + { + await LoadAsync(ct); + } for (var i = 0; i < retries; i++) { @@ -117,16 +100,5 @@ namespace Squidex.Infrastructure.States return default!; } - - private async Task EnsureLoadedAsync( - CancellationToken ct) - { - if (isLoaded) - { - return; - } - - await LoadAsync(ct); - } } } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs index c3560bd8d..e9b770286 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs @@ -14,7 +14,6 @@ namespace Squidex.Infrastructure.UsageTracking public sealed class BackgroundUsageTracker : DisposableObjectBase, IUsageTracker { private const int Intervall = 60 * 1000; - private const string FallbackCategory = "*"; private readonly IUsageRepository usageRepository; private readonly ILogger log; private readonly CompletionTimer timer; @@ -22,6 +21,8 @@ namespace Squidex.Infrastructure.UsageTracking public bool ForceWrite { get; set; } + public string FallbackCategory => "*"; + public BackgroundUsageTracker(IUsageRepository usageRepository, ILogger log) { @@ -96,6 +97,14 @@ namespace Squidex.Infrastructure.UsageTracking return usageRepository.DeleteAsync(key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return usageRepository.DeleteByKeyPatternAsync(pattern, ct); + } + public Task TrackAsync(DateTime date, string key, string? category, Counters counters, CancellationToken ct = default) { @@ -187,7 +196,7 @@ namespace Squidex.Infrastructure.UsageTracking return result; } - private static string GetCategory(string? category) + private string GetCategory(string? category) { return !string.IsNullOrWhiteSpace(category) ? category.Trim() : FallbackCategory; } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs index 6e5b006aa..05692da36 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs @@ -15,11 +15,10 @@ namespace Squidex.Infrastructure.UsageTracking private readonly IUsageTracker inner; private readonly IMemoryCache cache; + public string FallbackCategory => inner.FallbackCategory; + public CachingUsageTracker(IUsageTracker inner, IMemoryCache cache) { - Guard.NotNull(inner); - Guard.NotNull(cache); - this.inner = inner; this.cache = cache; } @@ -32,6 +31,14 @@ namespace Squidex.Infrastructure.UsageTracking return inner.DeleteAsync(key, ct); } + public Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default) + { + Guard.NotNull(pattern); + + return inner.DeleteByKeyPatternAsync(pattern, ct); + } + public Task>> QueryAsync(string key, DateTime fromDate, DateTime toDate, CancellationToken ct = default) { diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs index 232754e28..cb1fdd54a 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs @@ -20,5 +20,8 @@ namespace Squidex.Infrastructure.UsageTracking Task DeleteAsync(string key, CancellationToken ct = default); + + Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default); } } diff --git a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs index 72b4a5f84..9819d9452 100644 --- a/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs +++ b/backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs @@ -9,6 +9,8 @@ namespace Squidex.Infrastructure.UsageTracking { public interface IUsageTracker { + string FallbackCategory { get; } + Task TrackAsync(DateTime date, string key, string? category, Counters counters, CancellationToken ct = default); @@ -23,5 +25,8 @@ namespace Squidex.Infrastructure.UsageTracking Task DeleteAsync(string key, CancellationToken ct = default); + + Task DeleteByKeyPatternAsync(string pattern, + CancellationToken ct = default); } } diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs deleted file mode 100644 index bcd325b42..000000000 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs +++ /dev/null @@ -1,136 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using FakeItEasy; -using Squidex.Domain.Apps.Core.Contents; -using Squidex.Domain.Apps.Core.Schemas; -using Squidex.Domain.Apps.Core.Tags; -using Squidex.Domain.Apps.Core.TestHelpers; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Json.Objects; -using Xunit; - -namespace Squidex.Domain.Apps.Core.Operations.Tags -{ - public class TagNormalizerTests - { - private readonly ITagService tagService = A.Fake(); - private readonly DomainId appId = DomainId.NewGuid(); - private readonly DomainId schemaId = DomainId.NewGuid(); - private readonly Schema schema; - - public TagNormalizerTests() - { - schema = - new Schema("my-schema") - .AddTags(1, "tags1", Partitioning.Invariant) - .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema }) - .AddString(3, "string", Partitioning.Invariant) - .AddArray(4, "array", Partitioning.Invariant, f => f - .AddTags(401, "nestedTags1") - .AddTags(402, "nestedTags2", new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema }) - .AddString(403, "string")); - } - - [Fact] - public async Task Should_normalize_tags_with_old_data() - { - var newData = GenerateData("n_raw"); - var oldData = GenerateData("o_raw"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("n_raw2_1", "n_raw2_2", "n_raw4"), - A>.That.Is("o_raw2_1", "o_raw2_2", "o_raw4"), - default)) - .Returns(new Dictionary - { - ["n_raw2_2"] = "id2_2", - ["n_raw2_1"] = "id2_1", - ["n_raw4"] = "id4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, oldData); - - Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData)); - } - - [Fact] - public async Task Should_normalize_tags_without_old_data() - { - var newData = GenerateData("name"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("name2_1", "name2_2", "name4"), - A>.That.IsEmpty(), - default)) - .Returns(new Dictionary - { - ["name2_2"] = "id2_2", - ["name2_1"] = "id2_1", - ["name4"] = "id4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, null); - - Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData)); - } - - [Fact] - public async Task Should_denormalize_tags() - { - var newData = GenerateData("id"); - - A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), - A>.That.Is("id2_1", "id2_2", "id4"), - A>.That.IsEmpty(), - default)) - .Returns(new Dictionary - { - ["id2_2"] = "name2_2", - ["id2_1"] = "name2_1", - ["id4"] = "name4" - }); - - await tagService.NormalizeAsync(appId, schemaId, schema, newData, null); - - Assert.Equal(JsonValue.Array("name2_1", "name2_2"), newData["tags2"]!["iv"]); - Assert.Equal(JsonValue.Array("name4"), GetNestedTags(newData)); - } - - private static JsonValue GetNestedTags(ContentData newData) - { - var arrayValue = newData["array"]!["iv"].AsArray; - var arrayItem = arrayValue[0].AsObject; - - return arrayItem["nestedTags2"]; - } - - private static ContentData GenerateData(string prefix) - { - return new ContentData() - .AddField("tags1", - new ContentFieldData() - .AddInvariant(JsonValue.Array($"{prefix}1"))) - .AddField("tags2", - new ContentFieldData() - .AddInvariant(JsonValue.Array($"{prefix}2_1", $"{prefix}2_2"))) - .AddField("string", - new ContentFieldData() - .AddInvariant($"{prefix}stringValue")) - .AddField("array", - new ContentFieldData() - .AddInvariant( - JsonValue.Array( - new JsonObject() - .Add("nestedTags1", JsonValue.Array($"{prefix}3")) - .Add("nestedTags2", JsonValue.Array($"{prefix}4")) - .Add("string", $"{prefix}nestedStringValue")))); - } - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs index be9d7dfa4..c2dce3746 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs @@ -8,9 +8,11 @@ using FakeItEasy; using FluentAssertions; using NodaTime; +using Squidex.Domain.Apps.Core.Tags; using Squidex.Domain.Apps.Events.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.EventSourcing; +using Squidex.Infrastructure.States; using Squidex.Infrastructure.UsageTracking; using Xunit; @@ -18,13 +20,20 @@ namespace Squidex.Domain.Apps.Entities.Assets { public class AssetUsageTrackerTests { + private readonly IAssetLoader assetLoader = A.Fake(); + private readonly ISnapshotStore store = A.Fake>(); + private readonly ITagService tagService = A.Fake(); private readonly IUsageTracker usageTracker = A.Fake(); private readonly NamedId appId = NamedId.Of(DomainId.NewGuid(), "my-app"); + private readonly DomainId assetId = DomainId.NewGuid(); + private readonly DomainId assetKey; private readonly AssetUsageTracker sut; public AssetUsageTrackerTests() { - sut = new AssetUsageTracker(usageTracker); + assetKey = DomainId.Combine(appId, assetId); + + sut = new AssetUsageTracker(usageTracker, assetLoader, tagService, store); } [Fact] @@ -130,7 +139,7 @@ namespace Squidex.Domain.Apps.Entities.Assets @event.AppId = appId; var envelope = - Envelope.Create(@event) + Envelope.Create(@event) .SetTimestamp(Instant.FromDateTimeUtc(date)); Counters? countersSummary = null; @@ -142,7 +151,7 @@ namespace Squidex.Domain.Apps.Entities.Assets A.CallTo(() => usageTracker.TrackAsync(date, $"{appId.Id}_Assets", null, A._, default)) .Invokes(x => countersDate = x.GetArgument(3)); - await sut.On(envelope); + await sut.On(new[] { envelope }); var expected = new Counters { @@ -153,5 +162,294 @@ namespace Squidex.Domain.Apps.Entities.Assets countersSummary.Should().BeEquivalentTo(expected); countersDate.Should().BeEquivalentTo(expected); } + + [Fact] + public async Task Should_write_tags_when_asset_created() + { + var @event = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var envelope = + Envelope.Create(@event) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 1, + ["tag2"] = 1 + }); + } + + [Fact] + public async Task Should_group_tags_by_app() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1, envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 1 + }); + + A.CallTo(() => store.WriteManyAsync(A>>._, default)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_annotate() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetAnnotated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1, envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 0, + ["tag2"] = 1, + ["tag3"] = 1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_annotate_from_other_batch() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetAnnotated + { + AppId = appId, + Tags = new HashSet + { + "tag2", + "tag3" + }, + AssetId = assetId + }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope1 }); + await sut.On(new[] { envelope2 }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = 0, + ["tag3"] = 1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_previous_event_on_delete() + { + var @event1 = new AssetCreated + { + AppId = appId, + Tags = new HashSet + { + "tag1", + "tag2" + }, + AssetId = assetId + }; + + var @event2 = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope1 = + Envelope.Create(@event1) + .SetAggregateId(assetKey); + + var envelope2 = + Envelope.Create(@event2) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { Envelope.Create(@event1), Envelope.Create(@event2) }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = 0, + ["tag2"] = 0 + }); + } + + [Fact] + public async Task Should_merge_tags_with_stored_state_if_previous_event_not_in_cached() + { + var state = new AssetUsageTracker.State + { + Tags = new HashSet + { + "tag1", + "tag2" + } + }; + + A.CallTo(() => store.ReadAsync(assetKey, default)) + .Returns(new SnapshotResult(assetKey, state, 0)); + + var @event = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope = + Envelope.Create(@event) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = -1 + }); + } + + [Fact] + public async Task Should_merge_tags_with_asset_if_previous_tags_not_in_store() + { + IAssetEntity asset = new AssetEntity + { + Tags = new HashSet + { + "tag1", + "tag2" + } + }; + + A.CallTo(() => assetLoader.GetAsync(appId.Id, assetId, 41, default)) + .Returns(asset); + + var @event = new AssetDeleted { AppId = appId, AssetId = assetId }; + + var envelope = + Envelope.Create(@event) + .SetEventStreamNumber(42) + .SetAggregateId(assetKey); + + Dictionary? update = null; + + A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A>._, default)) + .Invokes(x => { update = x.GetArgument>(2); }); + + await sut.On(new[] { envelope }); + + update.Should().BeEquivalentTo(new Dictionary + { + ["tag1"] = -1, + ["tag2"] = -1 + }); + } } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs index 29e3e3c45..639954e61 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs @@ -63,7 +63,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject A.CallTo(() => assetQuery.FindAssetFolderAsync(AppId, parentId, A._)) .Returns(new List { A.Fake() }); - A.CallTo(() => tagService.NormalizeTagsAsync(AppId, TagGroups.Assets, A>._, A>._, default)) + A.CallTo(() => tagService.GetTagIdsAsync(AppId, TagGroups.Assets, A>._, default)) .ReturnsLazily(x => Task.FromResult(x.GetArgument>(2)?.ToDictionary(x => x) ?? new Dictionary())); var log = A.Fake>(); diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs index 1948119ab..54824a2b7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs @@ -80,7 +80,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries AppId = appId }; - A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2"), ct)) + A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2"), ct)) .Returns(new Dictionary { ["id1"] = "name1", @@ -158,7 +158,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries AppId = appId }; - A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2", "id3"), ct)) + A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A>.That.Is("id1", "id2", "id3"), ct)) .Returns(new Dictionary { ["id1"] = "name1", diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs index 6c81b6fb6..f0a9655f2 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs @@ -39,8 +39,9 @@ namespace Squidex.Domain.Apps.Entities.Tags [Fact] public async Task Should_delete_and_reset_state_if_cleaning() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct); + await sut.ClearAsync(appId, group, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); @@ -52,69 +53,76 @@ namespace Squidex.Domain.Apps.Entities.Tags } [Fact] - public async Task Should_rename_tag() + public async Task Should_unset_count_on_full_clear() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct); + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); - // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new"), null, ct); + // Clear is called by the event consumer to fill the counts again, therefore we do not delete other things. + await sut.ClearAsync(ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["tag1_new"] = 4 + ["tag1"] = 0, + ["tag2"] = 0 }, allTags); + + A.CallTo(() => state.Persistence.DeleteAsync(ct)) + .MustNotHaveHappened(); } [Fact] - public async Task Should_rename_tag_twice() + public async Task Should_rename_tag() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct); + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - // Rename again. - await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1_new2", ct); + await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct); // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new2"), null, ct); + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1_new"), ct); var allTags = await sut.GetTagsAsync(appId, group, ct); - Assert.Equal(new Dictionary - { - ["tag1_new2"] = 5 - }, allTags); + Assert.Equal(ids_0.Values, ids_1.Values); + } + + [Fact] + public async Task Should_rename_tag_twice() + { + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); + + await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct); + await sut.RenameTagAsync(appId, group, "tag2", "tag3", ct); + + // Forward the old name to the new name. + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2"), ct); + var ids_2 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag3"), ct); + + // Assert.Equal(ids_0.Values, ids_1.Values); + Assert.Equal(ids_0.Values, ids_2.Values); } [Fact] public async Task Should_rename_tag_back() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); + var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct); + await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct); // Rename back. - await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1", ct); + await sut.RenameTagAsync(appId, group, "tag2", "tag1", ct); // Forward the old name to the new name. - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct); + var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct); - var allTags = await sut.GetTagsAsync(appId, group, ct); - - Assert.Equal(new Dictionary - { - ["tag1"] = 3 - }, allTags); + Assert.Equal(ids_0.Values, ids_1.Values); } [Fact] @@ -124,9 +132,9 @@ namespace Squidex.Domain.Apps.Entities.Tags { Tags = new Dictionary { - ["id1"] = new Tag { Name = "name1", Count = 1 }, - ["id2"] = new Tag { Name = "name2", Count = 2 }, - ["id3"] = new Tag { Name = "name3", Count = 6 } + ["id1"] = new Tag { Name = "tag1", Count = 1 }, + ["id2"] = new Tag { Name = "tag2", Count = 2 }, + ["id3"] = new Tag { Name = "tag3", Count = 6 } } }; @@ -136,9 +144,9 @@ namespace Squidex.Domain.Apps.Entities.Tags Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 2, - ["name3"] = 6 + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 6 }, allTags); var export = await sut.GetExportableTagsAsync(appId, group, ct); @@ -147,65 +155,114 @@ namespace Squidex.Domain.Apps.Entities.Tags } [Fact] - public async Task Should_add_tags() + public async Task Should_add_tag_but_not_count_tags() { - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); + await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 2, - ["name3"] = 1 + ["tag1"] = 0, + ["tag2"] = 0, + ["tag3"] = 0 }, allTags); } [Fact] - public async Task Should_not_add_tags_if_already_added() + public async Task Should_add_and_increment_tags() { - var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2", "name3"), new HashSet(result1.Values), ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag2"]] = 1, + [ids["tag3"]] = 1 + }, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name1"] = 1, - ["name2"] = 1, - ["name3"] = 1 + ["tag1"] = 1, + ["tag2"] = 2, + ["tag3"] = 1 }, allTags); } [Fact] - public async Task Should_remove_tags() + public async Task Should_add_and_decrement_tags() { - var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); - var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct); + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct); - // Tags from the first normalization are decreased and removed if count reaches zero. - await sut.NormalizeTagsAsync(appId, group, null, new HashSet(result1.Values), ct); + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag1"]] = 1, + [ids["tag2"]] = 1 + }, ct); + + await sut.UpdateAsync(appId, group, new Dictionary + { + [ids["tag2"]] = -2, + [ids["tag3"]] = -2 + }, ct); var allTags = await sut.GetTagsAsync(appId, group, ct); Assert.Equal(new Dictionary { - ["name2"] = 1, - ["name3"] = 1 + ["tag1"] = 1, + ["tag2"] = 0, + ["tag3"] = 0 }, allTags); } + [Fact] + public async Task Should_not_update_non_existing_tags() + { + // We have no names for these IDs so we cannot update it. + await sut.UpdateAsync(appId, group, new Dictionary + { + ["id1"] = 1, + ["id2"] = 1 + }, ct); + + var allTags = await sut.GetTagsAsync(appId, group, ct); + + Assert.Empty(allTags); + } + [Fact] public async Task Should_resolve_tag_names() { // Get IDs from names. - var tagIds = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct); + var tagIds = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); // Get names from IDs (reverse operation). - var tagNames = await sut.GetTagIdsAsync(appId, group, HashSet.Of("name1", "name2", "invalid1"), ct); + var tagNames = await sut.GetTagNamesAsync(appId, group, tagIds.Values.ToHashSet(), ct); + + Assert.Equal(tagIds.Keys.ToArray(), tagNames.Values.ToArray()); + } + + [Fact] + public async Task Should_get_exportable_tags() + { + var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct); + + var allTags = await sut.GetExportableTagsAsync(appId, group, ct); - Assert.Equal(tagIds, tagNames); + allTags.Tags.Should().BeEquivalentTo(new Dictionary + { + [ids["tag1"]] = new Tag { Name = "tag1", Count = 0 }, + [ids["tag2"]] = new Tag { Name = "tag2", Count = 0 }, + }); } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs b/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs index de31dd4e0..5c499643c 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs @@ -129,24 +129,6 @@ namespace Squidex.Infrastructure Assert.Equal(24, valueDictionary[12]); } - [Fact] - public void GetOrNew_should_return_value_if_key_exists() - { - var list = new List(); - listDictionary[12] = list; - - Assert.Equal(list, listDictionary.GetOrNew(12)); - } - - [Fact] - public void GetOrNew_should_return_default_but_not_add_it_if_key_not_exists() - { - var list = new List(); - - Assert.Equal(list, listDictionary.GetOrNew(12)); - Assert.False(listDictionary.ContainsKey(12)); - } - [Fact] public void GetOrAddNew_should_return_value_if_key_exists() { diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs index 7cd1f421a..c4a76a177 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs @@ -18,16 +18,26 @@ namespace Squidex.Infrastructure.EventSourcing.Consume { private readonly IPersistenceFactory persistenceFactory = A.Fake>(); private readonly IMessageBus messaging = A.Fake(); - private readonly string consumerName = Guid.NewGuid().ToString(); + private readonly string consumerName1 = Guid.NewGuid().ToString(); + private readonly string consumerName2 = Guid.NewGuid().ToString(); private readonly EventConsumerManager sut; public EventConsumerManagerTests() { - sut = new EventConsumerManager(persistenceFactory, messaging); + var consumer1 = A.Fake(); + var consumer2 = A.Fake(); + + A.CallTo(() => consumer1.Name) + .Returns(consumerName1); + + A.CallTo(() => consumer2.Name) + .Returns(consumerName2); + + sut = new EventConsumerManager(persistenceFactory, new[] { consumer1, consumer2 }, messaging); } [Fact] - public async Task Should_get_states_from_store() + public async Task Should_get_states_from_store_without_old_consumer() { var snapshotStore = A.Fake>(); @@ -37,12 +47,17 @@ namespace Squidex.Infrastructure.EventSourcing.Consume A.CallTo(() => snapshotStore.ReadAllAsync(default)) .Returns(new List> { - new SnapshotResult(DomainId.Create("consumer1"), + new SnapshotResult(DomainId.Create(consumerName1), new EventConsumerState { Position = "1" }, 1), - new SnapshotResult(DomainId.Create("consumer2"), + new SnapshotResult(DomainId.Create(consumerName2), + new EventConsumerState + { + Position = "2" + }, 2), + new SnapshotResult(DomainId.Create("oldConsumer"), new EventConsumerState { Position = "2" @@ -54,15 +69,21 @@ namespace Squidex.Infrastructure.EventSourcing.Consume result.Should().BeEquivalentTo( new List { - new EventConsumerInfo { Name = "consumer1", Position = "1" }, - new EventConsumerInfo { Name = "consumer2", Position = "2" } + new EventConsumerInfo { Name = consumerName1, Position = "1" }, + new EventConsumerInfo { Name = consumerName2, Position = "2" } }); } + [Fact] + public async Task Should_throw_exception_when_calling_old_consumer() + { + await Assert.ThrowsAsync(() => sut.StartAsync("oldConsumer", default)); + } + [Fact] public async Task Should_publish_event_on_start() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(DomainId.Create(consumerName1), persistenceFactory) { Snapshot = new EventConsumerState { @@ -70,9 +91,9 @@ namespace Squidex.Infrastructure.EventSourcing.Consume } }; - var response = await sut.StartAsync(consumerName, default); + var response = await sut.StartAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); @@ -81,7 +102,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume [Fact] public async Task Should_publish_event_on_stop() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(DomainId.Create(consumerName1), persistenceFactory) { Snapshot = new EventConsumerState { @@ -89,9 +110,9 @@ namespace Squidex.Infrastructure.EventSourcing.Consume } }; - var response = await sut.StopAsync(consumerName, default); + var response = await sut.StopAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); @@ -100,7 +121,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume [Fact] public async Task Should_publish_event_on_reset() { - var testState = new TestState(DomainId.Create(consumerName), persistenceFactory) + var testState = new TestState(DomainId.Create(consumerName1), persistenceFactory) { Snapshot = new EventConsumerState { @@ -108,9 +129,9 @@ namespace Squidex.Infrastructure.EventSourcing.Consume } }; - var response = await sut.ResetAsync(consumerName, default); + var response = await sut.ResetAsync(consumerName1, default); - A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName), null, default)) + A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName1), null, default)) .MustHaveHappened(); Assert.Equal("42", response.Position); diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs new file mode 100644 index 000000000..380bf48b5 --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs @@ -0,0 +1,208 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using FakeItEasy; +using Squidex.Infrastructure.TestHelpers; +using Xunit; + +namespace Squidex.Infrastructure.States +{ + public class SimpleStateTests + { + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private readonly CancellationToken ct; + private readonly TestState testState = new TestState(DomainId.NewGuid()); + private readonly SimpleState sut; + + public SimpleStateTests() + { + ct = cts.Token; + + sut = new SimpleState(testState.PersistenceFactory, GetType(), testState.Id); + } + + [Fact] + public void Should_init_with_base_data() + { + Assert.Equal(-1, sut.Version); + Assert.NotNull(sut.Value); + } + + [Fact] + public async Task Should_get_state_from_persistence_on_load() + { + testState.Version = 42; + testState.Snapshot = new MyDomainState { Value = 50 }; + + await sut.LoadAsync(ct); + + Assert.Equal(42, sut.Version); + Assert.Equal(50, sut.Value.Value); + + A.CallTo(() => testState.Persistence.ReadAsync(-2, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_delete_when_clearing() + { + await sut.ClearAsync(ct); + + A.CallTo(() => testState.Persistence.DeleteAsync(ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_invoke_persistence_when_writing_state() + { + await sut.WriteAsync(ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_load_once_on_update() + { + await sut.UpdateAsync(x => true, ct: ct); + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.ReadAsync(-2, ct)) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_write_state_on_update_when_callback_returns_true() + { + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_not_write_state_on_update_when_callback_returns_false() + { + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_write_state_on_update_and_return_when_callback_returns_true() + { + var result = await sut.UpdateAsync(x => (true, 42), ct: ct); + + Assert.Equal(42, result); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_not_write_state_on_update_and_return_when_callback_returns_false() + { + var result = await sut.UpdateAsync(x => (false, 42), ct: ct); + + Assert.Equal(42, result); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustNotHaveHappened(); + } + + [Fact] + public async Task Should_retry_update_when_failed_with_inconsistency_issue() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5); + + await sut.UpdateAsync(x => true, ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + } + + [Fact] + public async Task Should_give_up_update_after_too_many_inconsistency_issues() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => true, ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + } + + [Fact] + public async Task Should_not_retry_update_with_other_exception() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InvalidOperationException()); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => true, ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + } + + [Fact] + public async Task Should_retry_update_and_return_when_failed_with_inconsistency_issue() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5); + + await sut.UpdateAsync(x => (true, 42), ct: ct); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 6); + } + + [Fact] + public async Task Should_give_up_update_and_return_after_too_many_inconsistency_issues() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => (true, 42), ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 20); + } + + [Fact] + public async Task Should_not_retry_update_and_return_with_other_exception() + { + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .Throws(new InvalidOperationException()); + + await Assert.ThrowsAsync(() => sut.UpdateAsync(x => (true, 42), ct: ct)); + + A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + + A.CallTo(() => testState.Persistence.ReadAsync(A._, ct)) + .MustHaveHappenedANumberOfTimesMatching(x => x == 1); + } + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs index 9ab205046..ba5124d89 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs @@ -14,9 +14,12 @@ namespace Squidex.Infrastructure.TestHelpers public sealed class TestState where T : class, new() { private readonly List> events = new List>(); + private readonly ISnapshotStore snapshotStore = A.Fake>(); private HandleSnapshot? handleSnapshot; private HandleEvent? handleEvent; + public DomainId Id { get; } + public IPersistenceFactory PersistenceFactory { get; } public IPersistence Persistence { get; } = A.Fake>(); @@ -42,8 +45,22 @@ namespace Squidex.Infrastructure.TestHelpers public TestState(DomainId id, IPersistenceFactory? persistenceFactory = null) { + Id = id; + PersistenceFactory = persistenceFactory ?? A.Fake>(); + A.CallTo(() => PersistenceFactory.Snapshots) + .Returns(snapshotStore); + + A.CallTo(() => Persistence.Version) + .ReturnsLazily(() => Version); + + A.CallTo(() => snapshotStore.ReadAllAsync(A._)) + .ReturnsLazily(() => new List> + { + new SnapshotResult(id, Snapshot, Version, true) + }.ToAsyncEnumerable()); + A.CallTo(() => PersistenceFactory.WithEventSourcing(A._, id, A._)) .Invokes(x => { diff --git a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs index 3b3cd2a0e..50f0e09cd 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs @@ -65,6 +65,21 @@ namespace Squidex.Infrastructure.UsageTracking await Assert.ThrowsAsync(() => sut.GetAsync(key, date, date, null, ct)); } + [Fact] + public void Should_provide_fallback_category() + { + Assert.Equal("*", sut.FallbackCategory); + } + + [Fact] + public async Task Should_forward_delete_prefix_call() + { + await sut.DeleteByKeyPatternAsync("pattern", ct); + + A.CallTo(() => usageStore.DeleteByKeyPatternAsync("pattern", ct)) + .MustHaveHappened(); + } + [Fact] public async Task Should_forward_delete_call() { diff --git a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs index 47474fff5..8f104e3f4 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs @@ -30,6 +30,24 @@ namespace Squidex.Infrastructure.UsageTracking sut = new CachingUsageTracker(inner, cache); } + [Fact] + public void Should_forward_fallback_category() + { + A.CallTo(() => inner.FallbackCategory) + .Returns("*"); + + Assert.Equal("*", sut.FallbackCategory); + } + + [Fact] + public async Task Should_forward_delete_prefix_call() + { + await sut.DeleteByKeyPatternAsync("pattern", ct); + + A.CallTo(() => inner.DeleteByKeyPatternAsync("pattern", ct)) + .MustHaveHappened(); + } + [Fact] public async Task Should_forward_delete_call() {