Consistency improvements (#906)

* Improve consistency for content repositories.

* Temp

* Improved consistency for assets.

* Improve tests

* Fixes to app plan management

* Remove referrer

* Fixes cancellation token.

* Fix method naming.

* Remove test loop.

* Another test to improve tests

* Small improvement for tests.

* Perhaps the potential to get better error messages.

* Revert batch change.

* Test logging.

* Fix JSON

* Better polling subscription.

* Store recent events.

* Test output.

* Test

* Another test

* Fix asset deleter.

* Build fix.

* Revert "Store recent events."

This reverts commit 127491d61d.

backend/src/Squidex.Domain.Apps.Core.Operations/Tags/ITagService.cs

@@ -14,10 +14,10 @@ namespace Squidex.Domain.Apps.Core.Tags
         Task<Dictionary<string, string>> GetTagIdsAsync(DomainId id, string group, HashSet<string> names,
             CancellationToken ct = default);
 
-        Task<Dictionary<string, string>> NormalizeTagsAsync(DomainId id, string group, HashSet<string>? names, HashSet<string>? ids,
+        Task<Dictionary<string, string>> GetTagNamesAsync(DomainId id, string group, HashSet<string> ids,
             CancellationToken ct = default);
 
-        Task<Dictionary<string, string>> DenormalizeTagsAsync(DomainId id, string group, HashSet<string> ids,
+        Task UpdateAsync(DomainId id, string group, Dictionary<string, int> updates,
             CancellationToken ct = default);
 
         Task<TagsSet> GetTagsAsync(DomainId id, string group,
@@ -34,5 +34,8 @@ namespace Squidex.Domain.Apps.Core.Tags
 
         Task ClearAsync(DomainId id, string group,
             CancellationToken ct = default);
+
+        Task ClearAsync(
+            CancellationToken ct = default);
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Tags/Tag.cs

@@ -7,10 +7,10 @@
 
 namespace Squidex.Domain.Apps.Core.Tags
 {
-    public sealed class Tag
+    public sealed record Tag
     {
         public string Name { get; set; }
 
-        public int Count { get; set; } = 1;
+        public int Count { get; set; }
     }
 }

backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagNormalizer.cs

@@ -1,151 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using Squidex.Domain.Apps.Core.Contents;
-using Squidex.Domain.Apps.Core.Schemas;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Json.Objects;
-
-namespace Squidex.Domain.Apps.Core.Tags
-{
-    public static class TagNormalizer
-    {
-        public static async Task NormalizeAsync(this ITagService tagService, DomainId appId, DomainId schemaId, Schema schema, ContentData newData, ContentData? oldData)
-        {
-            Guard.NotNull(tagService);
-            Guard.NotNull(schema);
-            Guard.NotNull(newData);
-
-            var newValues = new HashSet<string>();
-            var newArrays = new List<JsonValue>();
-
-            var oldValues = new HashSet<string>();
-            var oldArrays = new List<JsonValue>();
-
-            GetValues(schema, newValues, newArrays, newData);
-
-            if (oldData != null)
-            {
-                GetValues(schema, oldValues, oldArrays, oldData);
-            }
-
-            if (newValues.Count > 0)
-            {
-                var normalized = await tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), newValues, oldValues);
-
-                foreach (var source in newArrays)
-                {
-                    var array = source.AsArray;
-
-                    for (var i = 0; i < array.Count; i++)
-                    {
-                        if (normalized.TryGetValue(array[i].ToString(), out var result))
-                        {
-                            array[i] = result;
-                        }
-                    }
-                }
-            }
-        }
-
-        public static async Task DenormalizeAsync(this ITagService tagService, DomainId appId, DomainId schemaId, Schema schema, params ContentData[] datas)
-        {
-            Guard.NotNull(tagService);
-            Guard.NotNull(schema);
-
-            var tagsValues = new HashSet<string>();
-            var tagsArrays = new List<JsonValue>();
-
-            GetValues(schema, tagsValues, tagsArrays, datas);
-
-            if (tagsValues.Count > 0)
-            {
-                var denormalized = await tagService.DenormalizeTagsAsync(appId, TagGroups.Schemas(schemaId), tagsValues);
-
-                foreach (var source in tagsArrays)
-                {
-                    var array = source.AsArray;
-
-                    for (var i = 0; i < array.Count; i++)
-                    {
-                        if (denormalized.TryGetValue(array[i].ToString(), out var result))
-                        {
-                            array[i] = result;
-                        }
-                    }
-                }
-            }
-        }
-
-        private static void GetValues(Schema schema, HashSet<string> values, List<JsonValue> arrays, params ContentData[] datas)
-        {
-            foreach (var field in schema.Fields)
-            {
-                if (field is IField<TagsFieldProperties> tags && tags.Properties.Normalization == TagsFieldNormalization.Schema)
-                {
-                    foreach (var data in datas)
-                    {
-                        if (data.TryGetValue(field.Name, out var fieldData) && fieldData != null)
-                        {
-                            foreach (var partition in fieldData)
-                            {
-                                ExtractTags(partition.Value, values, arrays);
-                            }
-                        }
-                    }
-                }
-                else if (field is IArrayField arrayField)
-                {
-                    foreach (var nestedField in arrayField.Fields)
-                    {
-                        if (nestedField is IField<TagsFieldProperties> nestedTags && nestedTags.Properties.Normalization == TagsFieldNormalization.Schema)
-                        {
-                            foreach (var data in datas)
-                            {
-                                if (data.TryGetValue(field.Name, out var fieldData) && fieldData != null)
-                                {
-                                    foreach (var partition in fieldData)
-                                    {
-                                        if (partition.Value.Value is JsonArray a)
-                                        {
-                                            foreach (var value in a)
-                                            {
-                                                if (value.Value is JsonObject o)
-                                                {
-                                                    if (o.TryGetValue(nestedField.Name, out var nestedValue))
-                                                    {
-                                                        ExtractTags(nestedValue, values, arrays);
-                                                    }
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        private static void ExtractTags(JsonValue value, ISet<string> values, ICollection<JsonValue> arrays)
-        {
-            if (value.Value is JsonArray a)
-            {
-                foreach (var item in a)
-                {
-                    if (item.Value is string s)
-                    {
-                        values.Add(s);
-                    }
-                }
-
-                arrays.Add(value);
-            }
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Core.Operations/Tags/TagsExport.cs

@@ -9,27 +9,8 @@ namespace Squidex.Domain.Apps.Core.Tags
 {
     public class TagsExport
     {
-        public Dictionary<string, Tag>? Tags { get; set; }
+        public Dictionary<string, Tag> Tags { get; set; } = new Dictionary<string, Tag>();
 
-        public Dictionary<string, string>? Alias { get; set; }
+        public Dictionary<string, string> Alias { get; set; } = new Dictionary<string, string>();
-
-        public TagsExport Clone()
-        {
-            var alias = (Dictionary<string, string>?)null;
-
-            if (Alias != null)
-            {
-                alias = new Dictionary<string, string>(Alias);
-            }
-
-            var tags = (Dictionary<string, Tag>?)null;
-
-            if (Tags != null)
-            {
-                tags = new Dictionary<string, Tag>(Tags);
-            }
-
-            return new TagsExport { Alias = alias, Tags = tags };
-        }
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs

@@ -55,9 +55,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
         {
             using (Telemetry.Activities.StartActivity("MongoAssetFolderRepository/WriteAsync"))
             {
-                var entity = MongoAssetFolderEntity.Create(job);
+                var entityJob = job.As(MongoAssetFolderEntity.Create(job));
 
-                await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct);
+                await Collection.UpsertVersionedAsync(entityJob, ct);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs

@@ -260,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
                 Filter.Gt(x => x.Id, DomainId.Create(string.Empty)),
                 Filter.Eq(x => x.IndexedAppId, appId),
                 Filter.Ne(x => x.IsDeleted, true),
-                Filter.Ne(x => x.ParentId, parentId));
+                Filter.Eq(x => x.ParentId, parentId));
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs

@@ -55,9 +55,9 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
         {
             using (Telemetry.Activities.StartActivity("MongoAssetRepository/WriteAsync"))
             {
-                var entity = MongoAssetEntity.Create(job);
+                var entityJob = job.As(MongoAssetEntity.Create(job));
 
-                await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, entity, ct);
+                await Collection.UpsertVersionedAsync(entityJob, ct);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@@ -16,6 +16,7 @@ using Squidex.Domain.Apps.Entities.Schemas;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
 using Squidex.Infrastructure.Queries;
+using Squidex.Infrastructure.States;
 using Squidex.Infrastructure.Translations;
 
 #pragma warning disable IDE0060 // Remove unused parameter
@@ -251,25 +252,47 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
             return Collection.Find(FindAll).ToAsyncEnumerable(ct);
         }
 
-        public async Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity value,
+        public async Task UpsertAsync(SnapshotWriteJob<MongoContentEntity> job,
             CancellationToken ct = default)
         {
             if (queryInDedicatedCollection != null)
             {
-                await queryInDedicatedCollection.UpsertVersionedAsync(documentId, oldVersion, value, default);
+                await queryInDedicatedCollection.UpsertAsync(job, ct);
             }
 
-            await Collection.UpsertVersionedAsync(documentId, oldVersion, value.Version, value, default);
+            await Collection.ReplaceOneAsync(Filter.Eq(x => x.DocumentId, job.Key), job.Value, UpsertReplace, ct);
+        }
+
+        public async Task UpsertVersionedAsync(IClientSessionHandle session, SnapshotWriteJob<MongoContentEntity> job,
+            CancellationToken ct = default)
+        {
+            if (queryInDedicatedCollection != null)
+            {
+                await queryInDedicatedCollection.UpsertVersionedAsync(session, job, ct);
+            }
+
+            await Collection.UpsertVersionedAsync(session, job, ct);
         }
 
         public async Task RemoveAsync(DomainId key,
             CancellationToken ct = default)
         {
-            var previous = await Collection.FindOneAndDeleteAsync(x => x.DocumentId == key, null, default);
+            var previous = await Collection.FindOneAndDeleteAsync(x => x.DocumentId == key, null, ct);
+
+            if (queryInDedicatedCollection != null && previous != null)
+            {
+                await queryInDedicatedCollection.RemoveAsync(previous, ct);
+            }
+        }
+
+        public async Task RemoveAsync(IClientSessionHandle session, DomainId key,
+            CancellationToken ct = default)
+        {
+            var previous = await Collection.FindOneAndDeleteAsync(session, x => x.DocumentId == key, null, ct);
 
             if (queryInDedicatedCollection != null && previous != null)
             {
-                await queryInDedicatedCollection.RemoveAsync(previous, default);
+                await queryInDedicatedCollection.RemoveAsync(session, previous, ct);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs

@@ -7,6 +7,7 @@
 
 using Microsoft.Extensions.Options;
 using MongoDB.Driver;
+using MongoDB.Driver.Core.Clusters;
 using NodaTime;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Apps;
@@ -24,8 +25,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
     {
         private readonly MongoContentCollection collectionComplete;
         private readonly MongoContentCollection collectionPublished;
+        private readonly IMongoDatabase database;
         private readonly IAppProvider appProvider;
 
+        public bool CanUseTransactions { get; private set; }
+
         static MongoContentRepository()
         {
             BsonStringSerializer<Status>.Register();
@@ -34,6 +38,8 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider,
             IOptions<ContentOptions> options)
         {
+            this.database = database;
+
             collectionComplete =
                 new MongoContentCollection("States_Contents_All3", database,
                     ReadPreference.Primary, options.Value.OptimizeForSelfHosting);
@@ -50,6 +56,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             await collectionComplete.InitializeAsync(ct);
             await collectionPublished.InitializeAsync(ct);
+
+            var clusterVersion = await database.GetMajorVersionAsync(ct);
+            var clusteredAsReplica = database.Client.Cluster.Description.Type == ClusterType.ReplicaSet;
+
+            CanUseTransactions = clusteredAsReplica && clusterVersion >= 4;
         }
 
         public IAsyncEnumerable<IContentEntity> StreamAll(DomainId appId, HashSet<DomainId>? schemaIds,

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs

@@ -33,6 +33,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
                 var existing =
                     await collectionComplete.FindAsync(key, ct);
 
+                // Support for all versions, where we do not have full snapshots in the collection.
                 if (existing?.IsSnapshot == true)
                 {
                     return new SnapshotResult<ContentDomainObject.State>(existing.DocumentId, existing.ToState(), existing.Version);
@@ -67,6 +68,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             using (Telemetry.Activities.StartActivity("MongoContentRepository/RemoveAsync"))
             {
+                // Some data is corrupt and might throw an exception if we do not ignore it.
                 if (key == DomainId.Empty)
                 {
                     return;
@@ -83,14 +85,33 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteAsync"))
             {
+                // Some data is corrupt and might throw an exception if we do not ignore it.
                 if (!IsValid(job.Value))
                 {
                     return;
                 }
 
-                await Task.WhenAll(
+                if (!CanUseTransactions)
-                    UpsertFrontendAsync(job, ct),
+                {
-                    UpsertPublishedAsync(job, ct));
+                    // If transactions are not supported we update the documents without version checks,
+                    // otherwise we would not be able to recover from inconsistencies.
+                    await Task.WhenAll(
+                        UpsertCompleteAsync(job, default),
+                        UpsertPublishedAsync(job, default));
+                    return;
+                }
+
+                using (var session = await database.Client.StartSessionAsync(cancellationToken: ct))
+                {
+                    // Make an update with full transaction support to be more consistent.
+                    await session.WithTransactionAsync(async (session, ct) =>
+                    {
+                        await Task.WhenAll(
+                            UpsertVersionedCompleteAsync(session, job, ct),
+                            UpsertVersionedPublishedAsync(session, job, ct));
+                        return true;
+                    }, null, ct);
+                }
             }
         }
 
@@ -99,11 +120,11 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         {
             using (Telemetry.Activities.StartActivity("MongoContentRepository/WriteManyAsync"))
             {
-                var updates = new Dictionary<IMongoCollection<MongoContentEntity>, List<MongoContentEntity>>();
+                var collectionUpdates = new Dictionary<IMongoCollection<MongoContentEntity>, List<MongoContentEntity>>();
 
                 var add = new Action<IMongoCollection<MongoContentEntity>, MongoContentEntity>((collection, entity) =>
                 {
-                    updates.GetOrAddNew(collection).Add(entity);
+                    collectionUpdates.GetOrAddNew(collection).Add(entity);
                 });
 
                 foreach (var job in jobs)
@@ -123,7 +144,15 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
                     }
                 }
 
-                await Parallel.ForEachAsync(updates, ct, (update, ct) =>
+                var parallelOptions = new ParallelOptions
+                {
+                    CancellationToken = ct,
+                    // This is just an estimate, but we do not want ot have unlimited parallelism.
+                    MaxDegreeOfParallelism = 8
+                };
+
+                // Make one update per collection.
+                await Parallel.ForEachAsync(collectionUpdates, parallelOptions, (update, ct) =>
                 {
                     return new ValueTask(update.Key.InsertManyAsync(update.Value, InsertUnordered, ct));
                 });
@@ -131,38 +160,49 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
         }
 
         private async Task UpsertPublishedAsync(SnapshotWriteJob<ContentDomainObject.State> job,
-            CancellationToken ct = default)
+            CancellationToken ct)
         {
             if (ShouldWritePublished(job.Value))
             {
-                await UpsertPublishedContentAsync(job, ct);
+                var entityJob = job.As(await MongoContentEntity.CreatePublishedAsync(job, appProvider));
+
+                await collectionPublished.UpsertAsync(entityJob, ct);
             }
             else
             {
-                await DeletePublishedContentAsync(job.Value.UniqueId, ct);
+                await collectionPublished.RemoveAsync(job.Key, ct);
             }
         }
 
-        private Task DeletePublishedContentAsync(DomainId key,
+        private async Task UpsertVersionedPublishedAsync(IClientSessionHandle session, SnapshotWriteJob<ContentDomainObject.State> job,
-            CancellationToken ct = default)
+            CancellationToken ct)
         {
-            return collectionPublished.RemoveAsync(key, ct);
+            if (ShouldWritePublished(job.Value))
+            {
+                var entityJob = job.As(await MongoContentEntity.CreatePublishedAsync(job, appProvider));
+
+                await collectionPublished.UpsertVersionedAsync(session, entityJob, ct);
+            }
+            else
+            {
+                await collectionPublished.RemoveAsync(session, job.Key, ct);
+            }
         }
 
-        private async Task UpsertFrontendAsync(SnapshotWriteJob<ContentDomainObject.State> job,
+        private async Task UpsertCompleteAsync(SnapshotWriteJob<ContentDomainObject.State> job,
-            CancellationToken ct = default)
+            CancellationToken ct)
         {
-            var entity = await MongoContentEntity.CreateAsync(job, appProvider);
+            var entityJob = job.As(await MongoContentEntity.CreateAsync(job, appProvider));
 
-            await collectionComplete.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct);
+            await collectionComplete.UpsertAsync(entityJob, ct);
         }
 
-        private async Task UpsertPublishedContentAsync(SnapshotWriteJob<ContentDomainObject.State> job,
+        private async Task UpsertVersionedCompleteAsync(IClientSessionHandle session, SnapshotWriteJob<ContentDomainObject.State> job,
-            CancellationToken ct = default)
+            CancellationToken ct)
         {
-            var entity = await MongoContentEntity.CreatePublishedAsync(job, appProvider);
+            var entityJob = job.As(await MongoContentEntity.CreateAsync(job, appProvider));
 
-            await collectionPublished.UpsertVersionedAsync(entity.DocumentId, job.OldVersion, entity, ct);
+            await collectionComplete.UpsertVersionedAsync(session, entityJob, ct);
         }
 
         private static bool ShouldWritePublished(ContentDomainObject.State value)

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/Operations/QueryInDedicatedCollection.cs

@@ -15,6 +15,7 @@ using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
 using Squidex.Infrastructure.MongoDb.Queries;
 using Squidex.Infrastructure.Queries;
+using Squidex.Infrastructure.States;
 
 namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
 {
@@ -110,12 +111,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
             return ResultList.Create<IContentEntity>(contentTotal, contentEntities);
         }
 
-        public async Task UpsertVersionedAsync(DomainId documentId, long oldVersion, MongoContentEntity value,
+        public async Task UpsertAsync(SnapshotWriteJob<MongoContentEntity> job,
             CancellationToken ct = default)
         {
-            var collection = await GetCollectionAsync(value.AppId.Id, value.SchemaId.Id);
+            var collection = await GetCollectionAsync(job.Value.AppId.Id, job.Value.SchemaId.Id);
+
+            await collection.ReplaceOneAsync(Filter.Eq(x => x.DocumentId, job.Key), job.Value, UpsertReplace, ct);
+        }
+
+        public async Task UpsertVersionedAsync(IClientSessionHandle session, SnapshotWriteJob<MongoContentEntity> job,
+            CancellationToken ct = default)
+        {
+            var collection = await GetCollectionAsync(job.Value.AppId.Id, job.Value.SchemaId.Id);
 
-            await collection.UpsertVersionedAsync(documentId, oldVersion, value.Version, value, ct);
+            await collection.UpsertVersionedAsync(session, job, ct);
         }
 
         public async Task RemoveAsync(MongoContentEntity value,
@@ -126,6 +135,14 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations
             await collection.DeleteOneAsync(x => x.DocumentId == value.DocumentId, null, ct);
         }
 
+        public async Task RemoveAsync(IClientSessionHandle session, MongoContentEntity value,
+            CancellationToken ct = default)
+        {
+            var collection = await GetCollectionAsync(value.AppId.Id, value.SchemaId.Id);
+
+            await collection.DeleteOneAsync(session, x => x.DocumentId == value.DocumentId, null, ct);
+        }
+
         private static FilterDefinition<MongoContentEntity> BuildFilter(FilterNode<ClrValue>? filter)
         {
             var filters = new List<FilterDefinition<MongoContentEntity>>

backend/src/Squidex.Domain.Apps.Entities/Apps/AppUISettings.cs

@@ -20,9 +20,17 @@ namespace Squidex.Domain.Apps.Entities.Apps
         {
             public JsonObject Settings { get; set; } = new JsonObject();
 
-            public void Set(JsonObject settings)
+            public bool Set(JsonObject settings)
             {
-                Settings = settings;
+                var isChanged = false;
+
+                if (!Settings.Equals(settings))
+                {
+                    Settings = settings;
+                    isChanged = true;
+                }
+
+                return isChanged;
             }
 
             public bool Set(string path, JsonValue value)
@@ -134,7 +142,7 @@ namespace Squidex.Domain.Apps.Entities.Apps
         {
             var state = await GetStateAsync(appId, userId, ct);
 
-            await state.UpdateIfAsync(s => s.Remove(path), ct: ct);
+            await state.UpdateAsync(s => s.Remove(path), ct: ct);
         }
 
         public async Task SetAsync(DomainId appId, string? userId, string path, JsonValue value,
@@ -142,7 +150,7 @@ namespace Squidex.Domain.Apps.Entities.Apps
         {
             var state = await GetStateAsync(appId, userId, ct);
 
-            await state.UpdateIfAsync(s => s.Set(path, value), ct: ct);
+            await state.UpdateAsync(s => s.Set(path, value), ct: ct);
         }
 
         public async Task SetAsync(DomainId appId, string? userId, JsonObject settings,

backend/src/Squidex.Domain.Apps.Entities/Apps/Commands/ChangePlan.cs

@@ -12,7 +12,5 @@ namespace Squidex.Domain.Apps.Entities.Apps.Commands
         public bool FromCallback { get; set; }
 
         public string PlanId { get; set; }
-
-        public string Referer { get; set; }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Apps/DomainObject/AppDomainObject.cs

@@ -118,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                 case AssignContributor assignContributor:
                     return UpdateReturnAsync(assignContributor, async (c, ct) =>
                     {
-                        await GuardAppContributors.CanAssign(c, Snapshot, UserResolver(), GetPlan());
+                        await GuardAppContributors.CanAssign(c, Snapshot, Users(), GetPlan());
 
                         AssignContributor(c, !Snapshot.Contributors.ContainsKey(assignContributor.ContributorId));
 
@@ -255,46 +255,71 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                         return Snapshot;
                     }, ct);
 
-                case ChangePlan changePlan:
-                    return UpdateReturnAsync(changePlan, async (c, ct) =>
-                    {
-                        GuardApp.CanChangePlan(c, Snapshot, AppPlansProvider());
-
-                        if (c.FromCallback)
-                        {
-                            ChangePlan(c);
-
-                            return null;
-                        }
-                        else
-                        {
-                            var result = await AppPlanBillingManager().ChangePlanAsync(c.Actor.Identifier, Snapshot.NamedId(), c.PlanId, c.Referer, default);
-
-                            switch (result)
-                            {
-                                case PlanChangedResult:
-                                    ChangePlan(c);
-                                    break;
-                            }
-
-                            return result;
-                        }
-                    }, ct);
-
                 case DeleteApp delete:
                     return UpdateAsync(delete, async (c, ct) =>
                     {
-                        await AppPlanBillingManager().ChangePlanAsync(c.Actor.Identifier, Snapshot.NamedId(), null, null, ct);
+                        await Billing().UnsubscribeAsync(c.Actor.Identifier, Snapshot.NamedId(), default);
 
                         DeleteApp(c);
                     }, ct);
 
+                case ChangePlan changePlan:
+                    return ChangeBillingPlanAsync(changePlan, ct);
+
                 default:
                     ThrowHelper.NotSupportedException();
                     return default!;
             }
         }
 
+        private async Task<CommandResult> ChangeBillingPlanAsync(ChangePlan changePlan,
+            CancellationToken ct)
+        {
+            var userId = changePlan.Actor.Identifier;
+
+            var result = await UpdateReturnAsync(changePlan, async (c, ct) =>
+            {
+                GuardApp.CanChangePlan(c, Snapshot, Plans());
+
+                if (string.Equals(GetFreePlan()?.Id, c.PlanId, StringComparison.Ordinal))
+                {
+                    ResetPlan(c);
+
+                    return new PlanChangedResult(c.PlanId, true, null);
+                }
+
+                if (!c.FromCallback)
+                {
+                    var redirectUri = await Billing().MustRedirectToPortalAsync(userId, Snapshot.NamedId(), c.PlanId, ct);
+
+                    if (redirectUri != null)
+                    {
+                        return new PlanChangedResult(c.PlanId, false, redirectUri);
+                    }
+                }
+
+                ChangePlan(c);
+
+                return new PlanChangedResult(c.PlanId);
+            }, ct);
+
+            if (changePlan.FromCallback)
+            {
+                return result;
+            }
+
+            if (result.Payload is PlanChangedResult { Unsubscribed: true, RedirectUri: null })
+            {
+                await Billing().UnsubscribeAsync(userId, Snapshot.NamedId(), default);
+            }
+            else if (result.Payload is PlanChangedResult { RedirectUri: null })
+            {
+                await Billing().SubscribeAsync(userId, Snapshot.NamedId(), changePlan.PlanId, default);
+            }
+
+            return result;
+        }
+
         private void Create(CreateApp command)
         {
             var appId = NamedId.Of(command.AppId, command.Name);
@@ -321,14 +346,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
 
         private void ChangePlan(ChangePlan command)
         {
-            if (string.Equals(GetFreePlan()?.Id, command.PlanId, StringComparison.Ordinal))
+            Raise(command, new AppPlanChanged());
-            {
+        }
-                Raise(command, new AppPlanReset());
+
-            }
+        private void ResetPlan(ChangePlan command)
-            else
+        {
-            {
+            Raise(command, new AppPlanReset());
-                Raise(command, new AppPlanChanged());
-            }
         }
 
         private void Update(UpdateApp command)
@@ -455,29 +478,29 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
             return new AppSettingsUpdated { Settings = serviceProvider.GetRequiredService<InitialSettings>().Settings };
         }
 
-        private IAppPlansProvider AppPlansProvider()
+        private IAppPlansProvider Plans()
         {
             return serviceProvider.GetRequiredService<IAppPlansProvider>();
         }
 
-        private IAppPlanBillingManager AppPlanBillingManager()
+        private IAppPlanBillingManager Billing()
         {
             return serviceProvider.GetRequiredService<IAppPlanBillingManager>();
         }
 
-        private IUserResolver UserResolver()
+        private IUserResolver Users()
         {
             return serviceProvider.GetRequiredService<IUserResolver>();
         }
 
         private IAppLimitsPlan GetFreePlan()
         {
-            return AppPlansProvider().GetFreePlan();
+            return Plans().GetFreePlan();
         }
 
         private IAppLimitsPlan GetPlan()
         {
-            return AppPlansProvider().GetPlanForApp(Snapshot).Plan;
+            return Plans().GetPlanForApp(Snapshot).Plan;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IAppPlanBillingManager.cs

@@ -13,7 +13,13 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans
     {
         bool HasPortal { get; }
 
-        Task<IChangePlanResult> ChangePlanAsync(string userId, NamedId<DomainId> appId, string? planId, string? referer,
+        Task<Uri?> MustRedirectToPortalAsync(string userId, NamedId<DomainId> appId, string? planId,
+            CancellationToken ct = default);
+
+        Task SubscribeAsync(string userId, NamedId<DomainId> appId, string planId,
+            CancellationToken ct = default);
+
+        Task UnsubscribeAsync(string userId, NamedId<DomainId> appId,
             CancellationToken ct = default);
 
         Task<string> GetPortalLinkAsync(string userId,

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/NoopAppPlanBillingManager.cs

@@ -16,16 +16,28 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans
             get => false;
         }
 
-        public Task<IChangePlanResult> ChangePlanAsync(string userId, NamedId<DomainId> appId, string? planId, string? referer,
+        public Task<string> GetPortalLinkAsync(string userId,
             CancellationToken ct = default)
         {
-            return Task.FromResult<IChangePlanResult>(new PlanChangedResult());
+            return Task.FromResult(string.Empty);
         }
 
-        public Task<string> GetPortalLinkAsync(string userId,
+        public Task<Uri?> MustRedirectToPortalAsync(string userId, NamedId<DomainId> appId, string? planId,
             CancellationToken ct = default)
         {
-            return Task.FromResult(string.Empty);
+            return Task.FromResult<Uri?>(null);
+        }
+
+        public Task SubscribeAsync(string userId, NamedId<DomainId> appId, string planId,
+            CancellationToken ct = default)
+        {
+            return Task.CompletedTask;
+        }
+
+        public Task UnsubscribeAsync(string userId, NamedId<DomainId> appId,
+            CancellationToken ct = default)
+        {
+            return Task.CompletedTask;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangeAsyncResult.cs

@@ -1,13 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-namespace Squidex.Domain.Apps.Entities.Apps.Plans
-{
-    public sealed class PlanChangeAsyncResult : IChangePlanResult
-    {
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/PlanChangedResult.cs

@@ -5,9 +5,11 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
+
 namespace Squidex.Domain.Apps.Entities.Apps.Plans
 {
-    public sealed class PlanChangedResult : IChangePlanResult
+    public sealed record PlanChangedResult(string PlanId, bool Unsubscribed = false, Uri? RedirectUri = null)
     {
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/RedirectToCheckoutResult.cs

@@ -1,23 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using Squidex.Infrastructure;
-
-namespace Squidex.Domain.Apps.Entities.Apps.Plans
-{
-    public sealed class RedirectToCheckoutResult : IChangePlanResult
-    {
-        public Uri Url { get; }
-
-        public RedirectToCheckoutResult(Uri url)
-        {
-            Guard.NotNull(url);
-
-            Url = url;
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs

@@ -5,8 +5,10 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Infrastructure;
+using Squidex.Infrastructure.States;
 using Squidex.Infrastructure.UsageTracking;
 
 #pragma warning disable CS0649
@@ -18,11 +20,26 @@ namespace Squidex.Domain.Apps.Entities.Assets
         private const string CounterTotalCount = "TotalAssets";
         private const string CounterTotalSize = "TotalSize";
         private static readonly DateTime SummaryDate;
+        private readonly IAssetLoader assetLoader;
+        private readonly ISnapshotStore<State> store;
+        private readonly ITagService tagService;
         private readonly IUsageTracker usageTracker;
 
-        public AssetUsageTracker(IUsageTracker usageTracker)
+        [CollectionName("Index_TagHistory")]
+        public sealed class State
         {
+            public HashSet<string>? Tags { get; set; }
+        }
+
+        public AssetUsageTracker(IUsageTracker usageTracker, IAssetLoader assetLoader, ITagService tagService,
+            ISnapshotStore<State> store)
+        {
+            this.assetLoader = assetLoader;
+            this.tagService = tagService;
+            this.store = store;
             this.usageTracker = usageTracker;
+
+            ClearCache();
         }
 
         Task IDeleter.DeleteAppAsync(IAppEntity app,
@@ -48,12 +65,13 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             var usages = await usageTracker.QueryAsync(GetKey(appId), fromDate, toDate);
 
-            if (usages.TryGetValue("*", out var byCategory1))
+            if (usages.TryGetValue(usageTracker.FallbackCategory, out var byCategory1))
             {
                 AddCounters(enriched, byCategory1);
             }
             else if (usages.TryGetValue("Default", out var byCategory2))
             {
+                // Fallback for older versions where default was uses as tracking category.
                 AddCounters(enriched, byCategory2);
             }
 

backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs

@@ -5,9 +5,13 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
+using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Domain.Apps.Events.Assets;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.EventSourcing;
+using Squidex.Infrastructure.States;
 using Squidex.Infrastructure.UsageTracking;
 
 #pragma warning disable MA0048 // File name must match type name
@@ -16,6 +20,8 @@ namespace Squidex.Domain.Apps.Entities.Assets
 {
     public partial class AssetUsageTracker : IEventConsumer
     {
+        private IMemoryCache memoryCache;
+
         public int BatchSize
         {
             get => 1000;
@@ -36,18 +42,158 @@ namespace Squidex.Domain.Apps.Entities.Assets
             get => "^asset-";
         }
 
-        public Task On(Envelope<IEvent> @event)
+        private void ClearCache()
+        {
+            memoryCache?.Dispose();
+            memoryCache = new MemoryCache(Options.Create(new MemoryCacheOptions()));
+        }
+
+        public async Task ClearAsync()
+        {
+            // Will not remove data, but reset alls counts to zero.
+            await tagService.ClearAsync();
+
+            // Also clear the store and cache, because otherwise we would use data from the future when querying old tags.
+            ClearCache();
+
+            await store.ClearAsync();
+
+            // Use a well defined prefix query for the deletion to improve performance.
+            await usageTracker.DeleteByKeyPatternAsync("^([a-zA-Z0-9]+)_Assets");
+        }
+
+        public async Task On(IEnumerable<Envelope<IEvent>> events)
+        {
+            foreach (var @event in events)
+            {
+                // Usage tracking is done in the backgroud, therefore we do no use any batching.
+                await TrackUsageAsync(@event);
+            }
+
+            // Event consumers should only do one task, but too many consumers also hurt performance.
+            await AddTagsAsync(events);
+        }
+
+        private async Task AddTagsAsync(IEnumerable<Envelope<IEvent>> events)
+        {
+            var tagsPerApp = new Dictionary<DomainId, Dictionary<string, int>>();
+            var tagsPerAsset = new Dictionary<DomainId, State>();
+
+            void AddTagsToStore(DomainId appId, HashSet<string>? tagIds, int count)
+            {
+                if (tagIds != null)
+                {
+                    var perApp = tagsPerApp.GetOrAddNew(appId);
+
+                    foreach (var tag in tagIds)
+                    {
+                        perApp[tag] = perApp.GetOrDefault(tag) + count;
+                    }
+                }
+            }
+
+            void AddTagsToCache(DomainId key, HashSet<string>? tags, long version)
+            {
+                var state = new State { Tags = tags };
+
+                // Write tags to a buffer so that we can write them to a store in batches.
+                tagsPerAsset[key] = state;
+
+                // Write to the cache immediately, to be available for the next event. Use a relatively long cache time for live updates.
+                memoryCache.Set(key, state, TimeSpan.FromHours(1));
+            }
+
+            foreach (var @event in events)
+            {
+                var typedEvent = (AssetEvent)@event.Payload;
+
+                var appId = typedEvent.AppId.Id;
+                var assetId = typedEvent.AssetId;
+                var assetKey = @event.Headers.AggregateId();
+                var version = @event.Headers.EventStreamNumber();
+
+                switch (typedEvent)
+                {
+                    case AssetCreated assetCreated:
+                        {
+                            AddTagsToStore(appId, assetCreated.Tags, 1);
+                            AddTagsToCache(assetKey, assetCreated.Tags, version);
+                            break;
+                        }
+
+                    case AssetAnnotated assetAnnotated when assetAnnotated.Tags != null:
+                        {
+                            var oldTags = await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default);
+
+                            AddTagsToStore(appId, assetAnnotated.Tags, 1);
+                            AddTagsToStore(appId, oldTags, -1);
+                            AddTagsToCache(assetKey, assetAnnotated.Tags, version);
+                            break;
+                        }
+
+                    case AssetDeleted assetDeleted:
+                        {
+                            // We need the old tags here for permanent deletions.
+                            var oldTags =
+                                assetDeleted.OldTags ??
+                                await GetAndUpdateOldTagsAsync(appId, assetId, assetKey, version, default);
+
+                            AddTagsToStore(appId, oldTags, -1);
+                            break;
+                        }
+                }
+            }
+
+            // There is no good solution for batching anyway, so there is no need to build a method for that.
+            foreach (var (appId, updates) in tagsPerApp)
+            {
+                await tagService.UpdateAsync(appId, TagGroups.Assets, updates);
+            }
+
+            await store.WriteManyAsync(tagsPerAsset.Select(x => new SnapshotWriteJob<State>(x.Key, x.Value, 0)));
+        }
+
+        private async Task<HashSet<string>?> GetAndUpdateOldTagsAsync(DomainId appId, DomainId assetId, DomainId key, long version,
+            CancellationToken ct)
+        {
+            // Store the latest tags in memory for fast access.
+            if (memoryCache.TryGetValue<State>(key, out var state))
+            {
+                return state.Tags;
+            }
+
+            var stored = await store.ReadAsync(key, ct);
+
+            // Stored state can be null, if not serialized yet.
+            if (stored.Value != null)
+            {
+                return stored.Value.Tags;
+            }
+
+            // Some deleted events (like permanent deletion) have version of zero, but there is not previous event.
+            if (version == 0)
+            {
+                return null;
+            }
+
+            // This will replay a lot of events, so it is the slowest alternative.
+            var previousAsset = await assetLoader.GetAsync(appId, assetId, version - 1, ct);
+
+            return previousAsset?.Tags;
+        }
+
+        private Task TrackUsageAsync(Envelope<IEvent> @event)
         {
             switch (@event.Payload)
             {
-                case AssetCreated e:
+                case AssetCreated assetCreated:
-                    return UpdateSizeAsync(e.AppId.Id, GetDate(@event), e.FileSize, 1);
+                    return UpdateSizeAsync(assetCreated.AppId.Id, GetDate(@event), assetCreated.FileSize, 1);
 
-                case AssetUpdated e:
+                case AssetUpdated assetUpdated:
-                    return UpdateSizeAsync(e.AppId.Id, GetDate(@event), e.FileSize, 0);
+                    return UpdateSizeAsync(assetUpdated.AppId.Id, GetDate(@event), assetUpdated.FileSize, 0);
 
-                case AssetDeleted e:
+                case AssetDeleted assetDeleted:
-                    return UpdateSizeAsync(e.AppId.Id, GetDate(@event), -e.DeletedSize, -1);
+                    return UpdateSizeAsync(assetDeleted.AppId.Id, GetDate(@event), -assetDeleted.DeletedSize, -1);
             }
 
             return Task.CompletedTask;

backend/src/Squidex.Domain.Apps.Entities/Assets/BackupAssets.cs

@@ -9,6 +9,7 @@ using Squidex.Assets;
 using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Domain.Apps.Entities.Assets.DomainObject;
 using Squidex.Domain.Apps.Entities.Backup;
+using Squidex.Domain.Apps.Events.Apps;
 using Squidex.Domain.Apps.Events.Assets;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Commands;
@@ -71,6 +72,10 @@ namespace Squidex.Domain.Apps.Entities.Assets
         {
             switch (@event.Payload)
             {
+                case AppCreated:
+                    // Restore the tags first so that the processing of consecutive events have the necessary structure.
+                    await RestoreTagsAsync(context, ct);
+                    break;
                 case AssetFolderCreated:
                     assetFolderIds.Add(@event.Headers.AggregateId());
                     break;
@@ -100,8 +105,6 @@ namespace Squidex.Domain.Apps.Entities.Assets
         public async Task RestoreAsync(RestoreContext context,
             CancellationToken ct)
         {
-            await RestoreTagsAsync(context, ct);
-
             if (assetIds.Count > 0)
             {
                 await rebuilder.InsertManyAsync<AssetDomainObject, AssetDomainObject.State>(assetIds, BatchSize, ct);
@@ -116,26 +119,32 @@ namespace Squidex.Domain.Apps.Entities.Assets
         private async Task RestoreTagsAsync(RestoreContext context,
             CancellationToken ct)
         {
-            var tags = (Dictionary<string, Tag>?)null;
+            var export = new TagsExport();
 
             if (await context.Reader.HasFileAsync(TagsFile, ct))
             {
-                tags = await context.Reader.ReadJsonAsync<Dictionary<string, Tag>>(TagsFile, ct);
+                export.Tags = await context.Reader.ReadJsonAsync<Dictionary<string, Tag>>(TagsFile, ct);
             }
 
-            var alias = (Dictionary<string, string>?)null;
+            // For backwards compabibility we store the tags and the aliases in different locations.
-
             if (await context.Reader.HasFileAsync(TagsAliasFile, ct))
             {
-                alias = await context.Reader.ReadJsonAsync<Dictionary<string, string>>(TagsAliasFile, ct);
+                export.Alias = await context.Reader.ReadJsonAsync<Dictionary<string, string>>(TagsAliasFile, ct);
             }
 
-            if (alias == null && tags == null)
+            if (export.Alias == null && export.Tags == null)
             {
                 return;
             }
 
-            var export = new TagsExport { Tags = tags, Alias = alias };
+            if (export.Tags != null)
+            {
+                // Import the tags without count, because they will populated later by the event processor.
+                foreach (var (_, tag) in export.Tags)
+                {
+                    tag.Count = 0;
+                }
+            }
 
             await tagService.RebuildTagsAsync(context.AppId, TagGroups.Assets, export, ct);
         }
@@ -147,11 +156,13 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             if (tags.Tags != null)
             {
+                // Export the tags with count, even though we do not need it. But in general it makes the code easier.
                 await context.Writer.WriteJsonAsync(TagsFile, tags.Tags, ct);
             }
 
             if (tags.Alias?.Count > 0)
             {
+                // For backwards compabibility we store the tags and the aliases in different locations.
                 await context.Writer.WriteJsonAsync(TagsAliasFile, tags.Alias, ct);
             }
         }

backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetCommandMiddleware.cs

@@ -99,8 +99,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
             finally
             {
                 await assetFileStore.DeleteAsync(tempFile, ct);
-
-                await command.File.DisposeAsync();
             }
         }
 
@@ -119,8 +117,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
             finally
             {
                 await assetFileStore.DeleteAsync(tempFile, ct);
-
-                await command.File.DisposeAsync();
             }
         }
 
@@ -134,7 +130,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
                 if (result.IsChanged && context.Command is UploadAssetCommand)
                 {
                     var tempFile = context.ContextId.ToString();
-
                     try
                     {
                         await assetFileStore.CopyAsync(tempFile, asset.AppId.Id, asset.AssetId, asset.FileVersion, null, ct);

backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/AssetDomainObject.cs

@@ -166,7 +166,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
 
             if (create.Tags != null)
             {
-                create.Tags = await operation.NormalizeTags(create.Tags);
+                create.Tags = await operation.GetTagIdsAsync(create.Tags);
             }
 
             Create(create);
@@ -181,7 +181,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
 
             if (annotate.Tags != null)
             {
-                annotate.Tags = await operation.NormalizeTags(annotate.Tags);
+                annotate.Tags = await operation.GetTagIdsAsync(annotate.Tags);
             }
 
             Annotate(annotate);
@@ -224,8 +224,6 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
                 await operation.ExecuteDeleteScriptAsync(delete);
             }
 
-            await operation.UnsetTags();
-
             Delete(delete);
         }
 
@@ -262,7 +260,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
 
         private void Delete(DeleteAsset command)
         {
-            Raise(command, new AssetDeleted { DeletedSize = Snapshot.TotalSize });
+            Raise(command, new AssetDeleted { OldTags = Snapshot.Tags, DeletedSize = Snapshot.TotalSize });
         }
 
         private void Raise<T, TEvent>(T command, TEvent @event) where T : class where TEvent : AppEvent

backend/src/Squidex.Domain.Apps.Entities/Assets/DomainObject/Guards/TagsExtensions.cs

@@ -6,25 +6,26 @@
 // ==========================================================================
 
 using Squidex.Domain.Apps.Core.Tags;
+using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Assets.DomainObject.Guards
 {
     public static class TagsExtensions
     {
-        public static async Task<HashSet<string>> NormalizeTags(this AssetOperation operation, HashSet<string> tags)
+        public static async Task<HashSet<string>> GetTagIdsAsync(this AssetOperation operation, HashSet<string>? names)
         {
-            var tagService = operation.Resolve<ITagService>();
+            var result = new HashSet<string>(names?.Count ?? 0);
 
-            var normalized = await tagService.NormalizeTagsAsync(operation.App.Id, TagGroups.Assets, tags, operation.Snapshot.Tags);
+            if (names != null)
+            {
+                var tagService = operation.Resolve<ITagService>();
 
-            return new HashSet<string>(normalized.Values);
+                var normalized = await tagService.GetTagIdsAsync(operation.App.Id, TagGroups.Assets, names);
-        }
 
-        public static async Task UnsetTags(this AssetOperation operation)
+                result.AddRange(normalized.Values);
-        {
+            }
-            var tagService = operation.Resolve<ITagService>();
 
-            await tagService.NormalizeTagsAsync(operation.App.Id, TagGroups.Assets, null, operation.Snapshot.Tags);
+            return result;
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Assets/Queries/AssetEnricher.cs

@@ -161,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries
         {
             var uniqueIds = group.Where(x => x.Tags != null).SelectMany(x => x.Tags).ToHashSet();
 
-            return await tagService.DenormalizeTagsAsync(group.Key, TagGroups.Assets, uniqueIds, ct);
+            return await tagService.GetTagNamesAsync(group.Key, TagGroups.Assets, uniqueIds, ct);
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Comments/WatchingService.cs

@@ -22,7 +22,7 @@ namespace Squidex.Domain.Apps.Entities.Comments
 
             public Dictionary<string, Instant> Users { get; } = new Dictionary<string, Instant>();
 
-            public string[] Add(string watcherId, IClock clock)
+            public (bool, string[]) Add(string watcherId, IClock clock)
             {
                 var now = clock.GetCurrentInstant();
 
@@ -38,7 +38,7 @@ namespace Squidex.Domain.Apps.Entities.Comments
 
                 Users[watcherId] = now;
 
-                return Users.Keys.ToArray();
+                return (true, Users.Keys.ToArray());
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities/Contents/Counter/CounterService.cs

@@ -20,14 +20,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.Counter
         {
             public Dictionary<string, long> Counters { get; set; } = new Dictionary<string, long>();
 
-            public void Increment(string name)
+            public bool Increment(string name)
             {
                 Counters[name] = Counters.GetValueOrDefault(name) + 1;
+
+                return true;
             }
 
-            public void Reset(string name, long value)
+            public bool Reset(string name, long value)
             {
                 Counters[name] = value;
+
+                return true;
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities/OperationContextBase.cs

@@ -18,7 +18,8 @@ namespace Squidex.Domain.Apps.Entities
     {
         private readonly List<ValidationError> errors = new List<ValidationError>();
         private readonly IServiceProvider serviceProvider;
-        private readonly Func<TSnapShot> snapshot;
+        private readonly Func<TSnapShot> snapshotProvider;
+        private readonly TSnapShot snapshotInitial;
 
         public RefToken Actor => Command.Actor;
 
@@ -28,17 +29,22 @@ namespace Squidex.Domain.Apps.Entities
 
         public TCommand Command { get; init; }
 
-        public TSnapShot Snapshot => snapshot();
+        public TSnapShot Snapshot => snapshotProvider();
+
+        public TSnapShot SnapshotInitial => snapshotInitial;
 
         public ClaimsPrincipal? User => Command.User;
 
-        protected OperationContextBase(IServiceProvider serviceProvider, Func<TSnapShot> snapshot)
+        public Dictionary<string, object> Context { get; } = new Dictionary<string, object>();
+
+        protected OperationContextBase(IServiceProvider serviceProvider, Func<TSnapShot> snapshotProvider)
         {
             Guard.NotNull(serviceProvider);
-            Guard.NotNull(snapshot);
+            Guard.NotNull(snapshotProvider);
 
             this.serviceProvider = serviceProvider;
-            this.snapshot = snapshot;
+            this.snapshotProvider = snapshotProvider;
+            this.snapshotInitial = snapshotProvider();
         }
 
         public T Resolve<T>() where T : notnull

backend/src/Squidex.Domain.Apps.Entities/Tags/TagService.cs

@@ -5,9 +5,11 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Threading.Tasks.Dataflow;
 using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.States;
+using Squidex.Infrastructure.Tasks;
 
 namespace Squidex.Domain.Apps.Entities.Tags
 {
@@ -16,174 +18,176 @@ namespace Squidex.Domain.Apps.Entities.Tags
         private readonly IPersistenceFactory<State> persistenceFactory;
 
         [CollectionName("Index_Tags")]
-        public sealed class State : TagsExport
+        public sealed class State : TagsExport, IOnRead
         {
-            public void Rebuild(TagsExport export)
+            public ValueTask OnReadAsync()
             {
-                Tags = export.Tags;
+                if (Tags == null)
+                {
+                    Tags = new Dictionary<string, Tag>();
+                }
+
+                if (Alias == null)
+                {
+                    Alias = new Dictionary<string, string>();
+                }
 
-                Alias = export.Alias;
+                return default;
             }
 
-            public void Rename(string name, string newName)
+            public bool Rebuild(TagsExport export)
             {
-                Guard.NotNull(name);
+                if (export.Tags != null)
-                Guard.NotNull(newName);
+                {
+                    Tags = export.Tags;
+                }
 
-                name = NormalizeName(name);
+                if (export.Alias != null)
+                {
+                    Alias = export.Alias;
+                }
 
-                var (_, tag) = FindTag(name);
+                return true;
+            }
+
+            public bool Clear()
+            {
+                var isChanged = false;
 
-                if (tag == null)
+                foreach (var (_, tag) in Tags)
                 {
-                    return;
+                    isChanged = tag.Count > 0;
+
+                    tag.Count = 0;
+                }
+
+                return isChanged;
+            }
+
+            public bool Rename(string name, string newName)
+            {
+                name = NormalizeName(name);
+
+                if (!TryGetTag(name, out var tag))
+                {
+                    return false;
                 }
 
                 newName = NormalizeName(newName);
 
-                tag.Name = newName;
+                if (string.Equals(name, newName, StringComparison.OrdinalIgnoreCase))
+                {
+                    return false;
+                }
+
+                tag.Value.Name = newName;
 
-                if (Alias != null)
+                foreach (var alias in Alias.Where(x => x.Value == name).ToList())
                 {
-                    foreach (var alias in Alias.Where(x => x.Value == name).ToList())
+                    Alias.Remove(alias.Key);
-                    {
-                        Alias.Remove(alias.Key);
 
-                        if (alias.Key != newName)
+                    if (alias.Key != tag.Value.Name)
-                        {
+                    {
-                            Alias[alias.Key] = newName;
+                        Alias[alias.Key] = tag.Value.Name;
-                        }
                     }
                 }
 
-                Alias ??= new Dictionary<string, string>();
+                return true;
-                Alias[name] = newName;
             }
 
-            public Dictionary<string, string> Normalize(HashSet<string>? names, HashSet<string>? ids)
+            public bool Update(Dictionary<string, int> updates)
             {
-                var result = new Dictionary<string, string>();
+                var isChanged = false;
 
-                if (names != null)
+                foreach (var (id, update) in updates)
                 {
-                    foreach (var tag in names)
+                    if (update != 0 && Tags.TryGetValue(id, out var tag))
                     {
-                        var name = NormalizeName(tag);
+                        var newCount = Math.Max(0, tag.Count + update);
 
-                        if (!string.IsNullOrWhiteSpace(name))
+                        if (newCount != tag.Count)
                         {
-                            result.Add(name, GetId(name, ids));
+                            tag.Count = newCount;
-                        }
-                    }
-                }
 
-                if (ids != null)
+                            isChanged = true;
-                {
-                    foreach (var id in ids)
-                    {
-                        if (!result.ContainsValue(id))
-                        {
-                            if (Tags != null && Tags.TryGetValue(id, out var tagInfo))
-                            {
-                                tagInfo.Count--;
-
-                                if (tagInfo.Count <= 0)
-                                {
-                                    Tags.Remove(id);
-                                }
-                            }
                         }
                     }
                 }
 
-                return result;
+                return isChanged;
             }
 
-            public Dictionary<string, string> GetTagIds(HashSet<string> names)
+            public (bool, Dictionary<string, string>) GetIds(HashSet<string> names)
             {
-                Guard.NotNull(names);
+                var tagIds = new Dictionary<string, string>();
 
-                var result = new Dictionary<string, string>();
+                var isChanged = false;
 
-                foreach (var tag in names)
+                foreach (var name in names.Select(NormalizeName))
                 {
-                    var name = NormalizeName(tag);
+                    if (TryGetTag(name, out var tag))
+                    {
+                        tagIds[name] = tag.Key;
+                    }
+                    else
+                    {
+                        var id = Guid.NewGuid().ToString();
 
-                    var (id, _) = FindTag(name);
+                        Tags[id] = new Tag { Name = name };
+                        tagIds[name] = id;
 
-                    if (!string.IsNullOrWhiteSpace(id))
+                        isChanged = true;
-                    {
-                        result.Add(name, id);
                     }
                 }
 
-                return result;
+                return (isChanged, tagIds);
             }
 
-            public Dictionary<string, string> Denormalize(HashSet<string> ids)
+            public Dictionary<string, string> GetNames(HashSet<string> ids)
             {
-                var result = new Dictionary<string, string>();
+                var tagNames = new Dictionary<string, string>();
 
                 foreach (var id in ids)
                 {
-                    if (Tags?.TryGetValue(id, out var tagInfo) == true)
+                    if (Tags.TryGetValue(id, out var tagInfo))
                     {
-                        result[id] = tagInfo.Name;
+                        tagNames[id] = tagInfo.Name;
                     }
                 }
 
-                return result;
+                return tagNames;
             }
 
             public TagsSet GetTags(long version)
             {
-                var tags = Tags?.Values.ToDictionary(x => x.Name, x => x.Count) ?? new Dictionary<string, int>();
+                var clone = Tags.Values.ToDictionary(x => x.Name, x => x.Count);
 
-                return new TagsSet(tags, version);
+                return new TagsSet(clone, version);
             }
 
-            public TagsExport GetExportableTags()
+            private static string NormalizeName(string name)
             {
-                var clone = Clone();
+                return name.Trim().ToLowerInvariant();
-
-                return clone;
             }
 
-            private string GetId(string name, HashSet<string>? ids)
+            private bool TryGetTag(string name, out KeyValuePair<string, Tag> result)
             {
-                var (id, tag) = FindTag(name);
+                result = default;
 
-                if (tag != null)
+                if (Alias.TryGetValue(name, out var newName))
-                {
-                    if (ids == null || !ids.Contains(id))
-                    {
-                        tag.Count++;
-                    }
-                }
-                else
                 {
-                    id = DomainId.NewGuid().ToString();
+                    name = newName;
-
-                    Tags ??= new Dictionary<string, Tag>();
-                    Tags.Add(id, new Tag { Name = name });
                 }
 
-                return id;
+                var found = Tags.FirstOrDefault(x => x.Value.Name == name);
-            }
-
-            private static string NormalizeName(string name)
-            {
-                return name.Trim().ToLowerInvariant();
-            }
 
-            private KeyValuePair<string, Tag> FindTag(string name)
+                if (found.Value != null)
-            {
-                if (Alias?.TryGetValue(name, out var newName) == true)
                 {
-                    name = newName;
+                    result = new KeyValuePair<string, Tag>(found.Key, found.Value);
+                    return true;
                 }
 
-                return Tags?.FirstOrDefault(x => x.Value.Name == name) ?? default;
+                return false;
             }
         }
 
@@ -220,25 +224,25 @@ namespace Squidex.Domain.Apps.Entities.Tags
 
             var state = await GetStateAsync(id, group, ct);
 
-            return await state.UpdateAsync(s => s.GetTagIds(names), ct: ct);
+            return await state.UpdateAsync(s => s.GetIds(names), ct: ct);
         }
 
-        public async Task<Dictionary<string, string>> DenormalizeTagsAsync(DomainId id, string group, HashSet<string> ids,
+        public async Task<Dictionary<string, string>> GetTagNamesAsync(DomainId id, string group, HashSet<string> ids,
             CancellationToken ct = default)
         {
             Guard.NotNull(ids);
 
             var state = await GetStateAsync(id, group, ct);
 
-            return await state.UpdateAsync(s => s.Denormalize(ids), ct: ct);
+            return state.Value.GetNames(ids);
         }
 
-        public async Task<Dictionary<string, string>> NormalizeTagsAsync(DomainId id, string group, HashSet<string>? names, HashSet<string>? ids,
+        public async Task UpdateAsync(DomainId id, string group, Dictionary<string, int> update,
             CancellationToken ct = default)
         {
             var state = await GetStateAsync(id, group, ct);
 
-            return await state.UpdateAsync(s => s.Normalize(names, ids), ct: ct);
+            await state.UpdateAsync(s => s.Update(update), ct: ct);
         }
 
         public async Task<TagsSet> GetTagsAsync(DomainId id, string group,
@@ -254,7 +258,7 @@ namespace Squidex.Domain.Apps.Entities.Tags
         {
             var state = await GetStateAsync(id, group, ct);
 
-            return state.Value.GetExportableTags();
+            return state.Value;
         }
 
         public async Task ClearAsync(DomainId id, string group,
@@ -274,5 +278,53 @@ namespace Squidex.Domain.Apps.Entities.Tags
 
             return state;
         }
+
+        public async Task ClearAsync(
+            CancellationToken ct = default)
+        {
+            var writerBlock = new ActionBlock<SnapshotResult<State>[]>(async batch =>
+            {
+                try
+                {
+                    var isChanged = !batch.All(x => !x.Value.Clear());
+
+                    if (isChanged)
+                    {
+                        var jobs = batch.Select(x => new SnapshotWriteJob<State>(x.Key, x.Value, x.Version));
+
+                        await persistenceFactory.Snapshots.WriteManyAsync(jobs, ct);
+                    }
+                }
+                catch (OperationCanceledException ex)
+                {
+                    // Dataflow swallows operation cancelled exception.
+                    throw new AggregateException(ex);
+                }
+            },
+            new ExecutionDataflowBlockOptions
+            {
+                BoundedCapacity = 2,
+                MaxDegreeOfParallelism = 1,
+                MaxMessagesPerTask = 1,
+            });
+
+            // Create batches of 500 items to clear the tag count for better performance.
+            var batchBlock = new BatchBlock<SnapshotResult<State>>(500, new GroupingDataflowBlockOptions
+            {
+                BoundedCapacity = 500
+            });
+
+            batchBlock.BidirectionalLinkTo(writerBlock);
+
+            await foreach (var state in persistenceFactory.Snapshots.ReadAllAsync(ct))
+            {
+                // Uses back-propagation to not query additional items from the database, when queue is full.
+                await batchBlock.SendAsync(state, ct);
+            }
+
+            batchBlock.Complete();
+
+            await writerBlock.Completion;
+        }
     }
 }

backend/src/Squidex.Domain.Apps.Events/Assets/AssetDeleted.cs

@@ -13,5 +13,7 @@ namespace Squidex.Domain.Apps.Events.Assets
     public sealed class AssetDeleted : AssetEvent
     {
         public long DeletedSize { get; set; }
+
+        public HashSet<string>? OldTags { get; set; }
     }
 }

backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoBase.cs

@@ -52,9 +52,6 @@ namespace Squidex.Infrastructure.MongoDb
             BsonJsonConvention.Register();
             BsonJsonValueSerializer.Register();
             BsonStringSerializer<RefToken>.Register();
-            BsonStringSerializer<NamedId<DomainId>>.Register();
-            BsonStringSerializer<NamedId<Guid>>.Register();
-            BsonStringSerializer<NamedId<string>>.Register();
         }
     }
 }

backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs

@@ -119,21 +119,60 @@ namespace Squidex.Infrastructure.MongoDb
             }
         }
 
-        public static async Task<bool> UpsertVersionedAsync<T, TKey>(this IMongoCollection<T> collection, TKey key, long oldVersion, long newVersion, T document,
+        public static async Task<bool> UpsertVersionedAsync<T>(this IMongoCollection<T> collection, IClientSessionHandle session, SnapshotWriteJob<T> job,
             CancellationToken ct = default)
-            where T : IVersionedEntity<TKey> where TKey : notnull
+            where T : IVersionedEntity<DomainId>
         {
+            var (key, snapshot, newVersion, oldVersion) = job;
             try
             {
-                document.DocumentId = key;
+                snapshot.DocumentId = key;
-                document.Version = newVersion;
+                snapshot.Version = newVersion;
 
                 Expression<Func<T, bool>> filter =
                     oldVersion > EtagVersion.Any ?
                     x => x.DocumentId.Equals(key) && x.Version == oldVersion :
                     x => x.DocumentId.Equals(key);
 
-                var result = await collection.ReplaceOneAsync(filter, document, UpsertReplace, ct);
+                var result = await collection.ReplaceOneAsync(session, filter, job.Value, UpsertReplace, ct);
+
+                return result.IsAcknowledged && result.ModifiedCount == 1;
+            }
+            catch (MongoWriteException ex) when (ex.WriteError?.Category == ServerErrorCategory.DuplicateKey)
+            {
+                var existingVersion =
+                    await collection.Find(session, x => x.DocumentId.Equals(key)).Only(x => x.DocumentId, x => x.Version)
+                        .FirstOrDefaultAsync(ct);
+
+                if (existingVersion != null)
+                {
+                    var field = Field.Of<T>(x => nameof(x.Version));
+
+                    throw new InconsistentStateException(existingVersion[field].AsInt64, oldVersion);
+                }
+                else
+                {
+                    throw new InconsistentStateException(EtagVersion.Any, oldVersion);
+                }
+            }
+        }
+
+        public static async Task<bool> UpsertVersionedAsync<T>(this IMongoCollection<T> collection, SnapshotWriteJob<T> job,
+            CancellationToken ct = default)
+            where T : IVersionedEntity<DomainId>
+        {
+            var (key, snapshot, newVersion, oldVersion) = job;
+            try
+            {
+                snapshot.DocumentId = key;
+                snapshot.Version = newVersion;
+
+                Expression<Func<T, bool>> filter =
+                    oldVersion > EtagVersion.Any ?
+                    x => x.DocumentId.Equals(key) && x.Version == oldVersion :
+                    x => x.DocumentId.Equals(key);
+
+                var result = await collection.ReplaceOneAsync(filter, snapshot, UpsertReplace, ct);
 
                 return result.IsAcknowledged && result.ModifiedCount == 1;
             }

backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStoreBase.cs

@@ -39,6 +39,11 @@ namespace Squidex.Infrastructure.States
 
                 if (existing != null)
                 {
+                    if (existing.Document is IOnRead onRead)
+                    {
+                        await onRead.OnReadAsync();
+                    }
+
                     return new SnapshotResult<T>(existing.DocumentId, existing.Document, existing.Version);
                 }
 
@@ -51,9 +56,9 @@ namespace Squidex.Infrastructure.States
         {
             using (Telemetry.Activities.StartActivity("MongoSnapshotStoreBase/WriteAsync"))
             {
-                var document = CreateDocument(job.Key, job.Value, job.OldVersion);
+                var entityJob = job.As(CreateDocument(job.Key, job.Value, job.OldVersion));
 
-                await Collection.UpsertVersionedAsync(job.Key, job.OldVersion, job.NewVersion, document, ct);
+                await Collection.UpsertVersionedAsync(entityJob, ct);
             }
         }
 
@@ -95,6 +100,11 @@ namespace Squidex.Infrastructure.States
 
                 await foreach (var document in find.ToAsyncEnumerable(ct))
                 {
+                    if (document.Document is IOnRead onRead)
+                    {
+                        await onRead.OnReadAsync();
+                    }
+
                     yield return new SnapshotResult<T>(document.DocumentId, document.Document, document.Version, true);
                 }
             }

backend/src/Squidex.Infrastructure.MongoDb/UsageTracking/MongoUsageRepository.cs

@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using MongoDB.Bson;
 using MongoDB.Driver;
 using Squidex.Infrastructure.MongoDb;
 
@@ -42,6 +43,14 @@ namespace Squidex.Infrastructure.UsageTracking
             return Collection.DeleteManyAsync(x => x.Key == key, ct);
         }
 
+        public Task DeleteByKeyPatternAsync(string pattern,
+            CancellationToken ct = default)
+        {
+            Guard.NotNull(pattern);
+
+            return Collection.DeleteManyAsync(Filter.Regex(x => x.Key, new BsonRegularExpression(pattern)), ct);
+        }
+
         public async Task TrackUsagesAsync(UsageUpdate update,
             CancellationToken ct = default)
         {

backend/src/Squidex.Infrastructure/CollectionExtensions.cs

@@ -327,11 +327,6 @@ namespace Squidex.Infrastructure
             return dictionary.GetOrAdd(key, _ => default!);
         }
 
-        public static TValue GetOrNew<TKey, TValue>(this IReadOnlyDictionary<TKey, TValue> dictionary, TKey key) where TKey : notnull where TValue : class, new()
-        {
-            return dictionary.GetOrCreate(key, _ => new TValue());
-        }
-
         public static TValue GetOrAddNew<TKey, TValue>(this IDictionary<TKey, TValue> dictionary, TKey key) where TKey : notnull where TValue : class, new()
         {
             return dictionary.GetOrAdd(key, _ => new TValue());

backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerManager.cs

@@ -14,12 +14,14 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
     {
         private readonly IPersistenceFactory<EventConsumerState> persistence;
         private readonly IMessageBus messaging;
+        private readonly HashSet<string> activeNames;
 
-        public EventConsumerManager(IPersistenceFactory<EventConsumerState> persistence,
+        public EventConsumerManager(IPersistenceFactory<EventConsumerState> persistence, IEnumerable<IEventConsumer> eventConsumers,
             IMessageBus messaging)
         {
             this.persistence = persistence;
             this.messaging = messaging;
+            this.activeNames = eventConsumers.Select(x => x.Name).ToHashSet();
         }
 
         public async Task<List<EventConsumerInfo>> GetConsumersAsync(
@@ -27,7 +29,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
         {
             var snapshots = await persistence.Snapshots.ReadAllAsync(ct).ToListAsync(ct);
 
-            return snapshots.Select(x => x.Value.ToInfo(x.Key.ToString())).ToList();
+            return snapshots.Where(x => activeNames.Contains(x.Key.ToString())).Select(x => x.Value.ToInfo(x.Key.ToString())).ToList();
         }
 
         public async Task<EventConsumerInfo> ResetAsync(string consumerName,
@@ -67,7 +69,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
 
             await state.LoadAsync(ct);
 
-            if (state.Version <= EtagVersion.Empty)
+            if (state.Version <= EtagVersion.Empty || !activeNames.Contains(consumerName))
             {
                 throw new DomainObjectNotFoundException(consumerName);
             }

backend/src/Squidex.Infrastructure/EventSourcing/Consume/EventConsumerProcessor.cs

@@ -144,6 +144,11 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
 
         public virtual async Task ResetAsync()
         {
+            if (!eventConsumer.CanClear)
+            {
+                return;
+            }
+
             await UpdateAsync(async () =>
             {
                 Unsubscribe();
@@ -160,7 +165,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
         {
             if (events.Count > 0)
             {
-                await eventConsumer!.On(events);
+                await eventConsumer.On(events);
             }
         }
 
@@ -197,7 +202,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
                     }
 
                     log.LogCritical(ex, "Failed to update consumer {consumer} at position {position} from {caller}.",
-                        eventConsumer!.Name, position, caller);
+                        eventConsumer.Name, position, caller);
 
                     State = previousState.Stopped(ex);
                 }
@@ -214,17 +219,17 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
         {
             if (log.IsEnabled(LogLevel.Debug))
             {
-                log.LogDebug("Event consumer {consumer} reset started", eventConsumer!.Name);
+                log.LogDebug("Event consumer {consumer} reset started", eventConsumer.Name);
             }
 
             var watch = ValueStopwatch.StartNew();
             try
             {
-                await eventConsumer!.ClearAsync();
+                await eventConsumer.ClearAsync();
             }
             finally
             {
-                log.LogDebug("Event consumer {consumer} reset completed after {time}ms.", eventConsumer!.Name, watch.Stop());
+                log.LogDebug("Event consumer {consumer} reset completed after {time}ms.", eventConsumer.Name, watch.Stop());
             }
         }
 
@@ -264,7 +269,7 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
 
         protected virtual IEventSubscription CreateSubscription(IEventSubscriber<StoredEvent> subscriber)
         {
-            return eventStore.CreateSubscription(subscriber, eventConsumer!.EventsFilter, State.Position);
+            return eventStore.CreateSubscription(subscriber, eventConsumer.EventsFilter, State.Position);
         }
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs

@@ -17,6 +17,8 @@ namespace Squidex.Infrastructure.EventSourcing
 
         string EventsFilter => ".*";
 
+        bool CanClear => true;
+
         bool Handles(StoredEvent @event)
         {
             return true;

backend/src/Squidex.Domain.Apps.Entities/Apps/Plans/IChangePlanResult.cs → backend/src/Squidex.Infrastructure/States/IOnRead.cs

@@ -5,9 +5,10 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-namespace Squidex.Domain.Apps.Entities.Apps.Plans
+namespace Squidex.Infrastructure.States
 {
-    public interface IChangePlanResult
+    public interface IOnRead
     {
+        ValueTask OnReadAsync();
     }
 }

backend/src/Squidex.Infrastructure/States/ISnapshotStore.cs

@@ -33,5 +33,11 @@ namespace Squidex.Infrastructure.States
 
     public record struct SnapshotResult<T>(DomainId Key, T Value, long Version, bool IsValid = true);
 
-    public record struct SnapshotWriteJob<T>(DomainId Key, T Value, long NewVersion, long OldVersion = EtagVersion.Any);
+    public record struct SnapshotWriteJob<T>(DomainId Key, T Value, long NewVersion, long OldVersion = EtagVersion.Any)
+    {
+        public readonly SnapshotWriteJob<TOther> As<TOther>(TOther snapshot)
+        {
+            return new SnapshotWriteJob<TOther>(Key, snapshot, NewVersion, OldVersion);
+        }
+    }
 }

backend/src/Squidex.Infrastructure/States/NameReservationState.cs

@@ -14,11 +14,12 @@ namespace Squidex.Infrastructure.States
         {
             public List<NameReservation> Reservations { get; set; } = new List<NameReservation>();
 
-            public string? Reserve(DomainId id, string name)
+            public (bool, string?) Reserve(DomainId id, string name)
             {
                 string? token = null;
 
                 var reservation = Reservations.Find(x => x.Name == name);
+                var reserved = false;
 
                 if (reservation?.Id == id)
                 {
@@ -29,14 +30,15 @@ namespace Squidex.Infrastructure.States
                     token = RandomHash.Simple();
 
                     Reservations.Add(new NameReservation(token, name, id));
+                    reserved = true;
                 }
 
-                return token;
+                return (reserved, token);
             }
 
-            public void Remove(string? token)
+            public bool Remove(string? token)
             {
-                Reservations.RemoveAll(x => x.Token == token);
+                return Reservations.RemoveAll(x => x.Token == token) > 0;
             }
         }
 

backend/src/Squidex.Infrastructure/States/SimpleState.cs

@@ -64,66 +64,35 @@ namespace Squidex.Infrastructure.States
             return persistence.WriteEventAsync(envelope, ct);
         }
 
-        public async Task UpdateIfAsync(Func<T, bool> updater, int retries = 20,
+        public Task UpdateAsync(Func<T, bool> updater, int retries = 20,
             CancellationToken ct = default)
         {
-            await EnsureLoadedAsync(ct);
+            return UpdateAsync(state => (updater(state), None.Value), retries, ct);
-
-            for (var i = 0; i < retries; i++)
-            {
-                try
-                {
-                    if (!updater(Value))
-                    {
-                        return;
-                    }
-
-                    await WriteAsync(ct);
-                    return;
-                }
-                catch (InconsistentStateException) when (i < retries)
-                {
-                    await LoadAsync(ct);
-                }
-            }
         }
 
-        public async Task UpdateAsync(Action<T> updater, int retries = 20,
+        public async Task<TResult> UpdateAsync<TResult>(Func<T, (bool, TResult)> updater, int retries = 20,
             CancellationToken ct = default)
         {
-            await EnsureLoadedAsync(ct);
+            if (!isLoaded)
-
-            for (var i = 0; i < retries; i++)
             {
-                try
+                await LoadAsync(ct);
-                {
-                    updater(Value);
-
-                    await WriteAsync(ct);
-                    return;
-                }
-                catch (InconsistentStateException) when (i < retries)
-                {
-                    await LoadAsync(ct);
-                }
             }
-        }
-
-        public async Task<TResult> UpdateAsync<TResult>(Func<T, TResult> updater, int retries = 5,
-            CancellationToken ct = default)
-        {
-            await EnsureLoadedAsync(ct);
 
             for (var i = 0; i < retries; i++)
             {
                 try
                 {
-                    var result = updater(Value);
+                    var (isChanged, result) = updater(Value);
+
+                    if (!isChanged)
+                    {
+                        return result;
+                    }
 
                     await WriteAsync(ct);
                     return result;
                 }
-                catch (InconsistentStateException) when (i < retries)
+                catch (InconsistentStateException) when (i < retries - 1)
                 {
                     await LoadAsync(ct);
                 }
@@ -131,16 +100,5 @@ namespace Squidex.Infrastructure.States
 
             return default!;
         }
-
-        private async Task EnsureLoadedAsync(
-            CancellationToken ct)
-        {
-            if (isLoaded)
-            {
-                return;
-            }
-
-            await LoadAsync(ct);
-        }
     }
 }

backend/src/Squidex.Infrastructure/UsageTracking/BackgroundUsageTracker.cs

@@ -14,7 +14,6 @@ namespace Squidex.Infrastructure.UsageTracking
     public sealed class BackgroundUsageTracker : DisposableObjectBase, IUsageTracker
     {
         private const int Intervall = 60 * 1000;
-        private const string FallbackCategory = "*";
         private readonly IUsageRepository usageRepository;
         private readonly ILogger<BackgroundUsageTracker> log;
         private readonly CompletionTimer timer;
@@ -22,6 +21,8 @@ namespace Squidex.Infrastructure.UsageTracking
 
         public bool ForceWrite { get; set; }
 
+        public string FallbackCategory => "*";
+
         public BackgroundUsageTracker(IUsageRepository usageRepository,
             ILogger<BackgroundUsageTracker> log)
         {
@@ -96,6 +97,14 @@ namespace Squidex.Infrastructure.UsageTracking
             return usageRepository.DeleteAsync(key, ct);
         }
 
+        public Task DeleteByKeyPatternAsync(string pattern,
+            CancellationToken ct = default)
+        {
+            Guard.NotNull(pattern);
+
+            return usageRepository.DeleteByKeyPatternAsync(pattern, ct);
+        }
+
         public Task TrackAsync(DateTime date, string key, string? category, Counters counters,
             CancellationToken ct = default)
         {
@@ -187,7 +196,7 @@ namespace Squidex.Infrastructure.UsageTracking
             return result;
         }
 
-        private static string GetCategory(string? category)
+        private string GetCategory(string? category)
         {
             return !string.IsNullOrWhiteSpace(category) ? category.Trim() : FallbackCategory;
         }

backend/src/Squidex.Infrastructure/UsageTracking/CachingUsageTracker.cs

@@ -15,11 +15,10 @@ namespace Squidex.Infrastructure.UsageTracking
         private readonly IUsageTracker inner;
         private readonly IMemoryCache cache;
 
+        public string FallbackCategory => inner.FallbackCategory;
+
         public CachingUsageTracker(IUsageTracker inner, IMemoryCache cache)
         {
-            Guard.NotNull(inner);
-            Guard.NotNull(cache);
-
             this.inner = inner;
             this.cache = cache;
         }
@@ -32,6 +31,14 @@ namespace Squidex.Infrastructure.UsageTracking
             return inner.DeleteAsync(key, ct);
         }
 
+        public Task DeleteByKeyPatternAsync(string pattern,
+            CancellationToken ct = default)
+        {
+            Guard.NotNull(pattern);
+
+            return inner.DeleteByKeyPatternAsync(pattern, ct);
+        }
+
         public Task<Dictionary<string, List<(DateTime, Counters)>>> QueryAsync(string key, DateTime fromDate, DateTime toDate,
             CancellationToken ct = default)
         {

backend/src/Squidex.Infrastructure/UsageTracking/IUsageRepository.cs

@@ -20,5 +20,8 @@ namespace Squidex.Infrastructure.UsageTracking
 
         Task DeleteAsync(string key,
             CancellationToken ct = default);
+
+        Task DeleteByKeyPatternAsync(string pattern,
+            CancellationToken ct = default);
     }
 }

backend/src/Squidex.Infrastructure/UsageTracking/IUsageTracker.cs

@@ -9,6 +9,8 @@ namespace Squidex.Infrastructure.UsageTracking
 {
     public interface IUsageTracker
     {
+        string FallbackCategory { get; }
+
         Task TrackAsync(DateTime date, string key, string? category, Counters counters,
             CancellationToken ct = default);
 
@@ -23,5 +25,8 @@ namespace Squidex.Infrastructure.UsageTracking
 
         Task DeleteAsync(string key,
             CancellationToken ct = default);
+
+        Task DeleteByKeyPatternAsync(string pattern,
+            CancellationToken ct = default);
     }
 }

backend/src/Squidex/Areas/Api/Controllers/Plans/AppPlansController.cs

@@ -77,13 +77,13 @@ namespace Squidex.Areas.Api.Controllers.Plans
         [ApiCosts(0)]
         public async Task<IActionResult> PutPlan(string app, [FromBody] ChangePlanDto request)
         {
-            var context = await CommandBus.PublishAsync(request.ToCommand(HttpContext), HttpContext.RequestAborted);
+            var context = await CommandBus.PublishAsync(request.ToCommand(), HttpContext.RequestAborted);
 
             string? redirectUri = null;
 
-            if (context.PlainResult is RedirectToCheckoutResult result)
+            if (context.PlainResult is PlanChangedResult result)
             {
-                redirectUri = result.Url.ToString();
+                redirectUri = result.RedirectUri?.ToString();
             }
 
             return Ok(new PlanChangedDto { RedirectUri = redirectUri });

backend/src/Squidex/Areas/Api/Controllers/Plans/Models/ChangePlanDto.cs

@@ -19,12 +19,10 @@ namespace Squidex.Areas.Api.Controllers.Plans.Models
         [LocalizedRequired]
         public string PlanId { get; set; }
 
-        public ChangePlan ToCommand(HttpContext httpContext)
+        public ChangePlan ToCommand()
         {
             var result = SimpleMapper.Map(this, new ChangePlan());
 
-            result.Referer = httpContext.Request.Headers["Referer"];
-
             return result;
         }
     }

backend/src/Squidex/Config/Domain/LoggingServices.cs

@@ -18,11 +18,11 @@ namespace Squidex.Config.Domain
         public static void ConfigureForSquidex(this ILoggingBuilder builder, IConfiguration config)
         {
             builder.ClearProviders();
+
+            // Also adds semantic logging.
             builder.ConfigureSemanticLog(config);
 
             builder.AddConfiguration(config.GetSection("logging"));
-
-            builder.AddSemanticLog();
             builder.AddFilters();
 
             builder.Services.AddServices(config);

backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/Tags/TagNormalizerTests.cs

@@ -1,136 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using FakeItEasy;
-using Squidex.Domain.Apps.Core.Contents;
-using Squidex.Domain.Apps.Core.Schemas;
-using Squidex.Domain.Apps.Core.Tags;
-using Squidex.Domain.Apps.Core.TestHelpers;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Json.Objects;
-using Xunit;
-
-namespace Squidex.Domain.Apps.Core.Operations.Tags
-{
-    public class TagNormalizerTests
-    {
-        private readonly ITagService tagService = A.Fake<ITagService>();
-        private readonly DomainId appId = DomainId.NewGuid();
-        private readonly DomainId schemaId = DomainId.NewGuid();
-        private readonly Schema schema;
-
-        public TagNormalizerTests()
-        {
-            schema =
-                new Schema("my-schema")
-                    .AddTags(1, "tags1", Partitioning.Invariant)
-                    .AddTags(2, "tags2", Partitioning.Invariant, new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema })
-                    .AddString(3, "string", Partitioning.Invariant)
-                    .AddArray(4, "array", Partitioning.Invariant, f => f
-                        .AddTags(401, "nestedTags1")
-                        .AddTags(402, "nestedTags2", new TagsFieldProperties { Normalization = TagsFieldNormalization.Schema })
-                        .AddString(403, "string"));
-        }
-
-        [Fact]
-        public async Task Should_normalize_tags_with_old_data()
-        {
-            var newData = GenerateData("n_raw");
-            var oldData = GenerateData("o_raw");
-
-            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
-                A<HashSet<string>>.That.Is("n_raw2_1", "n_raw2_2", "n_raw4"),
-                A<HashSet<string>>.That.Is("o_raw2_1", "o_raw2_2", "o_raw4"),
-                default))
-                .Returns(new Dictionary<string, string>
-                {
-                    ["n_raw2_2"] = "id2_2",
-                    ["n_raw2_1"] = "id2_1",
-                    ["n_raw4"] = "id4"
-                });
-
-            await tagService.NormalizeAsync(appId, schemaId, schema, newData, oldData);
-
-            Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]);
-            Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData));
-        }
-
-        [Fact]
-        public async Task Should_normalize_tags_without_old_data()
-        {
-            var newData = GenerateData("name");
-
-            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
-                A<HashSet<string>>.That.Is("name2_1", "name2_2", "name4"),
-                A<HashSet<string>>.That.IsEmpty(),
-                default))
-                .Returns(new Dictionary<string, string>
-                {
-                    ["name2_2"] = "id2_2",
-                    ["name2_1"] = "id2_1",
-                    ["name4"] = "id4"
-                });
-
-            await tagService.NormalizeAsync(appId, schemaId, schema, newData, null);
-
-            Assert.Equal(JsonValue.Array("id2_1", "id2_2"), newData["tags2"]!["iv"]);
-            Assert.Equal(JsonValue.Array("id4"), GetNestedTags(newData));
-        }
-
-        [Fact]
-        public async Task Should_denormalize_tags()
-        {
-            var newData = GenerateData("id");
-
-            A.CallTo(() => tagService.NormalizeTagsAsync(appId, TagGroups.Schemas(schemaId),
-                A<HashSet<string>>.That.Is("id2_1", "id2_2", "id4"),
-                A<HashSet<string>>.That.IsEmpty(),
-                default))
-                .Returns(new Dictionary<string, string>
-                {
-                    ["id2_2"] = "name2_2",
-                    ["id2_1"] = "name2_1",
-                    ["id4"] = "name4"
-                });
-
-            await tagService.NormalizeAsync(appId, schemaId, schema, newData, null);
-
-            Assert.Equal(JsonValue.Array("name2_1", "name2_2"), newData["tags2"]!["iv"]);
-            Assert.Equal(JsonValue.Array("name4"), GetNestedTags(newData));
-        }
-
-        private static JsonValue GetNestedTags(ContentData newData)
-        {
-            var arrayValue = newData["array"]!["iv"].AsArray;
-            var arrayItem = arrayValue[0].AsObject;
-
-            return arrayItem["nestedTags2"];
-        }
-
-        private static ContentData GenerateData(string prefix)
-        {
-            return new ContentData()
-                .AddField("tags1",
-                    new ContentFieldData()
-                        .AddInvariant(JsonValue.Array($"{prefix}1")))
-                .AddField("tags2",
-                    new ContentFieldData()
-                        .AddInvariant(JsonValue.Array($"{prefix}2_1", $"{prefix}2_2")))
-                .AddField("string",
-                    new ContentFieldData()
-                        .AddInvariant($"{prefix}stringValue"))
-                .AddField("array",
-                    new ContentFieldData()
-                        .AddInvariant(
-                            JsonValue.Array(
-                                new JsonObject()
-                                    .Add("nestedTags1", JsonValue.Array($"{prefix}3"))
-                                    .Add("nestedTags2", JsonValue.Array($"{prefix}4"))
-                                    .Add("string", $"{prefix}nestedStringValue"))));
-        }
-    }
-}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/DomainObject/AppDomainObjectTests.cs

@@ -59,6 +59,9 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
             A.CallTo(() => appPlansProvider.GetPlan(planIdPaid))
                 .Returns(new ConfigAppLimitsPlan { Id = planIdPaid, MaxContributors = 30 });
 
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, A<string>._, default))
+                .Returns(Task.FromResult<Uri?>(null));
+
             // Create a non-empty setting, otherwise the event is not raised as it does not change the domain object.
             initialSettings = new InitialSettings
             {
@@ -217,14 +220,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
         {
             var command = new ChangePlan { PlanId = planIdPaid };
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default))
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default))
-                .Returns(new PlanChangedResult());
+                .Returns(Task.FromResult<Uri?>(null));
 
             await ExecuteCreateAsync();
 
             var result = await PublishIdempotentAsync(command);
 
-            Assert.True(result is PlanChangedResult);
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid));
 
             Assert.Equal(planIdPaid, sut.Snapshot.Plan!.PlanId);
 
@@ -232,6 +235,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                 .ShouldHaveSameEvents(
                     CreateEvent(new AppPlanChanged { PlanId = planIdPaid })
                 );
+
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default))
+                .MustHaveHappened();
+
+            A.CallTo(() => appPlansBillingManager.SubscribeAsync(Actor.Identifier, AppNamedId, planIdPaid, default))
+                .MustHaveHappened();
         }
 
         [Fact]
@@ -243,7 +252,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
 
             var result = await PublishIdempotentAsync(command);
 
-            result.ShouldBeEquivalent(None.Value);
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid));
 
             Assert.Equal(planIdPaid, sut.Snapshot.Plan!.PlanId);
 
@@ -252,7 +261,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                     CreateEvent(new AppPlanChanged { PlanId = planIdPaid })
                 );
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(A<string>._, A<NamedId<DomainId>>._, A<string?>._, A<string?>._, A<CancellationToken>._))
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(A<string>._, A<NamedId<DomainId>>._, A<string?>._, A<CancellationToken>._))
+                .MustNotHaveHappened();
+
+            A.CallTo(() => appPlansBillingManager.SubscribeAsync(A<string>._, A<NamedId<DomainId>>._, A<string?>._, A<CancellationToken>._))
                 .MustNotHaveHappened();
         }
 
@@ -261,15 +273,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
         {
             var command = new ChangePlan { PlanId = planIdFree, FromCallback = true };
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default))
-                .Returns(new PlanChangedResult());
-
             await ExecuteCreateAsync();
             await ExecuteChangePlanAsync();
 
             var result = await PublishIdempotentAsync(command);
 
-            result.ShouldBeEquivalent(None.Value);
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdFree, true));
 
             Assert.Null(sut.Snapshot.Plan);
 
@@ -278,7 +287,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                     CreateEvent(new AppPlanReset())
                 );
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(A<string>._, A<NamedId<DomainId>>._, planIdFree, A<string?>._, A<CancellationToken>._))
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(A<string>._, A<NamedId<DomainId>>._, A<string?>._, A<CancellationToken>._))
+                .MustHaveHappenedOnceExactly();
+
+            A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(A<string>._, A<NamedId<DomainId>>._, A<CancellationToken>._))
                 .MustNotHaveHappened();
         }
 
@@ -287,18 +299,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
         {
             var command = new ChangePlan { PlanId = planIdFree };
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default))
-                .Returns(new PlanChangedResult());
-
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdFree, command.Referer, default))
-                .Returns(new PlanChangedResult());
-
             await ExecuteCreateAsync();
             await ExecuteChangePlanAsync();
 
             var result = await PublishIdempotentAsync(command);
 
-            Assert.True(result is PlanChangedResult);
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdFree, true));
 
             Assert.Null(sut.Snapshot.Plan);
 
@@ -306,6 +312,12 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                 .ShouldHaveSameEvents(
                     CreateEvent(new AppPlanReset())
                 );
+
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default))
+                .MustHaveHappenedOnceExactly();
+
+            A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(A<string>._, A<NamedId<DomainId>>._, A<CancellationToken>._))
+                .MustHaveHappened();
         }
 
         [Fact]
@@ -313,14 +325,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
         {
             var command = new ChangePlan { PlanId = planIdPaid };
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, command.Referer, default))
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, default))
-                .Returns(new RedirectToCheckoutResult(new Uri("http://squidex.io")));
+                .Returns(new Uri("http://squidex.io"));
 
             await ExecuteCreateAsync();
 
             var result = await PublishIdempotentAsync(command);
 
-            result.ShouldBeEquivalent(new RedirectToCheckoutResult(new Uri("http://squidex.io")));
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid, false, new Uri("http://squidex.io")));
 
             Assert.Null(sut.Snapshot.Plan);
         }
@@ -334,9 +346,14 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
 
             var result = await PublishIdempotentAsync(command);
 
-            result.ShouldBeEquivalent(None.Value);
+            result.ShouldBeEquivalent(new PlanChangedResult(planIdPaid));
+
+            Assert.Equal(planIdPaid, sut.Snapshot.Plan?.PlanId);
+
+            A.CallTo(() => appPlansBillingManager.MustRedirectToPortalAsync(Actor.Identifier, AppNamedId, planIdPaid, A<CancellationToken>._))
+                .MustNotHaveHappened();
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(Actor.Identifier, AppNamedId, planIdPaid, A<string?>._, default))
+            A.CallTo(() => appPlansBillingManager.SubscribeAsync(Actor.Identifier, AppNamedId, planIdPaid, A<CancellationToken>._))
                 .MustNotHaveHappened();
         }
 
@@ -651,7 +668,7 @@ namespace Squidex.Domain.Apps.Entities.Apps.DomainObject
                     CreateEvent(new AppDeleted())
                 );
 
-            A.CallTo(() => appPlansBillingManager.ChangePlanAsync(command.Actor.Identifier, AppNamedId, null, A<string?>._, default))
+            A.CallTo(() => appPlansBillingManager.UnsubscribeAsync(command.Actor.Identifier, AppNamedId, default))
                 .MustHaveHappened();
         }
 

backend/tests/Squidex.Domain.Apps.Entities.Tests/Apps/Plans/NoopAppPlanBillingManagerTests.cs

@@ -20,15 +20,31 @@ namespace Squidex.Domain.Apps.Entities.Apps.Plans
         }
 
         [Fact]
-        public async Task Should_do_nothing_if_changing_plan()
+        public async Task Should_do_nothing_if_subscribing()
         {
-            await sut.ChangePlanAsync(null!, null!, null, null);
+            await sut.SubscribeAsync(null!, null!, null!);
+        }
+
+        [Fact]
+        public async Task Should_do_nothing_if_unsubscribing()
+        {
+            await sut.UnsubscribeAsync(null!, null!);
         }
 
         [Fact]
         public async Task Should_not_return_portal_link()
         {
-            Assert.Equal(string.Empty, await sut.GetPortalLinkAsync(null!));
+            var result = await sut.GetPortalLinkAsync(null!);
+
+            Assert.Empty(result);
+        }
+
+        [Fact]
+        public async Task Should_do_nothing_if_checking_for_redirect()
+        {
+            var result = await sut.MustRedirectToPortalAsync(null!, null!, null);
+
+            Assert.Null(result);
         }
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/AssetUsageTrackerTests.cs

@@ -8,9 +8,11 @@
 using FakeItEasy;
 using FluentAssertions;
 using NodaTime;
+using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Domain.Apps.Events.Assets;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.EventSourcing;
+using Squidex.Infrastructure.States;
 using Squidex.Infrastructure.UsageTracking;
 using Xunit;
 
@@ -18,13 +20,23 @@ namespace Squidex.Domain.Apps.Entities.Assets
 {
     public class AssetUsageTrackerTests
     {
+        private readonly IAssetLoader assetLoader = A.Fake<IAssetLoader>();
+        private readonly ISnapshotStore<AssetUsageTracker.State> store = A.Fake<ISnapshotStore<AssetUsageTracker.State>>();
+        private readonly ITagService tagService = A.Fake<ITagService>();
         private readonly IUsageTracker usageTracker = A.Fake<IUsageTracker>();
         private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app");
+        private readonly DomainId assetId = DomainId.NewGuid();
+        private readonly DomainId assetKey;
         private readonly AssetUsageTracker sut;
 
         public AssetUsageTrackerTests()
         {
-            sut = new AssetUsageTracker(usageTracker);
+            assetKey = DomainId.Combine(appId, assetId);
+
+            A.CallTo(() => usageTracker.FallbackCategory)
+                .Returns("*");
+
+            sut = new AssetUsageTracker(usageTracker, assetLoader, tagService, store);
         }
 
         [Fact]
@@ -130,7 +142,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
             @event.AppId = appId;
 
             var envelope =
-                Envelope.Create(@event)
+                Envelope.Create<IEvent>(@event)
                     .SetTimestamp(Instant.FromDateTimeUtc(date));
 
             Counters? countersSummary = null;
@@ -142,7 +154,7 @@ namespace Squidex.Domain.Apps.Entities.Assets
             A.CallTo(() => usageTracker.TrackAsync(date, $"{appId.Id}_Assets", null, A<Counters>._, default))
                 .Invokes(x => countersDate = x.GetArgument<Counters>(3));
 
-            await sut.On(envelope);
+            await sut.On(new[] { envelope });
 
             var expected = new Counters
             {
@@ -153,5 +165,294 @@ namespace Squidex.Domain.Apps.Entities.Assets
             countersSummary.Should().BeEquivalentTo(expected);
             countersDate.Should().BeEquivalentTo(expected);
         }
+
+        [Fact]
+        public async Task Should_write_tags_when_asset_created()
+        {
+            var @event = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                },
+                AssetId = assetId
+            };
+
+            var envelope =
+                Envelope.Create<IEvent>(@event)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = 1,
+                ["tag2"] = 1
+            });
+        }
+
+        [Fact]
+        public async Task Should_group_tags_by_app()
+        {
+            var @event1 = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                },
+                AssetId = assetId
+            };
+
+            var @event2 = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag2",
+                    "tag3"
+                },
+                AssetId = assetId
+            };
+
+            var envelope1 =
+                Envelope.Create<IEvent>(@event1)
+                    .SetAggregateId(assetKey);
+
+            var envelope2 =
+                Envelope.Create<IEvent>(@event2)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope1, envelope2 });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = 1,
+                ["tag2"] = 2,
+                ["tag3"] = 1
+            });
+
+            A.CallTo(() => store.WriteManyAsync(A<IEnumerable<SnapshotWriteJob<AssetUsageTracker.State>>>._, default))
+                .MustHaveHappenedOnceExactly();
+        }
+
+        [Fact]
+        public async Task Should_merge_tags_with_previous_event_on_annotate()
+        {
+            var @event1 = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                },
+                AssetId = assetId
+            };
+
+            var @event2 = new AssetAnnotated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag2",
+                    "tag3"
+                },
+                AssetId = assetId
+            };
+
+            var envelope1 =
+                Envelope.Create<IEvent>(@event1)
+                    .SetAggregateId(assetKey);
+
+            var envelope2 =
+                Envelope.Create<IEvent>(@event2)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope1, envelope2 });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = 0,
+                ["tag2"] = 1,
+                ["tag3"] = 1
+            });
+        }
+
+        [Fact]
+        public async Task Should_merge_tags_with_previous_event_on_annotate_from_other_batch()
+        {
+            var @event1 = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                },
+                AssetId = assetId
+            };
+
+            var @event2 = new AssetAnnotated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag2",
+                    "tag3"
+                },
+                AssetId = assetId
+            };
+
+            var envelope1 =
+                Envelope.Create<IEvent>(@event1)
+                    .SetAggregateId(assetKey);
+
+            var envelope2 =
+                Envelope.Create<IEvent>(@event2)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope1 });
+            await sut.On(new[] { envelope2 });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = -1,
+                ["tag2"] = 0,
+                ["tag3"] = 1
+            });
+        }
+
+        [Fact]
+        public async Task Should_merge_tags_with_previous_event_on_delete()
+        {
+            var @event1 = new AssetCreated
+            {
+                AppId = appId,
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                },
+                AssetId = assetId
+            };
+
+            var @event2 = new AssetDeleted { AppId = appId, AssetId = assetId };
+
+            var envelope1 =
+                Envelope.Create<IEvent>(@event1)
+                    .SetAggregateId(assetKey);
+
+            var envelope2 =
+                Envelope.Create<IEvent>(@event2)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { Envelope.Create<IEvent>(@event1), Envelope.Create<IEvent>(@event2) });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = 0,
+                ["tag2"] = 0
+            });
+        }
+
+        [Fact]
+        public async Task Should_merge_tags_with_stored_state_if_previous_event_not_in_cached()
+        {
+            var state = new AssetUsageTracker.State
+            {
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                }
+            };
+
+            A.CallTo(() => store.ReadAsync(assetKey, default))
+                .Returns(new SnapshotResult<AssetUsageTracker.State>(assetKey, state, 0));
+
+            var @event = new AssetDeleted { AppId = appId, AssetId = assetId };
+
+            var envelope =
+                Envelope.Create<IEvent>(@event)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = -1,
+                ["tag2"] = -1
+            });
+        }
+
+        [Fact]
+        public async Task Should_merge_tags_with_asset_if_previous_tags_not_in_store()
+        {
+            IAssetEntity asset = new AssetEntity
+            {
+                Tags = new HashSet<string>
+                {
+                    "tag1",
+                    "tag2"
+                }
+            };
+
+            A.CallTo(() => assetLoader.GetAsync(appId.Id, assetId, 41, default))
+                .Returns(asset);
+
+            var @event = new AssetDeleted { AppId = appId, AssetId = assetId };
+
+            var envelope =
+                Envelope.Create<IEvent>(@event)
+                    .SetEventStreamNumber(42)
+                    .SetAggregateId(assetKey);
+
+            Dictionary<string, int>? update = null;
+
+            A.CallTo(() => tagService.UpdateAsync(appId.Id, TagGroups.Assets, A<Dictionary<string, int>>._, default))
+                .Invokes(x => { update = x.GetArgument<Dictionary<string, int>>(2); });
+
+            await sut.On(new[] { envelope });
+
+            update.Should().BeEquivalentTo(new Dictionary<string, int>
+            {
+                ["tag1"] = -1,
+                ["tag2"] = -1
+            });
+        }
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/BackupAssetsTests.cs

@@ -10,6 +10,7 @@ using Squidex.Assets;
 using Squidex.Domain.Apps.Core.Tags;
 using Squidex.Domain.Apps.Entities.Assets.DomainObject;
 using Squidex.Domain.Apps.Entities.Backup;
+using Squidex.Domain.Apps.Events.Apps;
 using Squidex.Domain.Apps.Events.Assets;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Commands;
@@ -97,13 +98,19 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             var context = CreateRestoreContext();
 
+            var envelope =
+                new Envelope<IEvent>(new AppCreated
+                {
+                    AppId = appId
+                });
+
             A.CallTo(() => context.Reader.HasFileAsync(A<string>._, ct))
                 .Returns(true);
 
             A.CallTo(() => context.Reader.ReadJsonAsync<Dictionary<string, Tag>>(A<string>._, ct))
                 .Returns(tags);
 
-            await sut.RestoreAsync(context, ct);
+            await sut.RestoreEventAsync(envelope, context, ct);
 
             A.CallTo(() => tagService.RebuildTagsAsync(appId.Id, TagGroups.Assets, A<TagsExport>.That.Matches(x => x.Tags == tags), ct))
                 .MustHaveHappened();
@@ -116,13 +123,19 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             var context = CreateRestoreContext();
 
+            var envelope =
+                new Envelope<IEvent>(new AppCreated
+                {
+                    AppId = appId
+                });
+
             A.CallTo(() => context.Reader.HasFileAsync(A<string>._, ct))
                 .Returns(false).Once().Then.Returns(true);
 
             A.CallTo(() => context.Reader.ReadJsonAsync<Dictionary<string, string>>(A<string>._, ct))
                 .Returns(alias);
 
-            await sut.RestoreAsync(context, ct);
+            await sut.RestoreEventAsync(envelope, context, ct);
 
             A.CallTo(() => tagService.RebuildTagsAsync(appId.Id, TagGroups.Assets, A<TagsExport>.That.Matches(x => x.Alias == alias), ct))
                 .MustHaveHappened();
@@ -135,13 +148,19 @@ namespace Squidex.Domain.Apps.Entities.Assets
 
             var context = CreateRestoreContext();
 
+            var envelope =
+                new Envelope<IEvent>(new AppCreated
+                {
+                    AppId = appId
+                });
+
             A.CallTo(() => context.Reader.HasFileAsync(A<string>._, ct))
                 .Returns(false);
 
             A.CallTo(() => context.Reader.ReadJsonAsync<Dictionary<string, string>>(A<string>._, ct))
                 .Returns(alias);
 
-            await sut.RestoreAsync(context, ct);
+            await sut.RestoreEventAsync(envelope, context, ct);
 
             A.CallTo(() => context.Reader.ReadJsonAsync<Dictionary<string, string>>(A<string>._, A<CancellationToken>._))
                 .MustNotHaveHappened();

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/DomainObject/AssetDomainObjectTests.cs

@@ -63,7 +63,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
             A.CallTo(() => assetQuery.FindAssetFolderAsync(AppId, parentId, A<CancellationToken>._))
                 .Returns(new List<IAssetFolderEntity> { A.Fake<IAssetFolderEntity>() });
 
-            A.CallTo(() => tagService.NormalizeTagsAsync(AppId, TagGroups.Assets, A<HashSet<string>>._, A<HashSet<string>>._, default))
+            A.CallTo(() => tagService.GetTagIdsAsync(AppId, TagGroups.Assets, A<HashSet<string>>._, default))
                 .ReturnsLazily(x => Task.FromResult(x.GetArgument<HashSet<string>>(2)?.ToDictionary(x => x) ?? new Dictionary<string, string>()));
 
             var log = A.Fake<ILogger<AssetDomainObject>>();
@@ -367,7 +367,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
         }
 
         [Fact]
-        public async Task Delete_should_create_events_with_total_file_size_and_update_deleted_flag()
+        public async Task Delete_should_create_events_with_total_file_size_and_tags_and_update_deleted_flag()
         {
             var command = new DeleteAsset();
 
@@ -382,7 +382,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.DomainObject
 
             LastEvents
                 .ShouldHaveSameEvents(
-                    CreateAssetEvent(new AssetDeleted { DeletedSize = 2048 })
+                    CreateAssetEvent(new AssetDeleted { DeletedSize = 2048, OldTags = new HashSet<string>() })
                 );
 
             A.CallTo(() => scriptEngine.ExecuteAsync(A<ScriptVars>._, "<delete-script>", ScriptOptions(), default))

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/Queries/AssetEnricherTests.cs

@@ -80,7 +80,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries
                 AppId = appId
             };
 
-            A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A<HashSet<string>>.That.Is("id1", "id2"), ct))
+            A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A<HashSet<string>>.That.Is("id1", "id2"), ct))
                 .Returns(new Dictionary<string, string>
                 {
                     ["id1"] = "name1",
@@ -158,7 +158,7 @@ namespace Squidex.Domain.Apps.Entities.Assets.Queries
                 AppId = appId
             };
 
-            A.CallTo(() => tagService.DenormalizeTagsAsync(appId.Id, TagGroups.Assets, A<HashSet<string>>.That.Is("id1", "id2", "id3"), ct))
+            A.CallTo(() => tagService.GetTagNamesAsync(appId.Id, TagGroups.Assets, A<HashSet<string>>.That.Is("id1", "id2", "id3"), ct))
                 .Returns(new Dictionary<string, string>
                 {
                     ["id1"] = "name1",

backend/tests/Squidex.Domain.Apps.Entities.Tests/Tags/TagServiceTests.cs

@@ -39,8 +39,9 @@ namespace Squidex.Domain.Apps.Entities.Tags
         [Fact]
         public async Task Should_delete_and_reset_state_if_cleaning()
         {
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct);
+            await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct);
+            await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct);
+
             await sut.ClearAsync(appId, group, ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
@@ -52,69 +53,76 @@ namespace Squidex.Domain.Apps.Entities.Tags
         }
 
         [Fact]
-        public async Task Should_rename_tag()
+        public async Task Should_unset_count_on_full_clear()
         {
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
 
-            await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct);
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
+            {
+                [ids["tag1"]] = 1,
+                [ids["tag2"]] = 1
+            }, ct);
 
-            // Forward the old name to the new name.
+            // Clear is called by the event consumer to fill the counts again, therefore we do not delete other things.
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            await sut.ClearAsync(ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new"), null, ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["tag1_new"] = 4
+                ["tag1"] = 0,
+                ["tag2"] = 0
             }, allTags);
+
+            A.CallTo(() => state.Persistence.DeleteAsync(ct))
+                .MustNotHaveHappened();
         }
 
         [Fact]
-        public async Task Should_rename_tag_twice()
+        public async Task Should_rename_tag()
         {
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
-
-            await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct);
 
-            // Rename again.
+            await sut.RenameTagAsync(appId, group, "tag1", "tag1_new", ct);
-            await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1_new2", ct);
 
             // Forward the old name to the new name.
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1_new"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new1"), null, ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1_new2"), null, ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
 
-            Assert.Equal(new Dictionary<string, int>
+            Assert.Equal(ids_0.Values, ids_1.Values);
-            {
+        }
-                ["tag1_new2"] = 5
+
-            }, allTags);
+        [Fact]
+        public async Task Should_rename_tag_twice()
+        {
+            var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct);
+
+            await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct);
+            await sut.RenameTagAsync(appId, group, "tag2", "tag3", ct);
+
+            // Forward the old name to the new name.
+            var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2"), ct);
+            var ids_2 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag3"), ct);
+
+            // Assert.Equal(ids_0.Values, ids_1.Values);
+            Assert.Equal(ids_0.Values, ids_2.Values);
         }
 
         [Fact]
         public async Task Should_rename_tag_back()
         {
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            var ids_0 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
 
-            await sut.RenameTagAsync(appId, group, "tag1", "tag1_new1", ct);
+            await sut.RenameTagAsync(appId, group, "tag1", "tag2", ct);
 
             // Rename back.
-            await sut.RenameTagAsync(appId, group, "tag1_new1", "tag1", ct);
+            await sut.RenameTagAsync(appId, group, "tag2", "tag1", ct);
 
             // Forward the old name to the new name.
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("tag1"), null, ct);
+            var ids_1 = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1"), ct);
-
-            var allTags = await sut.GetTagsAsync(appId, group, ct);
 
-            Assert.Equal(new Dictionary<string, int>
+            Assert.Equal(ids_0.Values, ids_1.Values);
-            {
-                ["tag1"] = 3
-            }, allTags);
         }
 
         [Fact]
@@ -124,10 +132,11 @@ namespace Squidex.Domain.Apps.Entities.Tags
             {
                 Tags = new Dictionary<string, Tag>
                 {
-                    ["id1"] = new Tag { Name = "name1", Count = 1 },
+                    ["id1"] = new Tag { Name = "tag1", Count = 1 },
-                    ["id2"] = new Tag { Name = "name2", Count = 2 },
+                    ["id2"] = new Tag { Name = "tag2", Count = 2 },
-                    ["id3"] = new Tag { Name = "name3", Count = 6 }
+                    ["id3"] = new Tag { Name = "tag3", Count = 6 }
-                }
+                },
+                Alias = null!
             };
 
             await sut.RebuildTagsAsync(appId, group, tags, ct);
@@ -136,76 +145,146 @@ namespace Squidex.Domain.Apps.Entities.Tags
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["name1"] = 1,
+                ["tag1"] = 1,
-                ["name2"] = 2,
+                ["tag2"] = 2,
-                ["name3"] = 6
+                ["tag3"] = 6
             }, allTags);
 
             var export = await sut.GetExportableTagsAsync(appId, group, ct);
 
-            export.Should().BeEquivalentTo(tags);
+            Assert.Equal(tags.Tags, export.Tags);
+            Assert.Empty(export.Alias);
+        }
+
+        [Fact]
+        public async Task Should_rebuild_with_broken_export()
+        {
+            var tags = new TagsExport
+            {
+                Alias = new Dictionary<string, string>
+                {
+                    ["id1"] = "id2"
+                },
+                Tags = null!
+            };
+
+            await sut.RebuildTagsAsync(appId, group, tags, ct);
+
+            var export = await sut.GetExportableTagsAsync(appId, group, ct);
+
+            Assert.Equal(tags.Alias, export.Alias);
+            Assert.Empty(export.Tags);
         }
 
         [Fact]
-        public async Task Should_add_tags()
+        public async Task Should_add_tag_but_not_count_tags()
         {
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct);
+            await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct);
-            await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct);
+            await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag2", "tag3"), ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["name1"] = 1,
+                ["tag1"] = 0,
-                ["name2"] = 2,
+                ["tag2"] = 0,
-                ["name3"] = 1
+                ["tag3"] = 0
             }, allTags);
         }
 
         [Fact]
-        public async Task Should_not_add_tags_if_already_added()
+        public async Task Should_add_and_increment_tags()
         {
-            var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct);
+            var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct);
-            var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2", "name3"), new HashSet<string>(result1.Values), ct);
+
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
+            {
+                [ids["tag1"]] = 1,
+                [ids["tag2"]] = 1
+            }, ct);
+
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
+            {
+                [ids["tag2"]] = 1,
+                [ids["tag3"]] = 1
+            }, ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["name1"] = 1,
+                ["tag1"] = 1,
-                ["name2"] = 1,
+                ["tag2"] = 2,
-                ["name3"] = 1
+                ["tag3"] = 1
             }, allTags);
         }
 
         [Fact]
-        public async Task Should_remove_tags()
+        public async Task Should_add_and_decrement_tags()
         {
-            var result1 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct);
+            var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2", "tag3"), ct);
-            var result2 = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name2", "name3"), null, ct);
+
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
+            {
+                [ids["tag1"]] = 1,
+                [ids["tag2"]] = 1
+            }, ct);
 
-            // Tags from the first normalization are decreased and removed if count reaches zero.
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
-            await sut.NormalizeTagsAsync(appId, group, null, new HashSet<string>(result1.Values), ct);
+            {
+                [ids["tag2"]] = -2,
+                [ids["tag3"]] = -2
+            }, ct);
 
             var allTags = await sut.GetTagsAsync(appId, group, ct);
 
             Assert.Equal(new Dictionary<string, int>
             {
-                ["name2"] = 1,
+                ["tag1"] = 1,
-                ["name3"] = 1
+                ["tag2"] = 0,
+                ["tag3"] = 0
             }, allTags);
         }
 
+        [Fact]
+        public async Task Should_not_update_non_existing_tags()
+        {
+            // We have no names for these IDs so we cannot update it.
+            await sut.UpdateAsync(appId, group, new Dictionary<string, int>
+            {
+                ["id1"] = 1,
+                ["id2"] = 1
+            }, ct);
+
+            var allTags = await sut.GetTagsAsync(appId, group, ct);
+
+            Assert.Empty(allTags);
+        }
+
         [Fact]
         public async Task Should_resolve_tag_names()
         {
             // Get IDs from names.
-            var tagIds = await sut.NormalizeTagsAsync(appId, group, HashSet.Of("name1", "name2"), null, ct);
+            var tagIds = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct);
 
             // Get names from IDs (reverse operation).
-            var tagNames = await sut.GetTagIdsAsync(appId, group, HashSet.Of("name1", "name2", "invalid1"), ct);
+            var tagNames = await sut.GetTagNamesAsync(appId, group, tagIds.Values.ToHashSet(), ct);
+
+            Assert.Equal(tagIds.Keys.ToArray(), tagNames.Values.ToArray());
+        }
+
+        [Fact]
+        public async Task Should_get_exportable_tags()
+        {
+            var ids = await sut.GetTagIdsAsync(appId, group, HashSet.Of("tag1", "tag2"), ct);
 
-            Assert.Equal(tagIds, tagNames);
+            var allTags = await sut.GetExportableTagsAsync(appId, group, ct);
+
+            allTags.Tags.Should().BeEquivalentTo(new Dictionary<string, Tag>
+            {
+                [ids["tag1"]] = new Tag { Name = "tag1", Count = 0 },
+                [ids["tag2"]] = new Tag { Name = "tag2", Count = 0 },
+            });
         }
     }
 }

backend/tests/Squidex.Infrastructure.Tests/CollectionExtensionsTests.cs

@@ -112,7 +112,7 @@ namespace Squidex.Infrastructure
         {
             valueDictionary[12] = 34;
 
-            Assert.Equal(34, valueDictionary.GetOrAdd(12, x => 34));
+            Assert.Equal(34, valueDictionary.GetOrAdd(12, x => 44));
         }
 
         [Fact]
@@ -129,24 +129,6 @@ namespace Squidex.Infrastructure
             Assert.Equal(24, valueDictionary[12]);
         }
 
-        [Fact]
-        public void GetOrNew_should_return_value_if_key_exists()
-        {
-            var list = new List<int>();
-            listDictionary[12] = list;
-
-            Assert.Equal(list, listDictionary.GetOrNew(12));
-        }
-
-        [Fact]
-        public void GetOrNew_should_return_default_but_not_add_it_if_key_not_exists()
-        {
-            var list = new List<int>();
-
-            Assert.Equal(list, listDictionary.GetOrNew(12));
-            Assert.False(listDictionary.ContainsKey(12));
-        }
-
         [Fact]
         public void GetOrAddNew_should_return_value_if_key_exists()
         {

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerManagerTests.cs

@@ -18,16 +18,26 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
     {
         private readonly IPersistenceFactory<EventConsumerState> persistenceFactory = A.Fake<IPersistenceFactory<EventConsumerState>>();
         private readonly IMessageBus messaging = A.Fake<IMessageBus>();
-        private readonly string consumerName = Guid.NewGuid().ToString();
+        private readonly string consumerName1 = Guid.NewGuid().ToString();
+        private readonly string consumerName2 = Guid.NewGuid().ToString();
         private readonly EventConsumerManager sut;
 
         public EventConsumerManagerTests()
         {
-            sut = new EventConsumerManager(persistenceFactory, messaging);
+            var consumer1 = A.Fake<IEventConsumer>();
+            var consumer2 = A.Fake<IEventConsumer>();
+
+            A.CallTo(() => consumer1.Name)
+                .Returns(consumerName1);
+
+            A.CallTo(() => consumer2.Name)
+                .Returns(consumerName2);
+
+            sut = new EventConsumerManager(persistenceFactory, new[] { consumer1, consumer2 }, messaging);
         }
 
         [Fact]
-        public async Task Should_get_states_from_store()
+        public async Task Should_get_states_from_store_without_old_consumer()
         {
             var snapshotStore = A.Fake<ISnapshotStore<EventConsumerState>>();
 
@@ -37,12 +47,17 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
             A.CallTo(() => snapshotStore.ReadAllAsync(default))
                 .Returns(new List<SnapshotResult<EventConsumerState>>
                 {
-                    new SnapshotResult<EventConsumerState>(DomainId.Create("consumer1"),
+                    new SnapshotResult<EventConsumerState>(DomainId.Create(consumerName1),
                         new EventConsumerState
                         {
                             Position = "1"
                         }, 1),
-                    new SnapshotResult<EventConsumerState>(DomainId.Create("consumer2"),
+                    new SnapshotResult<EventConsumerState>(DomainId.Create(consumerName2),
+                        new EventConsumerState
+                        {
+                            Position = "2"
+                        }, 2),
+                    new SnapshotResult<EventConsumerState>(DomainId.Create("oldConsumer"),
                         new EventConsumerState
                         {
                             Position = "2"
@@ -54,25 +69,32 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
             result.Should().BeEquivalentTo(
                 new List<EventConsumerInfo>
                 {
-                    new EventConsumerInfo { Name = "consumer1", Position = "1" },
+                    new EventConsumerInfo { Name = consumerName1, Position = "1" },
-                    new EventConsumerInfo { Name = "consumer2", Position = "2" }
+                    new EventConsumerInfo { Name = consumerName2, Position = "2" }
                 });
         }
 
+        [Fact]
+        public async Task Should_throw_exception_when_calling_old_consumer()
+        {
+            await Assert.ThrowsAsync<DomainObjectNotFoundException>(() => sut.StartAsync("oldConsumer", default));
+        }
+
         [Fact]
         public async Task Should_publish_event_on_start()
         {
-            var testState = new TestState<EventConsumerState>(DomainId.Create(consumerName), persistenceFactory)
+            var testState = new TestState<EventConsumerState>(consumerName1, persistenceFactory)
             {
                 Snapshot = new EventConsumerState
                 {
                     Position = "42"
-                }
+                },
+                Version = 0
             };
 
-            var response = await sut.StartAsync(consumerName, default);
+            var response = await sut.StartAsync(consumerName1, default);
 
-            A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName), null, default))
+            A.CallTo(() => messaging.PublishAsync(new EventConsumerStart(consumerName1), null, default))
                 .MustHaveHappened();
 
             Assert.Equal("42", response.Position);
@@ -81,17 +103,18 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
         [Fact]
         public async Task Should_publish_event_on_stop()
         {
-            var testState = new TestState<EventConsumerState>(DomainId.Create(consumerName), persistenceFactory)
+            var testState = new TestState<EventConsumerState>(consumerName1, persistenceFactory)
             {
                 Snapshot = new EventConsumerState
                 {
                     Position = "42"
-                }
+                },
+                Version = 0
             };
 
-            var response = await sut.StopAsync(consumerName, default);
+            var response = await sut.StopAsync(consumerName1, default);
 
-            A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName), null, default))
+            A.CallTo(() => messaging.PublishAsync(new EventConsumerStop(consumerName1), null, default))
                 .MustHaveHappened();
 
             Assert.Equal("42", response.Position);
@@ -100,17 +123,18 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
         [Fact]
         public async Task Should_publish_event_on_reset()
         {
-            var testState = new TestState<EventConsumerState>(DomainId.Create(consumerName), persistenceFactory)
+            var testState = new TestState<EventConsumerState>(consumerName1, persistenceFactory)
             {
                 Snapshot = new EventConsumerState
                 {
                     Position = "42"
-                }
+                },
+                Version = 0
             };
 
-            var response = await sut.ResetAsync(consumerName, default);
+            var response = await sut.ResetAsync(consumerName1, default);
 
-            A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName), null, default))
+            A.CallTo(() => messaging.PublishAsync(new EventConsumerReset(consumerName1), null, default))
                 .MustHaveHappened();
 
             Assert.Equal("42", response.Position);

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Consume/EventConsumerProcessorTests.cs

@@ -71,6 +71,9 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
             A.CallTo(() => eventConsumer.Name)
                 .Returns(consumerName);
 
+            A.CallTo(() => eventConsumer.CanClear)
+                .Returns(true);
+
             A.CallTo(() => eventConsumer.Handles(A<StoredEvent>._))
                 .Returns(true);
 
@@ -205,6 +208,26 @@ namespace Squidex.Infrastructure.EventSourcing.Consume
                 .MustHaveHappenedOnceExactly();
         }
 
+        [Fact]
+        public async Task Should_not_reset_consumer_if_not_allowed()
+        {
+            A.CallTo(() => eventConsumer.CanClear)
+                .Returns(false);
+
+            await sut.InitializeAsync(default);
+            await sut.ActivateAsync();
+
+            await sut.StopAsync();
+            await sut.ResetAsync();
+
+            await sut.CompleteAsync();
+
+            AssertGrainState(isStopped: true, position: initialPosition);
+
+            A.CallTo(() => eventConsumer.ClearAsync())
+                .MustNotHaveHappened();
+        }
+
         [Fact]
         public async Task Should_invoke_and_update_position_if_event_received()
         {

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs

@@ -50,8 +50,6 @@ namespace Squidex.Infrastructure.EventSourcing
             get => sut.Value;
         }
 
-        protected abstract int SubscriptionDelayInMs { get; }
-
         protected EventStoreTests()
         {
 #pragma warning disable MA0056 // Do not call overridable members in constructor
@@ -318,15 +316,12 @@ namespace Squidex.Infrastructure.EventSourcing
 
             var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray();
 
-            var takeStep = count / 10;
+            for (var take = 0; take < count; take += count / 10)
-
-            for (var take = 0; take < count; take += takeStep)
             {
-                var expected = allExpected.TakeLast(take).ToArray();
+                var eventsExpected = allExpected.TakeLast(take).ToArray();
+                var eventsQueried = await Sut.QueryReverseAsync(streamName, take);
 
-                var readEvents = await Sut.QueryReverseAsync(streamName, take);
+                ShouldBeEquivalentTo(eventsQueried, eventsExpected);
-
-                ShouldBeEquivalentTo(readEvents, expected);
             }
         }
 
@@ -353,15 +348,12 @@ namespace Squidex.Infrastructure.EventSourcing
 
             var allExpected = events.Select((x, i) => new StoredEvent(streamName, "Position", i, events[i])).ToArray();
 
-            var takeStep = count / 10;
+            for (var take = 0; take < count; take += count / 10)
-
-            for (var take = 0; take < count; take += takeStep)
             {
-                var expected = allExpected.Reverse().Take(take).ToArray();
+                var eventsExpected = allExpected.Reverse().Take(take).ToArray();
+                var eventsQueried = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync();
 
-                var readEvents = await Sut.QueryAllReverseAsync(streamName, default, take).ToArrayAsync();
+                ShouldBeEquivalentTo(eventsQueried, eventsExpected);
-
-                ShouldBeEquivalentTo(readEvents, expected);
             }
         }
 
@@ -438,7 +430,12 @@ namespace Squidex.Infrastructure.EventSourcing
 
         private static EventData CreateEventData(int i)
         {
-            return new EventData($"Type{i}", new EnvelopeHeaders(), i.ToString(CultureInfo.InvariantCulture));
+            var headers = new EnvelopeHeaders
+            {
+                [CommonHeaders.EventId] = Guid.NewGuid().ToString()
+            };
+
+            return new EventData($"Type{i}", headers, i.ToString(CultureInfo.InvariantCulture));
         }
 
         private async Task<IReadOnlyList<StoredEvent>?> QueryAllAsync(string? streamFilter = null, string? position = null)

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/GetEventStoreTests.cs

@@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing
     {
         public GetEventStoreFixture _ { get; }
 
-        protected override int SubscriptionDelayInMs { get; } = 1000;
-
         public GetEventStoreTests(GetEventStoreFixture fixture)
         {
             _ = fixture;

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_Direct.cs

@@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing
     {
         public MongoEventStoreFixture _ { get; }
 
-        protected override int SubscriptionDelayInMs { get; } = 1000;
-
         public MongoEventStoreTests_Direct(MongoEventStoreDirectFixture fixture)
         {
             _ = fixture;

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoEventStoreTests_ReplicaSet.cs

@@ -16,8 +16,6 @@ namespace Squidex.Infrastructure.EventSourcing
     {
         public MongoEventStoreFixture _ { get; }
 
-        protected override int SubscriptionDelayInMs { get; } = 1000;
-
         public MongoEventStoreTests_ReplicaSet(MongoEventStoreReplicaSetFixture fixture)
         {
             _ = fixture;

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs

@@ -5,7 +5,9 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Globalization;
 using FakeItEasy;
+using FluentAssertions;
 using Xunit;
 
 namespace Squidex.Infrastructure.EventSourcing
@@ -15,61 +17,54 @@ namespace Squidex.Infrastructure.EventSourcing
         private readonly IEventStore eventStore = A.Fake<IEventStore>();
         private readonly IEventSubscriber<StoredEvent> eventSubscriber = A.Fake<IEventSubscriber<StoredEvent>>();
         private readonly string position = Guid.NewGuid().ToString();
+        private readonly string filter = "^my-stream";
 
         [Fact]
         public async Task Should_subscribe_on_start()
         {
-            var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position);
+            await SubscribeAsync(false);
 
-            await WaitAndStopAsync(sut);
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
-
-            A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A<int>._, A<CancellationToken>._))
                 .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
-        public async Task Should_propagate_exception_to_subscriber()
+        public async Task Should_forward_exception_to_subscriber()
         {
             var ex = new InvalidOperationException();
 
-            A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A<int>._, A<CancellationToken>._))
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
                 .Throws(ex);
 
-            var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position);
+            var sut = await SubscribeAsync(false);
-
-            await WaitAndStopAsync(sut);
 
             A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_propagate_operation_cancelled_exception_to_subscriber()
+        public async Task Should_forward_operation_cancelled_exception_to_subscriber()
         {
             var ex = new OperationCanceledException();
 
-            A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A<int>._, A<CancellationToken>._))
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
                 .Throws(ex);
 
-            var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position);
+            var sut = await SubscribeAsync(false);
-
-            await WaitAndStopAsync(sut);
 
             A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_propagate_aggregate_operation_cancelled_exception_to_subscriber()
+        public async Task Should_forward_aggregate_operation_cancelled_exception_to_subscriber()
         {
             var ex = new AggregateException(new OperationCanceledException());
 
-            A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A<int>._, A<CancellationToken>._))
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
                 .Throws(ex);
 
-            var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position);
+            var sut = await SubscribeAsync(false);
-
-            await WaitAndStopAsync(sut);
 
             A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex))
                 .MustHaveHappened();
@@ -78,21 +73,88 @@ namespace Squidex.Infrastructure.EventSourcing
         [Fact]
         public async Task Should_wake_up()
         {
-            var sut = new PollingSubscription(eventStore, eventSubscriber, "^my-stream", position);
+            var sut = await SubscribeAsync(true);
 
-            sut.WakeUp();
+            A.CallTo(() => eventStore.QueryAllAsync(filter, A<string>._, A<int>._, A<CancellationToken>._))
+                .MustHaveHappened(2, Times.Exactly);
+        }
 
-            await WaitAndStopAsync(sut);
+        [Fact]
+        public async Task Should_forward_events_to_subscriber()
+        {
+            var events = Enumerable.Range(0, 50).Select(CreateEvent).ToArray();
 
-            A.CallTo(() => eventStore.QueryAllAsync("^my-stream", position, A<int>._, A<CancellationToken>._))
+            var receivedEvents = new List<StoredEvent>();
-                .MustHaveHappened(2, Times.Exactly);
+
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
+                .Returns(events.ToAsyncEnumerable());
+
+            A.CallTo(() => eventSubscriber.OnNextAsync(A<IEventSubscription>._, A<StoredEvent>._))
+                .Invokes(x => receivedEvents.Add(x.GetArgument<StoredEvent>(1)!));
+
+            await SubscribeAsync(true);
+
+            receivedEvents.Should().BeEquivalentTo(events);
         }
 
-        private static async Task WaitAndStopAsync(IEventSubscription sut)
+        [Fact]
+        public async Task Should_continue_on_last_position()
         {
-            await Task.Delay(200);
+            var events1 = Enumerable.Range(10, 10).Select(CreateEvent).ToArray();
+            var events2 = Enumerable.Range(20, 10).Select(CreateEvent).ToArray();
+
+            var lastPosition = events1[^1].EventPosition;
+
+            var receivedEvents = new List<StoredEvent>();
+
+            A.CallTo(() => eventStore.QueryAllAsync(filter, position, A<int>._, A<CancellationToken>._))
+                .Returns(events1.ToAsyncEnumerable());
+
+            A.CallTo(() => eventStore.QueryAllAsync(filter, lastPosition, A<int>._, A<CancellationToken>._))
+                .Returns(events2.ToAsyncEnumerable());
+
+            A.CallTo(() => eventSubscriber.OnNextAsync(A<IEventSubscription>._, A<StoredEvent>._))
+                .Invokes(x => receivedEvents.Add(x.GetArgument<StoredEvent>(1)!));
+
+            await SubscribeAsync(true);
 
-            sut.Dispose();
+            receivedEvents.Should().BeEquivalentTo(events1.Union(events2));
+        }
+
+        private StoredEvent CreateEvent(int position)
+        {
+            return new StoredEvent(
+                "my-stream",
+                position.ToString(CultureInfo.InvariantCulture)!,
+                position,
+                new EventData(
+                    "type",
+                    new EnvelopeHeaders
+                    {
+                        [CommonHeaders.EventId] = Guid.NewGuid().ToString()
+                    },
+                    "payload"));
+        }
+
+        private async Task<IEventSubscription> SubscribeAsync(bool wakeup = true)
+        {
+            var sut = new PollingSubscription(eventStore, eventSubscriber, filter, position);
+
+            try
+            {
+                if (wakeup)
+                {
+                    sut.WakeUp();
+                }
+
+                await Task.Delay(200);
+            }
+            finally
+            {
+                sut.Dispose();
+            }
+
+            return sut;
         }
     }
 }

backend/tests/Squidex.Infrastructure.Tests/States/SimpleStateTests.cs

@@ -0,0 +1,208 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using FakeItEasy;
+using Squidex.Infrastructure.TestHelpers;
+using Xunit;
+
+namespace Squidex.Infrastructure.States
+{
+    public class SimpleStateTests
+    {
+        private readonly CancellationTokenSource cts = new CancellationTokenSource();
+        private readonly CancellationToken ct;
+        private readonly TestState<MyDomainState> testState = new TestState<MyDomainState>(DomainId.NewGuid());
+        private readonly SimpleState<MyDomainState> sut;
+
+        public SimpleStateTests()
+        {
+            ct = cts.Token;
+
+            sut = new SimpleState<MyDomainState>(testState.PersistenceFactory, GetType(), testState.Id);
+        }
+
+        [Fact]
+        public void Should_init_with_base_data()
+        {
+            Assert.Equal(-1, sut.Version);
+            Assert.NotNull(sut.Value);
+        }
+
+        [Fact]
+        public async Task Should_get_state_from_persistence_on_load()
+        {
+            testState.Version = 42;
+            testState.Snapshot = new MyDomainState { Value = 50 };
+
+            await sut.LoadAsync(ct);
+
+            Assert.Equal(42, sut.Version);
+            Assert.Equal(50, sut.Value.Value);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(-2, ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_delete_when_clearing()
+        {
+            await sut.ClearAsync(ct);
+
+            A.CallTo(() => testState.Persistence.DeleteAsync(ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_invoke_persistence_when_writing_state()
+        {
+            await sut.WriteAsync(ct);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_load_once_on_update()
+        {
+            await sut.UpdateAsync(x => true, ct: ct);
+            await sut.UpdateAsync(x => true, ct: ct);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(-2, ct))
+                .MustHaveHappenedOnceExactly();
+        }
+
+        [Fact]
+        public async Task Should_write_state_on_update_when_callback_returns_true()
+        {
+            await sut.UpdateAsync(x => true, ct: ct);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_not_write_state_on_update_when_callback_returns_false()
+        {
+            await sut.UpdateAsync(x => true, ct: ct);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_write_state_on_update_and_return_when_callback_returns_true()
+        {
+            var result = await sut.UpdateAsync(x => (true, 42), ct: ct);
+
+            Assert.Equal(42, result);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_not_write_state_on_update_and_return_when_callback_returns_false()
+        {
+            var result = await sut.UpdateAsync(x => (false, 42), ct: ct);
+
+            Assert.Equal(42, result);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustNotHaveHappened();
+        }
+
+        [Fact]
+        public async Task Should_retry_update_when_failed_with_inconsistency_issue()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5);
+
+            await sut.UpdateAsync(x => true, ct: ct);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 6);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 6);
+        }
+
+        [Fact]
+        public async Task Should_give_up_update_after_too_many_inconsistency_issues()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100);
+
+            await Assert.ThrowsAsync<InconsistentStateException>(() => sut.UpdateAsync(x => true, ct: ct));
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 20);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 20);
+        }
+
+        [Fact]
+        public async Task Should_not_retry_update_with_other_exception()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InvalidOperationException());
+
+            await Assert.ThrowsAsync<InvalidOperationException>(() => sut.UpdateAsync(x => true, ct: ct));
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 1);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 1);
+        }
+
+        [Fact]
+        public async Task Should_retry_update_and_return_when_failed_with_inconsistency_issue()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(5);
+
+            await sut.UpdateAsync(x => (true, 42), ct: ct);
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 6);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 6);
+        }
+
+        [Fact]
+        public async Task Should_give_up_update_and_return_after_too_many_inconsistency_issues()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InconsistentStateException(1, 2)).NumberOfTimes(100);
+
+            await Assert.ThrowsAsync<InconsistentStateException>(() => sut.UpdateAsync(x => (true, 42), ct: ct));
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 20);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 20);
+        }
+
+        [Fact]
+        public async Task Should_not_retry_update_and_return_with_other_exception()
+        {
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .Throws(new InvalidOperationException());
+
+            await Assert.ThrowsAsync<InvalidOperationException>(() => sut.UpdateAsync(x => (true, 42), ct: ct));
+
+            A.CallTo(() => testState.Persistence.WriteSnapshotAsync(sut.Value, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 1);
+
+            A.CallTo(() => testState.Persistence.ReadAsync(A<long>._, ct))
+                .MustHaveHappenedANumberOfTimesMatching(x => x == 1);
+        }
+    }
+}

backend/tests/Squidex.Infrastructure.Tests/TestHelpers/TestState.cs

@@ -14,9 +14,12 @@ namespace Squidex.Infrastructure.TestHelpers
     public sealed class TestState<T> where T : class, new()
     {
         private readonly List<Envelope<IEvent>> events = new List<Envelope<IEvent>>();
+        private readonly ISnapshotStore<T> snapshotStore = A.Fake<ISnapshotStore<T>>();
         private HandleSnapshot<T>? handleSnapshot;
         private HandleEvent? handleEvent;
 
+        public DomainId Id { get; }
+
         public IPersistenceFactory<T> PersistenceFactory { get; }
 
         public IPersistence<T> Persistence { get; } = A.Fake<IPersistence<T>>();
@@ -30,20 +33,24 @@ namespace Squidex.Infrastructure.TestHelpers
         {
         }
 
-        public void AddEvent(Envelope<IEvent> @event)
-        {
-            events.Add(@event);
-        }
-
-        public void AddEvent(IEvent @event)
-        {
-            events.Add(Envelope.Create(@event));
-        }
-
         public TestState(DomainId id, IPersistenceFactory<T>? persistenceFactory = null)
         {
+            Id = id;
+
             PersistenceFactory = persistenceFactory ?? A.Fake<IPersistenceFactory<T>>();
 
+            A.CallTo(() => PersistenceFactory.Snapshots)
+                .Returns(snapshotStore);
+
+            A.CallTo(() => Persistence.Version)
+                .ReturnsLazily(() => Version);
+
+            A.CallTo(() => snapshotStore.ReadAllAsync(A<CancellationToken>._))
+                .ReturnsLazily(() => new List<SnapshotResult<T>>
+                {
+                    new SnapshotResult<T>(id, Snapshot, Version, true)
+                }.ToAsyncEnumerable());
+
             A.CallTo(() => PersistenceFactory.WithEventSourcing(A<Type>._, id, A<HandleEvent>._))
                 .Invokes(x =>
                 {
@@ -98,5 +105,15 @@ namespace Squidex.Infrastructure.TestHelpers
                     Snapshot = new T();
                 });
         }
+
+        public void AddEvent(Envelope<IEvent> @event)
+        {
+            events.Add(@event);
+        }
+
+        public void AddEvent(IEvent @event)
+        {
+            events.Add(Envelope.Create(@event));
+        }
     }
 }

backend/tests/Squidex.Infrastructure.Tests/UsageTracking/BackgroundUsageTrackerTests.cs

@@ -65,6 +65,21 @@ namespace Squidex.Infrastructure.UsageTracking
             await Assert.ThrowsAsync<ObjectDisposedException>(() => sut.GetAsync(key, date, date, null, ct));
         }
 
+        [Fact]
+        public void Should_provide_fallback_category()
+        {
+            Assert.Equal("*", sut.FallbackCategory);
+        }
+
+        [Fact]
+        public async Task Should_forward_delete_prefix_call()
+        {
+            await sut.DeleteByKeyPatternAsync("pattern", ct);
+
+            A.CallTo(() => usageStore.DeleteByKeyPatternAsync("pattern", ct))
+                .MustHaveHappened();
+        }
+
         [Fact]
         public async Task Should_forward_delete_call()
         {

backend/tests/Squidex.Infrastructure.Tests/UsageTracking/CachingUsageTrackerTests.cs

@@ -30,6 +30,24 @@ namespace Squidex.Infrastructure.UsageTracking
             sut = new CachingUsageTracker(inner, cache);
         }
 
+        [Fact]
+        public void Should_forward_fallback_category()
+        {
+            A.CallTo(() => inner.FallbackCategory)
+                .Returns("*");
+
+            Assert.Equal("*", sut.FallbackCategory);
+        }
+
+        [Fact]
+        public async Task Should_forward_delete_prefix_call()
+        {
+            await sut.DeleteByKeyPatternAsync("pattern", ct);
+
+            A.CallTo(() => inner.DeleteByKeyPatternAsync("pattern", ct))
+                .MustHaveHappened();
+        }
+
         [Fact]
         public async Task Should_forward_delete_call()
         {

backend/tools/TestSuite/TestSuite.ApiTests/AssetTests.cs

@@ -319,6 +319,63 @@ namespace TestSuite.ApiTests
             Assert.Equal(fileNameRequest.FileName, asset_4.FileName);
         }
 
+        [Fact]
+        public async Task Should_annotate_asset_in_parallel()
+        {
+            var tag1 = $"tag_{Guid.NewGuid()}";
+            var tag2 = $"tag_{Guid.NewGuid()}";
+
+            var metadataRequest = new AnnotateAssetDto
+            {
+                Tags = new List<string>
+                {
+                    tag1,
+                    tag2
+                }
+            };
+
+
+            // STEP 1: Create asset
+            var asset_1 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png");
+
+
+            var numErrors = 0;
+            var numSuccess = 0;
+
+            // STEP 3: Make parallel upserts.
+            await Parallel.ForEachAsync(Enumerable.Range(0, 20), async (i, ct) =>
+            {
+                try
+                {
+                    await _.Assets.PutAssetAsync(_.AppName, asset_1.Id, metadataRequest);
+
+                    Interlocked.Increment(ref numSuccess);
+                }
+                catch (SquidexManagementException ex) when (ex.StatusCode is 409 or 412)
+                {
+                    Interlocked.Increment(ref numErrors);
+                    return;
+                }
+            });
+
+            // At least some errors and success should have happened.
+            Assert.True(numErrors > 0);
+            Assert.True(numSuccess > 0);
+
+
+            // STEP 3: Make an normal update to ensure nothing is corrupt.
+            await _.Assets.PutAssetAsync(_.AppName, asset_1.Id, metadataRequest);
+
+
+            // STEP 4: Check tags
+            var tags = await _.Assets.WaitForTagsAsync(_.AppName, tag1, TimeSpan.FromMinutes(2));
+
+            Assert.Contains(tag1, tags);
+            Assert.Contains(tag2, tags);
+            Assert.Equal(1, tags[tag1]);
+            Assert.Equal(1, tags[tag2]);
+        }
+
         [Fact]
         public async Task Should_protect_asset()
         {
@@ -490,10 +547,18 @@ namespace TestSuite.ApiTests
             var asset_1 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png", null, folder_2.Id);
 
 
-            // STEP 4: Delete folder.
+            // STEP 4: Create asset outside folder
+            var asset_2 = await _.UploadFileAsync("Assets/logo-squared.png", "image/png");
+
+
+            // STEP 5: Delete folder.
             await _.Assets.DeleteAssetFolderAsync(_.AppName, folder_1.Id);
 
+            // Ensure that asset in folder is deleted.
             Assert.True(await _.Assets.WaitForDeletionAsync(_.AppName, asset_1.Id, TimeSpan.FromSeconds(30)));
+
+            // Ensure that other asset is not deleted.
+            Assert.NotNull(await _.Assets.GetAssetAsync(_.AppName, asset_2.Id));
         }
 
         [Theory]

backend/tools/TestSuite/TestSuite.Shared/ClientExtensions.cs

@@ -38,6 +38,31 @@ namespace TestSuite
             return false;
         }
 
+        public static async Task<IDictionary<string, int>> WaitForTagsAsync(this IAssetsClient assetsClient, string app, string id, TimeSpan timeout)
+        {
+            try
+            {
+                using var cts = new CancellationTokenSource(timeout);
+
+                while (!cts.IsCancellationRequested)
+                {
+                    var tags = await assetsClient.GetTagsAsync(app, cts.Token);
+
+                    if (tags.TryGetValue(id, out var count) && count > 0)
+                    {
+                        return tags;
+                    }
+
+                    await Task.Delay(200, cts.Token);
+                }
+            }
+            catch (OperationCanceledException)
+            {
+            }
+
+            return await assetsClient.GetTagsAsync(app);
+        }
+
         public static async Task<BackupJobDto> WaitForBackupAsync(this IBackupsClient backupsClient, string app, TimeSpan timeout)
         {
             try