Partitioning (#1054)

* Sharding V1

* Revert naming.

* Revert more changes.

* Fix text index.

* Full text improvement.

* Improved full text.

* Update names.

* Revert search.

backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs

@@ -60,10 +60,10 @@ public static class CommandFactory
         {
             var document = new SearchDocument
             {
-                ["docId"] = upsert.DocId.ToBase64(),
+                ["docId"] = upsert.ToDocId(),
-                ["appId"] = upsert.AppId.Id.ToString(),
+                ["appId"] = upsert.UniqueContentId.AppId.ToString(),
-                ["appName"] = upsert.AppId.Name,
+                ["appName"] = string.Empty,
-                ["contentId"] = upsert.ContentId.ToString(),
+                ["contentId"] = upsert.UniqueContentId.ToString(),
                 ["schemaId"] = upsert.SchemaId.Id.ToString(),
                 ["schemaName"] = upsert.SchemaId.Name,
                 ["serveAll"] = upsert.ServeAll,
@@ -94,7 +94,7 @@ public static class CommandFactory
     {
         var document = new SearchDocument
         {
-            ["docId"] = update.DocId.ToBase64(),
+            ["docId"] = update.ToDocId(),
             ["serveAll"] = update.ServeAll,
             ["servePublished"] = update.ServePublished
         };
@@ -104,7 +104,7 @@ public static class CommandFactory
 
     private static void DeleteEntry(DeleteIndexEntry delete, IList<IndexDocumentsAction<SearchDocument>> batch)
     {
-        batch.Add(IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64()));
+        batch.Add(IndexDocumentsAction.Delete("docId", delete.ToDocId().ToBase64()));
     }
 
     private static string ToBase64(this string value)

backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs

@@ -56,7 +56,7 @@ public static class CommandFactory
             {
                 index = new
                 {
-                    _id = upsert.DocId,
+                    _id = upsert.ToDocId(),
                     _index = indexName
                 }
             });
@@ -82,9 +82,9 @@ public static class CommandFactory
 
             args.Add(new
             {
-                appId = upsert.AppId.Id.ToString(),
+                appId = upsert.UniqueContentId.AppId.ToString(),
-                appName = upsert.AppId.Name,
+                appName = string.Empty,
-                contentId = upsert.ContentId.ToString(),
+                contentId = upsert.UniqueContentId.ContentId.ToString(),
                 schemaId = upsert.SchemaId.Id.ToString(),
                 schemaName = upsert.SchemaId.Name,
                 serveAll = upsert.ServeAll,
@@ -102,7 +102,7 @@ public static class CommandFactory
         {
             update = new
             {
-                _id = update.DocId,
+                _id = update.ToDocId(),
                 _index = indexName
             }
         });
@@ -123,7 +123,7 @@ public static class CommandFactory
         {
             delete = new
             {
-                _id = delete.DocId,
+                _id = delete.ToDocId(),
                 _index = indexName
             }
         });

backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs

@@ -16,7 +16,7 @@ namespace Migrations.Migrations.MongoDb;
 
 public sealed class ConvertDocumentIds : MongoBase<BsonDocument>, IMigration
 {
-    private readonly IMongoDatabase database;
+    private readonly IMongoDatabase databaseDefault;
     private readonly IMongoDatabase databaseContent;
     private Scope scope;
 
@@ -27,9 +27,9 @@ public sealed class ConvertDocumentIds : MongoBase<BsonDocument>, IMigration
         Contents
     }
 
-    public ConvertDocumentIds(IMongoDatabase database, IMongoDatabase databaseContent)
+    public ConvertDocumentIds(IMongoDatabase databaseDefault, IMongoDatabase databaseContent)
     {
-        this.database = database;
+        this.databaseDefault = databaseDefault;
         this.databaseContent = databaseContent;
     }
 
@@ -58,8 +58,8 @@ public sealed class ConvertDocumentIds : MongoBase<BsonDocument>, IMigration
         switch (scope)
         {
             case Scope.Assets:
-                await RebuildAsync(database, ConvertParentId, "States_Assets", ct);
+                await RebuildAsync(databaseDefault, ConvertParentId, "States_Assets", ct);
-                await RebuildAsync(database, ConvertParentId, "States_AssetFolders", ct);
+                await RebuildAsync(databaseDefault, ConvertParentId, "States_AssetFolders", ct);
                 break;
             case Scope.Contents:
                 await RebuildAsync(databaseContent, null, "State_Contents_All", ct);

backend/src/Squidex.Domain.Apps.Core.Model/Contents/Json/WorkflowTransitionSurrogate.cs

@@ -5,7 +5,6 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using Squidex.Domain.Apps.Core.Apps;
 using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Core.Contents.Json;

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs

@@ -11,7 +11,6 @@ using Squidex.Domain.Apps.Core.Assets;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
 using Squidex.Infrastructure.States;
-using Squidex.Infrastructure.Translations;
 
 #pragma warning disable MA0048 // File name must match type name
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs

@@ -21,16 +21,19 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets;
 public sealed partial class MongoAssetRepository : MongoRepositoryBase<MongoAssetEntity>, IAssetRepository
 {
     private readonly MongoCountCollection countCollection;
+    private readonly string shardKey;
 
     static MongoAssetRepository()
     {
         MongoAssetEntity.RegisterClassMap();
     }
 
-    public MongoAssetRepository(IMongoDatabase database, ILogger<MongoAssetRepository> log)
+    public MongoAssetRepository(IMongoDatabase database, ILogger<MongoAssetRepository> log, string shardKey)
         : base(database)
     {
         countCollection = new MongoCountCollection(database, log, CollectionName());
+
+        this.shardKey = shardKey;
     }
 
     public IMongoCollection<MongoAssetEntity> GetInternalCollection()
@@ -40,7 +43,7 @@ public sealed partial class MongoAssetRepository : MongoRepositoryBase<MongoAsse
 
     protected override string CollectionName()
     {
-        return "States_Assets2";
+        return $"States_Assets2{shardKey}";
     }
 
     protected override Task SetupCollectionAsync(IMongoCollection<MongoAssetEntity> collection,

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoShardedAssetRepository.cs

@@ -0,0 +1,90 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using MongoDB.Driver;
+using Squidex.Domain.Apps.Core.Assets;
+using Squidex.Domain.Apps.Entities.Assets.Repositories;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.States;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.Assets;
+
+public sealed class MongoShardedAssetRepository : ShardedSnapshotStore<MongoAssetRepository, Asset>, IAssetRepository
+{
+    public MongoShardedAssetRepository(IShardingStrategy sharding, Func<string, MongoAssetRepository> factory)
+        : base(sharding, factory)
+    {
+    }
+
+    public IEnumerable<IMongoCollection<MongoAssetEntity>> GetInternalCollections()
+    {
+        return Shards.Select(x => x.GetInternalCollection());
+    }
+
+    public async Task<Asset?> FindAssetAsync(DomainId id,
+        CancellationToken ct = default)
+    {
+        Asset? result = null;
+
+        foreach (var shard in Shards)
+        {
+            if ((result = await shard.FindAssetAsync(id, ct)) != null)
+            {
+                return result;
+            }
+        }
+
+        return result;
+    }
+
+    public Task<Asset?> FindAssetAsync(DomainId appId, DomainId id, bool allowDeleted,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).FindAssetAsync(appId, id, allowDeleted, ct);
+    }
+
+    public Task<Asset?> FindAssetByHashAsync(DomainId appId, string hash, string fileName, long fileSize,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).FindAssetByHashAsync(appId, hash, fileName, fileSize, ct);
+    }
+
+    public Task<Asset?> FindAssetBySlugAsync(DomainId appId, string slug, bool allowDeleted,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).FindAssetBySlugAsync(appId, slug, allowDeleted, ct);
+    }
+
+    public Task<IResultList<Asset>> QueryAsync(DomainId appId, DomainId? parentId, Q q,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).QueryAsync(appId, parentId, q, ct);
+    }
+
+    public Task<IReadOnlyList<DomainId>> QueryChildIdsAsync(DomainId appId, DomainId parentId,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).QueryChildIdsAsync(appId, parentId, ct);
+    }
+
+    public Task<IReadOnlyList<DomainId>> QueryIdsAsync(DomainId appId, HashSet<DomainId> ids,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).QueryIdsAsync(appId, ids, ct);
+    }
+
+    public IAsyncEnumerable<Asset> StreamAll(DomainId appId,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).StreamAll(appId, ct);
+    }
+
+    protected override string GetShardKey(Asset state)
+    {
+        return GetShardKey(state.AppId.Id);
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/BsonUniqueContentIdSerializer.cs

@@ -0,0 +1,183 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Text;
+using MongoDB.Bson.Serialization;
+using MongoDB.Bson.Serialization.Serializers;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Infrastructure;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb;
+
+public sealed class BsonUniqueContentIdSerializer : SerializerBase<UniqueContentId>
+{
+    private const byte GuidLength = 16;
+    private static readonly BsonUniqueContentIdSerializer Instance = new BsonUniqueContentIdSerializer();
+
+    public static void Register()
+    {
+        BsonSerializer.TryRegisterSerializer(Instance);
+    }
+
+    private BsonUniqueContentIdSerializer()
+    {
+    }
+
+    public static UniqueContentId NextAppId(DomainId appId)
+    {
+        static void IncrementByteArray(byte[] bytes)
+        {
+            for (var i = 0; i < bytes.Length; i++)
+            {
+                var value = bytes[i];
+                if (value < byte.MaxValue)
+                {
+                    value += 1;
+                    bytes[i] = value;
+                    break;
+                }
+            }
+        }
+
+        if (Guid.TryParse(appId.ToString(), out var id))
+        {
+            var bytes = id.ToByteArray();
+
+            IncrementByteArray(bytes);
+
+            return new UniqueContentId(DomainId.Create(new Guid(bytes)), DomainId.Empty);
+        }
+        else
+        {
+            var bytes = Encoding.UTF8.GetBytes(appId.ToString());
+
+            IncrementByteArray(bytes);
+
+            return new UniqueContentId(DomainId.Create(Encoding.UTF8.GetString(bytes)), DomainId.Empty);
+        }
+    }
+
+    public override UniqueContentId Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
+    {
+        var buffer = context.Reader.ReadBytes()!;
+        var offset = 0;
+
+        static DomainId ReadId(byte[] buffer, ref int offset)
+        {
+            DomainId id;
+
+            // If we have reached the end of the buffer then
+            if (offset >= buffer.Length)
+            {
+                return default;
+            }
+
+            var length = buffer[offset++];
+            // Special length indicator for all guids.
+            if (length == 0xFF)
+            {
+                id = DomainId.Create(new Guid(buffer.AsSpan(offset, GuidLength)));
+                offset += GuidLength;
+            }
+            else
+            {
+                id = DomainId.Create(Encoding.UTF8.GetString(buffer.AsSpan(offset, length)));
+                offset += length;
+            }
+
+            return id;
+        }
+
+        return new UniqueContentId(ReadId(buffer, ref offset), ReadId(buffer, ref offset));
+    }
+
+    public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, UniqueContentId value)
+    {
+        var appId = CheckId(value.AppId);
+
+        var contentId = CheckId(value.ContentId);
+
+        var isEmptyContentId =
+            contentId.IsGuid &&
+            contentId.Guid == default;
+
+        // Do not write empty Ids to the buffer to allow prefix searches.
+        var contentLength = !isEmptyContentId ? contentId.Length + 1 : 0;
+
+        var bufferLength = appId.Length + 1 + contentLength;
+        var bufferArray = new byte[bufferLength];
+
+        var offset = Write(bufferArray, 0,
+            appId.IsGuid,
+            appId.Guid,
+            appId.Source,
+            appId.Length);
+
+        if (!isEmptyContentId)
+        {
+            // Do not write the empty content id, so we can search for app as well.
+            Write(bufferArray, offset,
+                contentId.IsGuid,
+                contentId.Guid,
+                contentId.Source,
+                contentId.Length);
+        }
+
+        static int Write(byte[] buffer, int offset, bool isGuid, Guid guid, string id, int idLength)
+        {
+            if (isGuid)
+            {
+                // Special length indicator for all guids.
+                buffer[offset++] = 0xFF;
+                WriteGuid(buffer.AsSpan(offset), guid);
+
+                return offset + GuidLength;
+            }
+            else
+            {
+                // We assume that we use relatively small IDs, not longer than 254 bytes.
+                buffer[offset++] = (byte)idLength;
+                WriteString(buffer.AsSpan(offset), id);
+
+                return offset + idLength;
+            }
+        }
+
+        context.Writer.WriteBytes(bufferArray);
+    }
+
+    private static (int Length, bool IsGuid, Guid Guid, string Source) CheckId(DomainId id)
+    {
+        var source = id.ToString();
+
+        var idIsGuid = Guid.TryParse(source, out var idGuid);
+        var idLength = GuidLength;
+
+        if (!idIsGuid)
+        {
+            idLength = (byte)Encoding.UTF8.GetByteCount(source);
+
+            // We only use a single byte to write the length, therefore we do not allow large strings.
+            if (idLength > 254)
+            {
+                ThrowHelper.InvalidOperationException("Cannot write long IDs.");
+            }
+        }
+
+        return (idLength, idIsGuid, idGuid, source);
+    }
+
+    private static void WriteString(Span<byte> span, string id)
+    {
+        Encoding.UTF8.GetBytes(id, span);
+    }
+
+    private static void WriteGuid(Span<byte> span, Guid guid)
+    {
+        guid.TryWriteBytes(span);
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs

@@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging;
 using MongoDB.Driver;
 using NodaTime;
 using Squidex.Domain.Apps.Core.Apps;
-using Squidex.Domain.Apps.Core.Assets;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Core.Schemas;
 using Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations;
@@ -97,9 +96,11 @@ public sealed class MongoContentCollection : MongoRepositoryBase<MongoContentEnt
         return collection.Indexes.CreateManyAsync(operations.SelectMany(x => x.CreateIndexes()), ct);
     }
 
-    public Task ResetScheduledAsync(DomainId documentId,
+    public Task ResetScheduledAsync(DomainId appId, DomainId contentId,
         CancellationToken ct)
     {
+        var documentId = DomainId.Combine(appId, contentId);
+
         return Collection.UpdateOneAsync(
             x => x.DocumentId == documentId,
             Update

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs

@@ -42,7 +42,7 @@ public partial class MongoContentRepository : MongoBase<MongoContentEntity>, ICo
         MongoContentEntity.RegisterClassMap();
     }
 
-    public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider,
+    public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider, string shardKey,
         IOptions<ContentOptions> options, ILogger<MongoContentRepository> log)
     {
         this.appProvider = appProvider;
@@ -50,11 +50,11 @@ public partial class MongoContentRepository : MongoBase<MongoContentEntity>, ICo
         this.options = options.Value;
 
         collectionComplete =
-            new MongoContentCollection("States_Contents_All3", database, log,
+            new MongoContentCollection($"States_Contents_All3{shardKey}", database, log,
                 ReadPreference.Primary, options.Value.OptimizeForSelfHosting);
 
         collectionPublished =
-            new MongoContentCollection("States_Contents_Published3", database, log,
+            new MongoContentCollection($"States_Contents_Published3{shardKey}", database, log,
                 ReadPreference.Secondary, options.Value.OptimizeForSelfHosting);
     }
 
@@ -124,10 +124,10 @@ public partial class MongoContentRepository : MongoBase<MongoContentEntity>, ICo
         return GetCollection(scope).QueryIdsAsync(app, schema, filterNode, ct);
     }
 
-    public Task ResetScheduledAsync(DomainId documentId, SearchScope scope,
+    public Task ResetScheduledAsync(DomainId appId, DomainId contentId, SearchScope scope,
         CancellationToken ct = default)
     {
-        return GetCollection(SearchScope.All).ResetScheduledAsync(documentId, ct);
+        return GetCollection(SearchScope.All).ResetScheduledAsync(appId, contentId, ct);
     }
 
     private MongoContentCollection GetCollection(SearchScope scope)

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs

@@ -0,0 +1,98 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Runtime.CompilerServices;
+using NodaTime;
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Core.Contents;
+using Squidex.Domain.Apps.Core.Schemas;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Repositories;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.Queries;
+using Squidex.Infrastructure.States;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.Contents;
+
+public sealed class MongoShardedContentRepository : ShardedSnapshotStore<MongoContentRepository, WriteContent>, IContentRepository
+{
+    public MongoShardedContentRepository(IShardingStrategy sharding, Func<string, MongoContentRepository> factory)
+        : base(sharding, factory)
+    {
+    }
+
+    public Task<Content?> FindContentAsync(App app, Schema schema, DomainId id, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).FindContentAsync(app, schema, id, scope, ct);
+    }
+
+    public Task<bool> HasReferrersAsync(App app, DomainId reference, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).HasReferrersAsync(app, reference, scope, ct);
+    }
+
+    public Task<IResultList<Content>> QueryAsync(App app, List<Schema> schemas, Q q, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).QueryAsync(app, schemas, q, scope, ct);
+    }
+
+    public Task<IResultList<Content>> QueryAsync(App app, Schema schema, Q q, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).QueryAsync(app, schema, q, scope, ct);
+    }
+
+    public Task<IReadOnlyList<ContentIdStatus>> QueryIdsAsync(App app, Schema schema, FilterNode<ClrValue> filterNode, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).QueryIdsAsync(app, schema, filterNode, scope, ct);
+    }
+
+    public Task<IReadOnlyList<ContentIdStatus>> QueryIdsAsync(App app, HashSet<DomainId> ids, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).QueryIdsAsync(app, ids, scope, ct);
+    }
+
+    public Task ResetScheduledAsync(DomainId appId, DomainId id, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).ResetScheduledAsync(appId, id, scope, ct);
+    }
+
+    public IAsyncEnumerable<Content> StreamAll(DomainId appId, HashSet<DomainId>? schemaIds, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).StreamAll(appId, schemaIds, scope, ct);
+    }
+
+    public IAsyncEnumerable<Content> StreamReferencing(DomainId appId, DomainId references, int take, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(appId).StreamReferencing(appId, references, take, scope, ct);
+    }
+
+    public async IAsyncEnumerable<Content> StreamScheduledWithoutDataAsync(Instant now, SearchScope scope,
+       [EnumeratorCancellation] CancellationToken ct = default)
+    {
+        foreach (var shard in Shards)
+        {
+            await foreach (var content in shard.StreamScheduledWithoutDataAsync(now, scope, ct))
+            {
+                yield return content;
+            }
+        }
+    }
+
+    protected override string GetShardKey(WriteContent state)
+    {
+        return GetShardKey(state.AppId.Id);
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/ShardedSnapshotStore.cs

@@ -0,0 +1,101 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System.Runtime.CompilerServices;
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.States;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb;
+
+public abstract class ShardedSnapshotStore<T, TState> : ShardedService<T>, ISnapshotStore<TState>, IDeleter where T : ISnapshotStore<TState>, IDeleter
+{
+    protected ShardedSnapshotStore(IShardingStrategy sharding, Func<string, T> factory)
+        : base(sharding, factory)
+    {
+    }
+
+    protected abstract string GetShardKey(TState state);
+
+    public Task WriteAsync(SnapshotWriteJob<TState> job,
+        CancellationToken ct = default)
+    {
+        var shard = Shard(GetShardKey(job.Value));
+
+        return shard.WriteAsync(job, ct);
+    }
+
+    public Task<SnapshotResult<TState>> ReadAsync(DomainId key,
+        CancellationToken ct = default)
+    {
+        var shard = Shard(GetAppId(key));
+
+        return shard.ReadAsync(key, ct);
+    }
+
+    public Task RemoveAsync(DomainId key,
+        CancellationToken ct = default)
+    {
+        var shard = Shard(GetAppId(key));
+
+        return shard.RemoveAsync(key, ct);
+    }
+
+    public Task DeleteAppAsync(App app,
+        CancellationToken ct)
+    {
+        var shard = Shard(app.Id);
+
+        return shard.DeleteAppAsync(app, ct);
+    }
+
+    public async IAsyncEnumerable<SnapshotResult<TState>> ReadAllAsync(
+        [EnumeratorCancellation] CancellationToken ct = default)
+    {
+        foreach (var shard in Shards)
+        {
+            await foreach (var asset in shard.ReadAllAsync(ct))
+            {
+                yield return asset;
+            }
+        }
+    }
+
+    public async Task ClearAsync(
+        CancellationToken ct = default)
+    {
+        foreach (var shard in Shards)
+        {
+            await shard.ClearAsync(ct);
+        }
+    }
+
+    public async Task WriteManyAsync(IEnumerable<SnapshotWriteJob<TState>> jobs,
+        CancellationToken ct = default)
+    {
+        // Some commands might share a shared, therefore we don't group by app id.
+        foreach (var byShard in jobs.GroupBy(c => GetShardKey(c.Value)))
+        {
+            var shard = Shard(byShard.Key);
+
+            await shard.WriteManyAsync(byShard.ToArray(), ct);
+        }
+    }
+
+    private static DomainId GetAppId(DomainId key)
+    {
+        // This is a leaky abstraction, but the only option to implement that in a fast way.
+        var parts = key.ToString().Split(DomainId.IdSeparator);
+
+        if (parts.Length != 2)
+        {
+            throw new InvalidOperationException("The key does not contain an app id.");
+        }
+
+        return DomainId.Create(parts[0]);
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs

@@ -25,15 +25,15 @@ public sealed class AtlasTextIndex : MongoTextIndexBase<Dictionary<string, strin
     private static readonly LuceneQueryAnalyzer QueryParser =
         new LuceneQueryAnalyzer(LuceneVersion.LUCENE_48, "*",
             new StandardAnalyzer(LuceneVersion.LUCENE_48, CharArraySet.EMPTY_SET));
-    private readonly AtlasOptions options;
+    private readonly AtlasOptions atlasOptions;
-    private readonly IHttpClientFactory httpClientFactory;
+    private readonly IHttpClientFactory atlasClient;
     private string index;
 
-    public AtlasTextIndex(IMongoDatabase database, IHttpClientFactory httpClientFactory, IOptions<AtlasOptions> options)
+    public AtlasTextIndex(IMongoDatabase database, IHttpClientFactory atlasClient, IOptions<AtlasOptions> atlasOptions, string shardKey)
-        : base(database)
+        : base(database, shardKey, new CommandFactory<Dictionary<string, string>>(BuildTexts))
     {
-        this.httpClientFactory = httpClientFactory;
+        this.atlasClient = atlasClient;
-        this.options = options.Value;
+        this.atlasOptions = atlasOptions.Value;
     }
 
     protected override async Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<Dictionary<string, string>>> collection,
@@ -41,31 +41,10 @@ public sealed class AtlasTextIndex : MongoTextIndexBase<Dictionary<string, strin
     {
         await base.SetupCollectionAsync(collection, ct);
 
-        index = await AtlasIndexDefinition.CreateIndexAsync(options, httpClientFactory,
+        index = await AtlasIndexDefinition.CreateIndexAsync(atlasOptions, atlasClient,
             Database.DatabaseNamespace.DatabaseName, CollectionName(), ct);
     }
 
-    protected override Dictionary<string, string> BuildTexts(Dictionary<string, string> source)
-    {
-        var texts = new Dictionary<string, string>();
-
-        foreach (var (key, value) in source)
-        {
-            var text = value;
-
-            var languageCode = AtlasIndexDefinition.GetFieldName(key);
-
-            if (texts.TryGetValue(languageCode, out var existing))
-            {
-                text = $"{existing} {value}";
-            }
-
-            texts[languageCode] = text;
-        }
-
-        return texts;
-    }
-
     public override async Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope,
         CancellationToken ct = default)
     {
@@ -158,4 +137,25 @@ public sealed class AtlasTextIndex : MongoTextIndexBase<Dictionary<string, strin
 
         return results.Select(x => x.ContentId).ToList();
     }
+
+    private static Dictionary<string, string> BuildTexts(Dictionary<string, string> source)
+    {
+        var texts = new Dictionary<string, string>();
+
+        foreach (var (key, value) in source)
+        {
+            var text = value;
+
+            var languageCode = AtlasIndexDefinition.GetFieldName(key);
+
+            if (texts.TryGetValue(languageCode, out var existing))
+            {
+                text = $"{existing} {value}";
+            }
+
+            texts[languageCode] = text;
+        }
+
+        return texts;
+    }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs

@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using MongoDB.Bson;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Infrastructure.MongoDb;
@@ -41,17 +42,17 @@ public sealed class CommandFactory<T> : MongoBase<MongoTextIndexEntity<T>> where
         writes.Add(
             new UpdateOneModel<MongoTextIndexEntity<T>>(
                 Filter.And(
-                    Filter.Eq(x => x.DocId, upsert.DocId),
+                    FilterByCommand(upsert),
                     Filter.Exists(x => x.GeoField, false),
                     Filter.Exists(x => x.GeoObject, false)),
                 Update
                     .Set(x => x.ServeAll, upsert.ServeAll)
                     .Set(x => x.ServePublished, upsert.ServePublished)
                     .Set(x => x.Texts, BuildTexts(upsert))
-                    .SetOnInsert(x => x.Id, Guid.NewGuid().ToString())
+                    .SetOnInsert(x => x.Id, ObjectId.GenerateNewId())
-                    .SetOnInsert(x => x.DocId, upsert.DocId)
+                    .SetOnInsert(x => x.AppId, upsert.UniqueContentId.AppId)
-                    .SetOnInsert(x => x.AppId, upsert.AppId.Id)
+                    .SetOnInsert(x => x.ContentId, upsert.UniqueContentId.ContentId)
-                    .SetOnInsert(x => x.ContentId, upsert.ContentId)
+                    .SetOnInsert(x => x.Stage, upsert.Stage)
                     .SetOnInsert(x => x.SchemaId, upsert.SchemaId.Id))
             {
                 IsUpsert = true
@@ -62,9 +63,9 @@ public sealed class CommandFactory<T> : MongoBase<MongoTextIndexEntity<T>> where
             if (!upsert.IsNew)
             {
                 writes.Add(
-                    new DeleteOneModel<MongoTextIndexEntity<T>>(
+                    new DeleteManyModel<MongoTextIndexEntity<T>>(
                         Filter.And(
-                            Filter.Eq(x => x.DocId, upsert.DocId),
+                            FilterByCommand(upsert),
                             Filter.Exists(x => x.GeoField),
                             Filter.Exists(x => x.GeoObject))));
             }
@@ -75,15 +76,15 @@ public sealed class CommandFactory<T> : MongoBase<MongoTextIndexEntity<T>> where
                     new InsertOneModel<MongoTextIndexEntity<T>>(
                         new MongoTextIndexEntity<T>
                         {
-                            Id = Guid.NewGuid().ToString(),
+                            Id = ObjectId.GenerateNewId(),
-                            AppId = upsert.AppId.Id,
+                            AppId = upsert.UniqueContentId.AppId,
-                            DocId = upsert.DocId,
+                            ContentId = upsert.UniqueContentId.ContentId,
-                            ContentId = upsert.ContentId,
                             GeoField = field,
                             GeoObject = geoObject,
                             SchemaId = upsert.SchemaId.Id,
                             ServeAll = upsert.ServeAll,
-                            ServePublished = upsert.ServePublished
+                            ServePublished = upsert.ServePublished,
+                            Stage = upsert.Stage,
                         }));
             }
         }
@@ -98,7 +99,7 @@ public sealed class CommandFactory<T> : MongoBase<MongoTextIndexEntity<T>> where
     {
         writes.Add(
             new UpdateOneModel<MongoTextIndexEntity<T>>(
-                Filter.Eq(x => x.DocId, update.DocId),
+                FilterByCommand(update),
                 Update
                     .Set(x => x.ServeAll, update.ServeAll)
                     .Set(x => x.ServePublished, update.ServePublished)));
@@ -107,7 +108,15 @@ public sealed class CommandFactory<T> : MongoBase<MongoTextIndexEntity<T>> where
     private static void DeleteEntry(DeleteIndexEntry delete, List<WriteModel<MongoTextIndexEntity<T>>> writes)
     {
         writes.Add(
-            new DeleteOneModel<MongoTextIndexEntity<T>>(
+            new DeleteManyModel<MongoTextIndexEntity<T>>(
-                Filter.Eq(x => x.DocId, delete.DocId)));
+                FilterByCommand(delete)));
+    }
+
+    private static FilterDefinition<MongoTextIndexEntity<T>> FilterByCommand(IndexCommand command)
+    {
+        return Filter.And(
+            Filter.Eq(x => x.AppId, command.UniqueContentId.AppId),
+            Filter.Eq(x => x.ContentId, command.UniqueContentId.ContentId),
+            Filter.Eq(x => x.Stage, command.Stage));
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs

@@ -0,0 +1,55 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Infrastructure;
+using Squidex.Infrastructure.States;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
+
+public sealed class MongoShardedTextIndex<T> : ShardedService<MongoTextIndexBase<T>>, ITextIndex where T : class
+{
+    public MongoShardedTextIndex(IShardingStrategy sharding, Func<string, MongoTextIndexBase<T>> factory)
+        : base(sharding, factory)
+    {
+    }
+
+    public async Task ClearAsync(
+        CancellationToken ct = default)
+    {
+        foreach (var shard in Shards)
+        {
+            await shard.ClearAsync(ct);
+        }
+    }
+
+    public async Task ExecuteAsync(IndexCommand[] commands,
+        CancellationToken ct = default)
+    {
+        // Some commands might share a shared, therefore we don't group by app id.
+        foreach (var byShard in commands.GroupBy(c => GetShardKey(c.UniqueContentId.AppId)))
+        {
+            var shard = Shard(byShard.Key);
+
+            await shard.ExecuteAsync(byShard.ToArray(), ct);
+        }
+    }
+
+    public Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).SearchAsync(app, query, scope, ct);
+    }
+
+    public Task<List<DomainId>?> SearchAsync(App app, GeoQuery query, SearchScope scope,
+        CancellationToken ct = default)
+    {
+        return Shard(app.Id).SearchAsync(app, query, scope, ct);
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs

@@ -6,13 +6,30 @@
 // ==========================================================================
 
 using MongoDB.Driver;
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
 
 public sealed class MongoTextIndex : MongoTextIndexBase<List<MongoTextIndexEntityText>>
 {
-    public MongoTextIndex(IMongoDatabase database)
+    private record struct SearchOperation
-        : base(database)
+    {
+        required public App App { get; init; }
+
+        required public List<(DomainId Id, double Score)> Results { get; init; }
+
+        required public string SearchTerms { get; init; }
+
+        required public int Take { get; set; }
+
+        required public SearchScope SearchScope { get; init; }
+    }
+
+    public MongoTextIndex(IMongoDatabase database, string shardKey)
+        : base(database, shardKey, new CommandFactory<List<MongoTextIndexEntityText>>(BuildTexts))
     {
     }
 
@@ -24,19 +41,92 @@ public sealed class MongoTextIndex : MongoTextIndexBase<List<MongoTextIndexEntit
         await collection.Indexes.CreateOneAsync(
             new CreateIndexModel<MongoTextIndexEntity<List<MongoTextIndexEntityText>>>(
                 Index
-                    .Text("t.t")
                     .Ascending(x => x.AppId)
-                    .Ascending(x => x.ServeAll)
+                    .Text("t.t")),
-                    .Ascending(x => x.ServePublished)
-                    .Ascending(x => x.SchemaId)),
             cancellationToken: ct);
     }
 
-    protected override List<MongoTextIndexEntityText> BuildTexts(Dictionary<string, string> source)
+    public override async Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope,
+        CancellationToken ct = default)
     {
-        return source.Select(x => new MongoTextIndexEntityText
+        Guard.NotNull(app);
+        Guard.NotNull(query);
+
+        if (string.IsNullOrWhiteSpace(query.Text))
         {
-            Text = Tokenizer.TokenizerTerms(x.Value, x.Key)
+            return null;
-        }).ToList();
+        }
+
+        // Use a custom tokenizer to leverage stop words from multiple languages.
+        var search = new SearchOperation
+        {
+            App = app,
+            SearchTerms = Tokenizer.Query(query.Text),
+            SearchScope = scope,
+            Results = [],
+            Take = query.Take
+        };
+
+        if (query.RequiredSchemaIds?.Count > 0)
+        {
+            await SearchBySchemaAsync(search, query.RequiredSchemaIds, 1, ct);
+        }
+        else if (query.PreferredSchemaId == null)
+        {
+            await SearchByAppAsync(search, 1, ct);
+        }
+        else
+        {
+            // We cannot write queries that prefer results from the same schema, therefore make two queries.
+            search.Take /= 2;
+
+            // Increasing the scoring of the results from the schema by 10 percent.
+            await SearchBySchemaAsync(search, Enumerable.Repeat(query.PreferredSchemaId.Value, 1), 1.1, ct);
+            await SearchByAppAsync(search, 1, ct);
+        }
+
+        return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
+    }
+
+    private Task SearchBySchemaAsync(SearchOperation search, IEnumerable<DomainId> schemaIds, double factor,
+        CancellationToken ct = default)
+    {
+        var filter =
+            Filter.And(
+                Filter.Eq(x => x.AppId, search.App.Id),
+                Filter.Text(search.SearchTerms, "none"),
+                Filter.In(x => x.SchemaId, schemaIds),
+                FilterByScope(search.SearchScope));
+
+        return SearchAsync(search, filter, factor, ct);
+    }
+
+    private Task SearchByAppAsync(SearchOperation search, double factor,
+        CancellationToken ct = default)
+    {
+        var filter =
+            Filter.And(
+                Filter.Eq(x => x.AppId, search.App.Id),
+                Filter.Text(search.SearchTerms, "none"),
+                FilterByScope(search.SearchScope));
+
+        return SearchAsync(search, filter, factor, ct);
+    }
+
+    private async Task SearchAsync(SearchOperation search, FilterDefinition<MongoTextIndexEntity<List<MongoTextIndexEntityText>>> filter, double factor,
+        CancellationToken ct = default)
+    {
+        var byText =
+            await GetCollection(search.SearchScope).Find(filter).Limit(search.Take)
+                .Project<MongoTextResult>(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score"))
+                .ToListAsync(ct);
+
+        search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor)));
+    }
+
+    private static List<MongoTextIndexEntityText> BuildTexts(Dictionary<string, string> source)
+    {
+        // Use a custom tokenizer to leverage stop words from multiple languages.
+        return source.Select(x => MongoTextIndexEntityText.FromText(Tokenizer.Terms(x.Value, x.Key))).ToList();
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs

@@ -18,16 +18,17 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
 
 public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndexEntity<T>>, ITextIndex, IDeleter where T : class
 {
-    private readonly CommandFactory<T> commandFactory;
+    private readonly CommandFactory<T> factory;
+    private readonly string shardKey;
 
     protected sealed class MongoTextResult
     {
         [BsonId]
         [BsonElement]
-        public string Id { get; set; }
+        public ObjectId Id { get; set; }
 
         [BsonRequired]
-        [BsonElement("_ci")]
+        [BsonElement("c")]
         public DomainId ContentId { get; set; }
 
         [BsonIgnoreIfDefault]
@@ -35,23 +36,11 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         public double Score { get; set; }
     }
 
-    private sealed class SearchOperation
+    protected MongoTextIndexBase(IMongoDatabase database, string shardKey, CommandFactory<T> factory)
-    {
-        public List<(DomainId Id, double Score)> Results { get; } = [];
-
-        public string SearchTerms { get; set; }
-
-        public App App { get; set; }
-
-        public SearchScope SearchScope { get; set; }
-    }
-
-    protected MongoTextIndexBase(IMongoDatabase database)
         : base(database)
     {
-#pragma warning disable MA0056 // Do not call overridable members in constructor
+        this.shardKey = shardKey;
-        commandFactory = new CommandFactory<T>(BuildTexts);
+        this.factory = factory;
-#pragma warning restore MA0056 // Do not call overridable members in constructor
     }
 
     protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity<T>> collection,
@@ -60,13 +49,13 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         return collection.Indexes.CreateManyAsync(new[]
         {
             new CreateIndexModel<MongoTextIndexEntity<T>>(
-                Index.Ascending(x => x.DocId)),
+                Index
+                    .Ascending(x => x.AppId)
+                    .Ascending(x => x.ContentId)),
 
             new CreateIndexModel<MongoTextIndexEntity<T>>(
                 Index
                     .Ascending(x => x.AppId)
-                    .Ascending(x => x.ServeAll)
-                    .Ascending(x => x.ServePublished)
                     .Ascending(x => x.SchemaId)
                     .Ascending(x => x.GeoField)
                     .Geo2DSphere(x => x.GeoObject))
@@ -75,11 +64,9 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
 
     protected override string CollectionName()
     {
-        return "TextIndex";
+        return $"TextIndex2{shardKey}";
     }
 
-    protected abstract T BuildTexts(Dictionary<string, string> source);
-
     async Task IDeleter.DeleteAppAsync(App app,
         CancellationToken ct)
     {
@@ -93,7 +80,7 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
 
         foreach (var command in commands)
         {
-            commandFactory.CreateCommands(command, writes);
+            factory.CreateCommands(command, writes);
         }
 
         if (writes.Count == 0)
@@ -107,7 +94,7 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         }
         catch (MongoBulkWriteException ex)
         {
-            // Ignore invalid geo data.
+            // Ignore invalid geo data when writing content. Our insert is unordered anyway.
             if (ex.WriteErrors.Any(error => error.Code != MongoDbErrorCodes.Errror16755_InvalidGeoData))
             {
                 throw;
@@ -121,12 +108,14 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         Guard.NotNull(app);
         Guard.NotNull(query);
 
+        // Use the filter in the correct order to leverage the index in the best way.
         var findFilter =
             Filter.And(
                 Filter.Eq(x => x.AppId, app.Id),
                 Filter.Eq(x => x.SchemaId, query.SchemaId),
-                Filter_ByScope(scope),
+                Filter.Eq(x => x.GeoField, query.Field),
-                Filter.GeoWithinCenterSphere(x => x.GeoObject, query.Longitude, query.Latitude, query.Radius / 6378100));
+                Filter.GeoWithinCenterSphere(x => x.GeoObject, query.Longitude, query.Latitude, query.Radius / 6378100),
+                FilterByScope(scope));
 
         var byGeo =
             await GetCollection(scope).Find(findFilter).Limit(query.Take)
@@ -136,83 +125,10 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         return byGeo.Select(x => x.ContentId).ToList();
     }
 
-    public virtual async Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope,
+    public abstract Task<List<DomainId>?> SearchAsync(App app, TextQuery query, SearchScope scope,
-        CancellationToken ct = default)
+        CancellationToken ct = default);
-    {
-        Guard.NotNull(app);
-        Guard.NotNull(query);
-
-        if (string.IsNullOrWhiteSpace(query.Text))
-        {
-            return null;
-        }
-
-        var search = new SearchOperation
-        {
-            App = app,
-            SearchTerms = Tokenizer.TokenizeQuery(query.Text),
-            SearchScope = scope
-        };
-
-        if (query.RequiredSchemaIds?.Count > 0)
-        {
-            await SearchBySchemaAsync(search, query.RequiredSchemaIds, query.Take, 1, ct);
-        }
-        else if (query.PreferredSchemaId == null)
-        {
-            await SearchByAppAsync(search, query.Take, 1, ct);
-        }
-        else
-        {
-            var halfBucket = query.Take / 2;
-
-            var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1);
-
-            await SearchBySchemaAsync(search, schemaIds, halfBucket, 1.1, ct);
-            await SearchByAppAsync(search, halfBucket, 1, ct);
-        }
-
-        return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList();
-    }
-
-    private Task SearchBySchemaAsync(SearchOperation search, IEnumerable<DomainId> schemaIds, int take, double factor,
-        CancellationToken ct = default)
-    {
-        var filter =
-            Filter.And(
-                Filter.Eq(x => x.AppId, search.App.Id),
-                Filter.In(x => x.SchemaId, schemaIds),
-                Filter_ByScope(search.SearchScope),
-                Filter.Text(search.SearchTerms, "none"));
-
-        return SearchAsync(search, filter, take, factor, ct);
-    }
-
-    private Task SearchByAppAsync(SearchOperation search, int take, double factor,
-        CancellationToken ct = default)
-    {
-        var filter =
-            Filter.And(
-                Filter.Eq(x => x.AppId, search.App.Id),
-                Filter.Exists(x => x.SchemaId),
-                Filter_ByScope(search.SearchScope),
-                Filter.Text(search.SearchTerms, "none"));
-
-        return SearchAsync(search, filter, take, factor, ct);
-    }
-
-    private async Task SearchAsync(SearchOperation search, FilterDefinition<MongoTextIndexEntity<T>> filter, int take, double factor,
-        CancellationToken ct = default)
-    {
-        var byText =
-            await GetCollection(search.SearchScope).Find(filter).Limit(take)
-                .Project<MongoTextResult>(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score"))
-                .ToListAsync(ct);
-
-        search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor)));
-    }
 
-    private static FilterDefinition<MongoTextIndexEntity<T>> Filter_ByScope(SearchScope scope)
+    protected static FilterDefinition<MongoTextIndexEntity<T>> FilterByScope(SearchScope scope)
     {
         if (scope == SearchScope.All)
         {
@@ -224,7 +140,7 @@ public abstract class MongoTextIndexBase<T> : MongoRepositoryBase<MongoTextIndex
         }
     }
 
-    private IMongoCollection<MongoTextIndexEntity<T>> GetCollection(SearchScope scope)
+    protected IMongoCollection<MongoTextIndexEntity<T>> GetCollection(SearchScope scope)
     {
         if (scope == SearchScope.All)
         {

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs

@@ -16,34 +16,34 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Text;
 public sealed class MongoTextIndexEntity<T>
 {
     [BsonId]
-    [BsonElement("_id")]
+    public ObjectId Id { get; set; }
-    public string Id { get; set; }
 
     [BsonRequired]
-    [BsonElement(nameof(DocId))]
+    [BsonElement("a")]
-    public string DocId { get; set; }
+    [BsonRepresentation(BsonType.String)]
+    public DomainId AppId { get; set; }
 
     [BsonRequired]
-    [BsonElement("_ci")]
+    [BsonElement("s")]
     [BsonRepresentation(BsonType.String)]
-    public DomainId ContentId { get; set; }
+    public DomainId SchemaId { get; set; }
 
     [BsonRequired]
-    [BsonElement("_ai")]
+    [BsonElement("c")]
     [BsonRepresentation(BsonType.String)]
-    public DomainId AppId { get; set; }
+    public DomainId ContentId { get; set; }
 
     [BsonRequired]
-    [BsonElement("_si")]
+    [BsonElement("x")]
-    [BsonRepresentation(BsonType.String)]
+    [BsonRepresentation(BsonType.Int32)]
-    public DomainId SchemaId { get; set; }
+    public byte Stage { get; set; }
 
     [BsonRequired]
-    [BsonElement("fa")]
+    [BsonElement("e")]
     public bool ServeAll { get; set; }
 
     [BsonRequired]
-    [BsonElement("fp")]
+    [BsonElement("p")]
     public bool ServePublished { get; set; }
 
     [BsonIgnoreIfNull]
@@ -51,11 +51,11 @@ public sealed class MongoTextIndexEntity<T>
     public T Texts { get; set; }
 
     [BsonIgnoreIfNull]
-    [BsonElement("gf")]
+    [BsonElement("g")]
     public string GeoField { get; set; }
 
     [BsonIgnoreIfNull]
-    [BsonElement("go")]
+    [BsonElement("o")]
     [BsonJson]
     [BsonRepresentation(BsonType.Document)]
     public Geometry GeoObject { get; set; }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs

@@ -8,6 +8,7 @@
 using MongoDB.Bson.Serialization;
 using MongoDB.Driver;
 using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.MongoDb;
@@ -18,21 +19,14 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState
 {
     static MongoTextIndexerState()
     {
+        BsonUniqueContentIdSerializer.Register();
+
         BsonClassMap.TryRegisterClassMap<TextContentState>(cm =>
         {
-            cm.MapIdField(x => x.UniqueContentId);
+            cm.MapIdProperty(x => x.UniqueContentId);
-
-            cm.MapProperty(x => x.AppId)
-                .SetElementName("a");
-
-            cm.MapProperty(x => x.DocIdCurrent)
-                .SetElementName("c");
 
-            cm.MapProperty(x => x.DocIdNew)
+            cm.MapProperty(x => x.State)
-                .SetElementName("n").SetIgnoreIfNull(true);
+                .SetElementName("s");
-
-            cm.MapProperty(x => x.DocIdForPublished)
-                .SetElementName("p").SetIgnoreIfNull(true);
         });
     }
 
@@ -41,16 +35,6 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState
     {
     }
 
-    protected override Task SetupCollectionAsync(IMongoCollection<TextContentState> collection,
-        CancellationToken ct)
-    {
-        return collection.Indexes.CreateManyAsync(new[]
-        {
-            new CreateIndexModel<TextContentState>(
-                Index.Ascending(x => x.AppId))
-        }, ct);
-    }
-
     protected override string CollectionName()
     {
         return "TextIndexerState";
@@ -59,10 +43,15 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState
     async Task IDeleter.DeleteAppAsync(App app,
         CancellationToken ct)
     {
-        await Collection.DeleteManyAsync(Filter.Eq(x => x.AppId, app.Id), ct);
+        var filter =
+            Filter.And(
+                Filter.Gte(x => x.UniqueContentId, new UniqueContentId(app.Id, DomainId.Empty)),
+                Filter.Lt(x => x.UniqueContentId, BsonUniqueContentIdSerializer.NextAppId(app.Id)));
+
+        await Collection.DeleteManyAsync(filter, ct);
     }
 
-    public async Task<Dictionary<DomainId, TextContentState>> GetAsync(HashSet<DomainId> ids,
+    public async Task<Dictionary<UniqueContentId, TextContentState>> GetAsync(HashSet<UniqueContentId> ids,
         CancellationToken ct = default)
     {
         var entities = await Collection.Find(Filter.In(x => x.UniqueContentId, ids)).ToListAsync(ct);
@@ -77,7 +66,7 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState
 
         foreach (var update in updates)
         {
-            if (update.IsDeleted)
+            if (update.State == TextState.Deleted)
             {
                 writes.Add(
                     new DeleteOneModel<TextContentState>(
@@ -88,9 +77,9 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase<TextContentState
                 writes.Add(
                     new ReplaceOneModel<TextContentState>(
                         Filter.Eq(x => x.UniqueContentId, update.UniqueContentId), update)
-                    {
+                        {
-                        IsUpsert = true
+                            IsUpsert = true
-                    });
+                        });
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs

@@ -59,7 +59,7 @@ public static class Tokenizer
         }
     }
 
-    public static string TokenizeQuery(string query)
+    public static string Query(string query)
     {
         query = query.Trim();
 
@@ -73,10 +73,10 @@ public static class Tokenizer
             textReader.Read();
         }
 
-        return TokenizeWord(textReader, textLanguage);
+        return Word(textReader, textLanguage);
     }
 
-    public static string TokenizerTerms(string query, string language)
+    public static string Terms(string query, string language)
     {
         var stopWords =
             string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ?
@@ -87,7 +87,7 @@ public static class Tokenizer
         return Tokenize(new StringReader(query), stopWords);
     }
 
-    private static string TokenizeWord(TextReader reader, string language)
+    private static string Word(TextReader reader, string language)
     {
         var stopWords =
             string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ?

backend/src/Squidex.Domain.Apps.Entities/Contents/ContentSchedulerProcess.cs

@@ -93,7 +93,7 @@ public sealed class ContentSchedulerProcess : IBackgroundProcess
         }
         catch (DomainObjectNotFoundException)
         {
-            await contentRepository.ResetScheduledAsync(content.UniqueId, default);
+            await contentRepository.ResetScheduledAsync(content.AppId.Id, id, default);
         }
         catch (Exception ex)
         {

backend/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs

@@ -43,6 +43,6 @@ public interface IContentRepository
     Task<bool> HasReferrersAsync(App app, DomainId reference, SearchScope scope,
         CancellationToken ct = default);
 
-    Task ResetScheduledAsync(DomainId documentId, SearchScope scope,
+    Task ResetScheduledAsync(DomainId appId, DomainId contentId, SearchScope scope,
         CancellationToken ct = default);
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs

@@ -11,9 +11,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text;
 
 public abstract class IndexCommand
 {
-    public NamedId<DomainId> AppId { get; set; }
+    public UniqueContentId UniqueContentId { get; set; }
 
     public NamedId<DomainId> SchemaId { get; set; }
 
-    public string DocId { get; set; }
+    public byte Stage { get; set; }
+
+    public string ToDocId()
+    {
+        return $"{UniqueContentId.AppId}__{UniqueContentId.ContentId}_{Stage}";
+    }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs

@@ -13,7 +13,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State;
 public sealed class CachingTextIndexerState : ITextIndexerState
 {
     private readonly ITextIndexerState inner;
-    private readonly LRUCache<DomainId, Tuple<TextContentState?>> cache = new LRUCache<DomainId, Tuple<TextContentState?>>(10000);
+    private readonly LRUCache<UniqueContentId, Tuple<TextContentState?>> cache = new LRUCache<UniqueContentId, Tuple<TextContentState?>>(10000);
 
     public CachingTextIndexerState(ITextIndexerState inner)
     {
@@ -30,14 +30,14 @@ public sealed class CachingTextIndexerState : ITextIndexerState
         cache.Clear();
     }
 
-    public async Task<Dictionary<DomainId, TextContentState>> GetAsync(HashSet<DomainId> ids,
+    public async Task<Dictionary<UniqueContentId, TextContentState>> GetAsync(HashSet<UniqueContentId> ids,
         CancellationToken ct = default)
     {
         Guard.NotNull(ids);
 
-        var missingIds = new HashSet<DomainId>();
+        var missingIds = new HashSet<UniqueContentId>();
 
-        var result = new Dictionary<DomainId, TextContentState>();
+        var result = new Dictionary<UniqueContentId, TextContentState>();
 
         foreach (var id in ids)
         {
@@ -81,7 +81,7 @@ public sealed class CachingTextIndexerState : ITextIndexerState
 
         foreach (var update in updates)
         {
-            if (update.IsDeleted)
+            if (update.State == TextState.Deleted)
             {
                 cache.Set(update.UniqueContentId, Tuple.Create<TextContentState?>(null));
             }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs

@@ -5,13 +5,11 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using Squidex.Infrastructure;
-
 namespace Squidex.Domain.Apps.Entities.Contents.Text.State;
 
 public interface ITextIndexerState
 {
-    Task<Dictionary<DomainId, TextContentState>> GetAsync(HashSet<DomainId> ids,
+    Task<Dictionary<UniqueContentId, TextContentState>> GetAsync(HashSet<UniqueContentId> ids,
         CancellationToken ct = default);
 
     Task SetAsync(List<TextContentState> updates,

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs

@@ -11,7 +11,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State;
 
 public sealed class InMemoryTextIndexerState : ITextIndexerState
 {
-    private readonly Dictionary<DomainId, TextContentState> states = [];
+    private readonly Dictionary<UniqueContentId, TextContentState> states = [];
 
     public Task ClearAsync(
         CancellationToken ct = default)
@@ -21,12 +21,12 @@ public sealed class InMemoryTextIndexerState : ITextIndexerState
         return Task.CompletedTask;
     }
 
-    public Task<Dictionary<DomainId, TextContentState>> GetAsync(HashSet<DomainId> ids,
+    public Task<Dictionary<UniqueContentId, TextContentState>> GetAsync(HashSet<UniqueContentId> ids,
         CancellationToken ct = default)
     {
         Guard.NotNull(ids);
 
-        var result = new Dictionary<DomainId, TextContentState>();
+        var result = new Dictionary<UniqueContentId, TextContentState>();
 
         foreach (var id in ids)
         {
@@ -46,7 +46,7 @@ public sealed class InMemoryTextIndexerState : ITextIndexerState
 
         foreach (var update in updates)
         {
-            if (update.IsDeleted)
+            if (update.State == TextState.Deleted)
             {
                 states.Remove(update.UniqueContentId);
             }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs

@@ -5,45 +5,11 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using Squidex.Infrastructure;
-
 namespace Squidex.Domain.Apps.Entities.Contents.Text.State;
 
 public sealed class TextContentState
 {
-    public DomainId AppId { get; set; }
+    public UniqueContentId UniqueContentId { get; set; }
-
-    public DomainId UniqueContentId { get; set; }
-
-    public string DocIdCurrent { get; set; }
-
-    public string? DocIdNew { get; set; }
-
-    public string? DocIdForPublished { get; set; }
-
-    public bool IsDeleted { get; set; }
-
-    public void GenerateDocIdNew()
-    {
-        if (DocIdCurrent?.EndsWith("_2", StringComparison.Ordinal) != false)
-        {
-            DocIdNew = $"{UniqueContentId}_1";
-        }
-        else
-        {
-            DocIdNew = $"{UniqueContentId}_2";
-        }
-    }
 
-    public void GenerateDocIdCurrent()
+    public TextState State { get; set; }
-    {
-        if (DocIdNew?.EndsWith("_2", StringComparison.Ordinal) != false)
-        {
-            DocIdCurrent = $"{UniqueContentId}_1";
-        }
-        else
-        {
-            DocIdCurrent = $"{UniqueContentId}_2";
-        }
-    }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextState.cs

@@ -0,0 +1,19 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text.State;
+
+public enum TextState
+{
+    Stage0_Draft__Stage1_None,
+    Stage0_Published__Stage1_None,
+    Stage0_Published__Stage1_Draft,
+    Stage1_Draft__Stage0_None,
+    Stage1_Published__Stage0_None,
+    Stage1_Published__Stage0_Draft,
+    Deleted,
+}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs

@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using LibGit2Sharp;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
 using Squidex.Domain.Apps.Events.Contents;
@@ -16,7 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text;
 
 public sealed class TextIndexingProcess : IEventConsumer
 {
-    private const string NotFound = "<404>";
     private readonly IJsonSerializer serializer;
     private readonly ITextIndex textIndex;
     private readonly ITextIndexerState textIndexerState;
@@ -25,7 +25,7 @@ public sealed class TextIndexingProcess : IEventConsumer
 
     public int BatchDelay => 1000;
 
-    public string Name => "TextIndexer5";
+    public string Name => "TextIndexer6";
 
     public StreamFilter EventsFilter { get; } = StreamFilter.Prefix("content-");
 
@@ -36,14 +36,15 @@ public sealed class TextIndexingProcess : IEventConsumer
 
     private sealed class Updates
     {
-        private readonly Dictionary<DomainId, TextContentState> states;
+        private readonly Dictionary<UniqueContentId, TextContentState> currentState;
+        private readonly Dictionary<UniqueContentId, TextContentState> currentUpdates;
+        private readonly Dictionary<(UniqueContentId, byte), IndexCommand> commands = [];
         private readonly IJsonSerializer serializer;
-        private readonly Dictionary<DomainId, TextContentState> updates = [];
-        private readonly Dictionary<string, IndexCommand> commands = [];
 
-        public Updates(Dictionary<DomainId, TextContentState> states, IJsonSerializer serializer)
+        public Updates(Dictionary<UniqueContentId, TextContentState> states, IJsonSerializer serializer)
         {
-            this.states = states;
+            currentState = states;
+            currentUpdates = [];
             this.serializer = serializer;
         }
 
@@ -54,9 +55,9 @@ public sealed class TextIndexingProcess : IEventConsumer
                 await textIndex.ExecuteAsync(commands.Values.ToArray());
             }
 
-            if (updates.Count > 0)
+            if (currentUpdates.Count > 0)
             {
-                await textIndexerState.SetAsync(updates.Values.ToList());
+                await textIndexerState.SetAsync(currentUpdates.Values.ToList());
             }
         }
 
@@ -98,211 +99,228 @@ public sealed class TextIndexingProcess : IEventConsumer
 
         private void Create(ContentEvent @event, ContentData data)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
             var state = new TextContentState
             {
-                AppId = @event.AppId.Id,
                 UniqueContentId = uniqueId
             };
 
-            state.GenerateDocIdCurrent();
-
             Index(@event,
                 new UpsertIndexEntry
                 {
-                    ContentId = @event.ContentId,
+                    UniqueContentId = uniqueId,
-                    DocId = state.DocIdCurrent,
                     GeoObjects = data.ToGeo(serializer),
+                    IsNew = true,
+                    Stage = 0,
                     ServeAll = true,
                     ServePublished = false,
                     Texts = data.ToTexts(),
-                    IsNew = true
                 });
 
-            states[state.UniqueContentId] = state;
+            currentState[state.UniqueContentId] = state;
-
+            currentUpdates[state.UniqueContentId] = state;
-            updates[state.UniqueContentId] = state;
         }
 
         private void CreateDraft(ContentEvent @event)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state))
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                state.GenerateDocIdNew();
+                switch (state.State)
+                {
+                    case TextState.Stage0_Published__Stage1_None:
+                        state.State = TextState.Stage0_Published__Stage1_Draft;
+                        break;
+                    case TextState.Stage1_Published__Stage0_None:
+                        state.State = TextState.Stage1_Published__Stage0_Draft;
+                        break;
+                }
 
-                updates[state.UniqueContentId] = state;
+                currentUpdates[state.UniqueContentId] = state;
             }
         }
 
         private void Unpublish(ContentEvent @event)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state) && state.DocIdForPublished != null)
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                Index(@event,
+                switch (state.State)
-                    new UpdateIndexEntry
+                {
-                    {
+                    case TextState.Stage0_Published__Stage1_None:
-                        DocId = state.DocIdForPublished,
+                        CoreUpdate(@event, uniqueId, 0, true, false);
-                        ServeAll = true,
-                        ServePublished = false
-                    });
 
-                state.DocIdForPublished = null;
+                        state.State = TextState.Stage0_Draft__Stage1_None;
+                        break;
+                    case TextState.Stage1_Published__Stage0_None:
+                        CoreUpdate(@event, uniqueId, 1, true, false);
 
-                updates[state.UniqueContentId] = state;
+                        state.State = TextState.Stage1_Draft__Stage0_None;
+                        break;
+                }
+
+                currentUpdates[state.UniqueContentId] = state;
             }
         }
 
         private void Update(ContentEvent @event, ContentData data)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state))
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                if (state.DocIdNew != null)
+                switch (state.State)
                 {
-                    Index(@event,
+                    case TextState.Stage0_Draft__Stage1_None:
-                        new UpsertIndexEntry
+                        CoreUpsert(@event, uniqueId, 0, true, false, data);
-                        {
+                        break;
-                            ContentId = @event.ContentId,
+                    case TextState.Stage0_Published__Stage1_None:
-                            DocId = state.DocIdNew,
+                        CoreUpsert(@event, uniqueId, 0, true, true, data);
-                            GeoObjects = data.ToGeo(serializer),
+                        break;
-                            ServeAll = true,
+                    case TextState.Stage0_Published__Stage1_Draft:
-                            ServePublished = false,
+                        CoreUpsert(@event, uniqueId, 1, true, false, data);
-                            Texts = data.ToTexts()
+                        CoreUpdate(@event, uniqueId, 0, false, true);
-                        });
+                        break;
-
+                    case TextState.Stage1_Draft__Stage0_None:
-                    Index(@event,
+                        CoreUpsert(@event, uniqueId, 1, true, false, data);
-                        new UpdateIndexEntry
+                        break;
-                        {
+                    case TextState.Stage1_Published__Stage0_None:
-                            DocId = state.DocIdCurrent,
+                        CoreUpsert(@event, uniqueId, 1, true, true, data);
-                            ServeAll = false,
+                        break;
-                            ServePublished = true
+                    case TextState.Stage1_Published__Stage0_Draft:
-                        });
+                        CoreUpsert(@event, uniqueId, 0, true, false, data);
+                        CoreUpdate(@event, uniqueId, 1, false, true);
+                        break;
                 }
-                else
-                {
-                    var isPublished = state.DocIdCurrent == state.DocIdForPublished;
 
-                    Index(@event,
+                currentUpdates[state.UniqueContentId] = state;
-                        new UpsertIndexEntry
-                        {
-                            ContentId = @event.ContentId,
-                            DocId = state.DocIdCurrent,
-                            GeoObjects = data.ToGeo(serializer),
-                            ServeAll = true,
-                            ServePublished = isPublished,
-                            Texts = data.ToTexts()
-                        });
-                }
             }
         }
 
         private void Publish(ContentEvent @event)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state))
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                if (state.DocIdNew != null)
+                switch (state.State)
                 {
-                    Index(@event,
+                    case TextState.Stage0_Published__Stage1_Draft:
-                        new UpdateIndexEntry
+                        CoreUpdate(@event, uniqueId, 1, true, true);
-                        {
+                        CoreDelete(@event, uniqueId, 0);
-                            DocId = state.DocIdNew,
+
-                            ServeAll = true,
+                        state.State = TextState.Stage1_Published__Stage0_None;
-                            ServePublished = true
+                        break;
-                        });
+                    case TextState.Stage1_Published__Stage0_Draft:
-
+                        CoreUpdate(@event, uniqueId, 0, true, true);
-                    Index(@event,
+                        CoreDelete(@event, uniqueId, 1);
-                        new DeleteIndexEntry
+
-                        {
+                        state.State = TextState.Stage0_Published__Stage1_None;
-                            DocId = state.DocIdCurrent
+                        break;
-                        });
+                    case TextState.Stage0_Draft__Stage1_None:
-
+                        CoreUpdate(@event, uniqueId, 0, true, true);
-                    state.DocIdForPublished = state.DocIdNew;
+
-                    state.DocIdCurrent = state.DocIdNew;
+                        state.State = TextState.Stage0_Published__Stage1_None;
+                        break;
+                    case TextState.Stage1_Draft__Stage0_None:
+                        CoreUpdate(@event, uniqueId, 1, true, true);
+
+                        state.State = TextState.Stage1_Published__Stage0_None;
+                        break;
                 }
-                else
-                {
-                    Index(@event,
-                        new UpdateIndexEntry
-                        {
-                            DocId = state.DocIdCurrent,
-                            ServeAll = true,
-                            ServePublished = true
-                        });
 
-                    state.DocIdForPublished = state.DocIdCurrent;
+                currentUpdates[state.UniqueContentId] = state;
-                }
-
-                state.DocIdNew = null;
-
-                updates[state.UniqueContentId] = state;
             }
         }
 
         private void DeleteDraft(ContentEvent @event)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state) && state.DocIdNew != null)
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                Index(@event,
+                switch (state.State)
-                    new UpdateIndexEntry
+                {
-                    {
+                    case TextState.Stage0_Published__Stage1_Draft:
-                        DocId = state.DocIdCurrent,
+                        CoreUpdate(@event, uniqueId, 0, true, true);
-                        ServeAll = true,
+                        CoreDelete(@event, uniqueId, 1);
-                        ServePublished = true
+
-                    });
+                        state.State = TextState.Stage0_Published__Stage1_None;
-
+                        break;
-                Index(@event,
+                    case TextState.Stage1_Published__Stage0_Draft:
-                    new DeleteIndexEntry
+                        CoreUpdate(@event, uniqueId, 1, true, true);
-                    {
+                        CoreDelete(@event, uniqueId, 0);
-                        DocId = state.DocIdNew
+
-                    });
+                        state.State = TextState.Stage1_Published__Stage0_None;
-
+                        break;
-                state.DocIdNew = null;
+                }
 
-                updates[state.UniqueContentId] = state;
+                currentUpdates[state.UniqueContentId] = state;
             }
         }
 
         private void Delete(ContentEvent @event)
         {
-            var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId);
+            var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId);
 
-            if (states.TryGetValue(uniqueId, out var state))
+            if (currentState.TryGetValue(uniqueId, out var state))
             {
-                Index(@event,
+                CoreDelete(@event, uniqueId, 0);
-                    new DeleteIndexEntry
+                CoreDelete(@event, uniqueId, 1);
-                    {
-                        DocId = state.DocIdCurrent
-                    });
 
-                Index(@event,
+                state.State = TextState.Deleted;
-                    new DeleteIndexEntry
-                    {
-                        DocId = state.DocIdNew ?? NotFound
-                    });
 
-                state.IsDeleted = true;
+                currentUpdates[state.UniqueContentId] = state;
-
-                updates[state.UniqueContentId] = state;
             }
         }
 
+        private void CoreUpsert(ContentEvent @event, UniqueContentId uniqueId, byte stage, bool all, bool published, ContentData data)
+        {
+            Index(@event,
+                new UpsertIndexEntry
+                {
+                    UniqueContentId = uniqueId,
+                    GeoObjects = data.ToGeo(serializer),
+                    Stage = stage,
+                    ServeAll = all,
+                    ServePublished = published,
+                    Texts = data.ToTexts()
+                });
+        }
+
+        private void CoreUpdate(ContentEvent @event, UniqueContentId uniqueId, byte stage, bool all, bool published)
+        {
+            Index(@event,
+                new UpdateIndexEntry
+                {
+                    UniqueContentId = uniqueId,
+                    Stage = stage,
+                    ServeAll = all,
+                    ServePublished = published,
+                });
+        }
+
+        private void CoreDelete(ContentEvent @event, UniqueContentId uniqueId, byte stage)
+        {
+            Index(@event,
+                new DeleteIndexEntry
+                {
+                    UniqueContentId = uniqueId,
+                    Stage = stage,
+                });
+        }
+
         private void Index(ContentEvent @event, IndexCommand command)
         {
-            command.AppId = @event.AppId;
             command.SchemaId = @event.SchemaId;
 
+            var key = (command.UniqueContentId, command.Stage);
+
             if (command is UpdateIndexEntry update &&
-                commands.TryGetValue(command.DocId, out var existing) &&
+                commands.TryGetValue(key, out var existing) &&
                 existing is UpsertIndexEntry upsert)
             {
                 upsert.ServeAll = update.ServeAll;
@@ -310,7 +328,7 @@ public sealed class TextIndexingProcess : IEventConsumer
             }
             else
             {
-                commands[command.DocId] = command;
+                commands[key] = command;
             }
         }
     }
@@ -345,12 +363,12 @@ public sealed class TextIndexingProcess : IEventConsumer
         await updates.WriteAsync(textIndex, textIndexerState);
     }
 
-    private Task<Dictionary<DomainId, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events)
+    private Task<Dictionary<UniqueContentId, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events)
     {
         var ids =
             events
                 .Select(x => x.Payload).OfType<ContentEvent>()
-                .Select(x => DomainId.Combine(x.AppId, x.ContentId))
+                .Select(x => new UniqueContentId(x.AppId.Id, x.ContentId))
                 .ToHashSet();
 
         return textIndexerState.GetAsync(ids);

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UniqueContentId.cs

@@ -0,0 +1,16 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Infrastructure;
+
+#pragma warning disable SA1313 // Parameter names should begin with lower-case letter
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text;
+
+public record struct UniqueContentId(DomainId AppId, DomainId ContentId)
+{
+}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UpsertIndexEntry.cs

@@ -6,7 +6,6 @@
 // ==========================================================================
 
 using NetTopologySuite.Geometries;
-using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text;
 
@@ -20,7 +19,5 @@ public sealed class UpsertIndexEntry : IndexCommand
 
     public bool ServePublished { get; set; }
 
-    public DomainId ContentId { get; set; }
-
     public bool IsNew { get; set; }
 }

backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj

@@ -10,6 +10,11 @@
     <DebugType>full</DebugType>
     <DebugSymbols>True</DebugSymbols>
   </PropertyGroup>
+  <ItemGroup>
+    <Compile Remove="Notifications\**" />
+    <EmbeddedResource Remove="Notifications\**" />
+    <None Remove="Notifications\**" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Squidex.Domain.Apps.Core.Model\Squidex.Domain.Apps.Core.Model.csproj" />
     <ProjectReference Include="..\Squidex.Domain.Apps.Core.Operations\Squidex.Domain.Apps.Core.Operations.csproj" />

backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDomainIdSerializer.cs

@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Text;
 using MongoDB.Bson;
 using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Serializers;
@@ -13,7 +14,7 @@ namespace Squidex.Infrastructure.MongoDb;
 
 public sealed class BsonDomainIdSerializer : SerializerBase<DomainId>, IBsonPolymorphicSerializer, IRepresentationConfigurable<BsonDomainIdSerializer>
 {
-    private static readonly BsonDomainIdSerializer Instance = new BsonDomainIdSerializer();
+    private static readonly BsonDomainIdSerializer Instance = new BsonDomainIdSerializer(BsonType.String);
 
     public static void Register()
     {
@@ -29,7 +30,12 @@ public sealed class BsonDomainIdSerializer : SerializerBase<DomainId>, IBsonPoly
         get => true;
     }
 
-    public BsonType Representation { get; } = BsonType.String;
+    public BsonType Representation { get; }
+
+    public BsonDomainIdSerializer(BsonType representation)
+    {
+        Representation = representation;
+    }
 
     public override DomainId Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args)
     {
@@ -45,7 +51,7 @@ public sealed class BsonDomainIdSerializer : SerializerBase<DomainId>, IBsonPoly
                     return DomainId.Create(binary.ToGuid());
                 }
 
-                return DomainId.Create(binary.ToString());
+                return DomainId.Create(Encoding.UTF8.GetString(binary.Bytes));
             default:
                 ThrowHelper.NotSupportedException();
                 return default!;
@@ -54,17 +60,52 @@ public sealed class BsonDomainIdSerializer : SerializerBase<DomainId>, IBsonPoly
 
     public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, DomainId value)
     {
-        context.Writer.WriteString(value.ToString());
+        switch (Representation)
+        {
+            case BsonType.String:
+                context.Writer.WriteString(value.ToString());
+                break;
+            case BsonType.Binary:
+                if (Guid.TryParse(value.ToString(), out var guid))
+                {
+#pragma warning disable CS0618 // Type or member is obsolete
+                    if (context.Writer.Settings.GuidRepresentation == GuidRepresentation.CSharpLegacy)
+                    {
+                        context.Writer.WriteBinaryData(new BsonBinaryData(guid.ToByteArray(), BsonBinarySubType.UuidLegacy, GuidRepresentation.CSharpLegacy));
+                    }
+                    else
+                    {
+                        context.Writer.WriteBinaryData(new BsonBinaryData(guid, GuidRepresentation.Standard));
+                    }
+#pragma warning restore CS0618 // Type or member is obsolete
+                }
+                else
+                {
+                    var buffer = Encoding.UTF8.GetBytes(value.ToString());
+
+                    context.Writer.WriteBytes(buffer);
+                }
+
+                break;
+            default:
+                ThrowHelper.NotSupportedException();
+                break;
+        }
     }
 
     public BsonDomainIdSerializer WithRepresentation(BsonType representation)
     {
-        if (representation != BsonType.String)
+        if (representation is not BsonType.String and not BsonType.Binary)
         {
             ThrowHelper.NotSupportedException();
             return default!;
         }
 
+        if (representation != Representation)
+        {
+            return new BsonDomainIdSerializer(representation);
+        }
+
         return this;
     }
 

backend/src/Squidex.Infrastructure/DomainId.cs

@@ -13,6 +13,7 @@ namespace Squidex.Infrastructure;
 public readonly struct DomainId : IEquatable<DomainId>, IComparable<DomainId>
 {
     private static readonly string EmptyString = Guid.Empty.ToString();
+
     public static readonly DomainId Empty = default;
     public static readonly string IdSeparator = "--";
 

backend/src/Squidex.Infrastructure/States/ShardedService.cs

@@ -0,0 +1,68 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Hosting;
+
+namespace Squidex.Infrastructure.States;
+
+public abstract class ShardedService<T> : IInitializable
+{
+    private readonly Dictionary<string, T> shards = new Dictionary<string, T>();
+    private readonly IShardingStrategy sharding;
+    private readonly Func<string, T> factory;
+
+    protected IEnumerable<T> Shards => shards.Values;
+
+    protected ShardedService(IShardingStrategy sharding, Func<string, T> factory)
+    {
+        this.sharding = sharding;
+        this.factory = factory;
+    }
+
+    public async Task InitializeAsync(
+        CancellationToken ct)
+    {
+        foreach (var shardKey in sharding.GetShardKeys())
+        {
+            var inner = factory(shardKey);
+
+            if (inner is IInitializable initializable)
+            {
+                await initializable.InitializeAsync(ct);
+            }
+
+            shards[shardKey] = inner;
+        }
+    }
+
+    public async Task ReleaseAsync(
+        CancellationToken ct)
+    {
+        foreach (var inner in shards.Values)
+        {
+            if (inner is IInitializable initializable)
+            {
+                await initializable.ReleaseAsync(ct);
+            }
+        }
+    }
+
+    protected string GetShardKey<TKey>(TKey key) where TKey : notnull
+    {
+        return sharding.GetShardKey(key);
+    }
+
+    protected T Shard<TKey>(TKey key) where TKey : notnull
+    {
+        return shards[GetShardKey(key)];
+    }
+
+    protected string GetShardKey(DomainId appId)
+    {
+        return sharding.GetShardKey(appId);
+    }
+}

backend/src/Squidex.Infrastructure/States/Sharding.cs

@@ -0,0 +1,66 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+#pragma warning disable MA0048 // File name must match type name
+
+namespace Squidex.Infrastructure.States;
+
+public interface IShardingStrategy
+{
+    string GetShardKey<T>(T key) where T : notnull;
+
+    IEnumerable<string> GetShardKeys();
+}
+
+public sealed class SingleSharding : IShardingStrategy
+{
+    public static readonly IShardingStrategy Instance = new SingleSharding();
+
+    private SingleSharding()
+    {
+    }
+
+    public string GetShardKey<T>(T key) where T : notnull
+    {
+        return string.Empty;
+    }
+
+    public IEnumerable<string> GetShardKeys()
+    {
+        yield return string.Empty;
+    }
+}
+
+public sealed class PartitionedSharding : IShardingStrategy
+{
+    private readonly int numPartitions;
+
+    public PartitionedSharding(int numPartitions)
+    {
+        this.numPartitions = numPartitions;
+    }
+
+    public string GetShardKey<T>(T key) where T : notnull
+    {
+        var partition = Math.Abs(key.GetHashCode()) % numPartitions;
+
+        return GetShardKey(partition);
+    }
+
+    public IEnumerable<string> GetShardKeys()
+    {
+        for (var i = 0; i < numPartitions; i++)
+        {
+            yield return GetShardKey(i);
+        }
+    }
+
+    private static string GetShardKey(int partition)
+    {
+        return $"_{partition}";
+    }
+}

backend/src/Squidex/Config/Domain/StoreServices.cs

@@ -99,9 +99,6 @@ public static class StoreServices
                 services.AddTransientAs(c => new ConvertDocumentIds(GetDatabase(c, mongoDatabaseName), GetDatabase(c, mongoContentDatabaseName)))
                     .As<IMigration>();
 
-                services.AddSingletonAs(c => ActivatorUtilities.CreateInstance<MongoContentRepository>(c, GetDatabase(c, mongoContentDatabaseName)))
-                    .As<IContentRepository>().As<ISnapshotStore<WriteContent>>().As<IDeleter>();
-
                 services.AddTransientAs<ConvertRuleEventsJson>()
                     .As<IMigration>();
 
@@ -138,9 +135,6 @@ public static class StoreServices
                 services.AddSingletonAs<MongoUserStore>()
                     .As<IUserStore<IdentityUser>>().As<IUserFactory>();
 
-                services.AddSingletonAs<MongoAssetRepository>()
-                    .As<IAssetRepository>().As<ISnapshotStore<Asset>>().As<IDeleter>();
-
                 services.AddSingletonAs<MongoAssetFolderRepository>()
                     .As<IAssetFolderRepository>().As<ISnapshotStore<AssetFolder>>().As<IDeleter>();
 
@@ -162,6 +156,20 @@ public static class StoreServices
                 services.AddSingletonAs<MongoTextIndexerState>()
                     .As<ITextIndexerState>().As<IDeleter>();
 
+                services.AddSingletonAs(c =>
+                {
+                    return new MongoShardedAssetRepository(GetSharding(config, "store:mongoDB:assetShardCount"),
+                        shardKey => ActivatorUtilities.CreateInstance<MongoAssetRepository>(c, shardKey));
+                }).As<IAssetRepository>().As<ISnapshotStore<Asset>>().As<IDeleter>();
+
+                services.AddSingletonAs(c =>
+                {
+                    var contentDatabase = GetDatabase(c, mongoContentDatabaseName);
+
+                    return new MongoShardedContentRepository(GetSharding(config, "store:mongoDB:contentShardCount"),
+                        shardKey => ActivatorUtilities.CreateInstance<MongoContentRepository>(c, shardKey, contentDatabase));
+                }).As<IContentRepository>().As<ISnapshotStore<WriteContent>>().As<IDeleter>();
+
                 services.AddOpenIddict()
                     .AddCore(builder =>
                     {
@@ -191,13 +199,19 @@ public static class StoreServices
                         };
                     });
 
-                    services.AddSingletonAs<AtlasTextIndex>()
+                    services.AddSingletonAs(c =>
-                        .AsOptional<ITextIndex>().As<IDeleter>();
+                    {
+                        return new MongoShardedTextIndex<Dictionary<string, string>>(GetSharding(config, "store:mongoDB:textShardCount"),
+                            shardKey => ActivatorUtilities.CreateInstance<AtlasTextIndex>(c, shardKey));
+                    }).AsOptional<ITextIndex>().As<IDeleter>();
                 }
                 else
                 {
-                    services.AddSingletonAs<MongoTextIndex>()
+                    services.AddSingletonAs(c =>
-                        .AsOptional<ITextIndex>().As<IDeleter>();
+                    {
+                        return new MongoShardedTextIndex<List<MongoTextIndexEntityText>>(GetSharding(config, "store:mongoDB:textShardCount"),
+                            shardKey => ActivatorUtilities.CreateInstance<MongoTextIndex>(c, shardKey));
+                    }).AsOptional<ITextIndex>().As<IDeleter>();
                 }
 
                 services.AddInitializer<JsonSerializerOptions>("Serializer (BSON)", jsonSerializerOptions =>
@@ -230,6 +244,13 @@ public static class StoreServices
         });
     }
 
+    private static IShardingStrategy GetSharding(IConfiguration config, string name)
+    {
+        var shardCount = config.GetValue<int>(name);
+
+        return shardCount > 0 && shardCount <= 100 ? new PartitionedSharding(shardCount) : SingleSharding.Instance;
+    }
+
     private static IMongoDatabase GetDatabase(IServiceProvider serviceProvider, string name)
     {
         return serviceProvider.GetRequiredService<IMongoClient>().GetDatabase(name);

backend/src/Squidex/appsettings.json

@@ -530,6 +530,15 @@
             // The database for all your content collections (one collection per app).
             "contentDatabase": "SquidexContent",
 
+            // Defines the number of collections for contents.
+            "contentShardCount": 0,
+
+            // Defines the number of collections for assets.
+            "assetShardCount": 0,
+
+            // Defines the number of collections for texts.
+            "textShardCount": 0,
+
             // The database for all your other read collections.
             "database": "Squidex",
 

backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs

@@ -31,7 +31,6 @@ public class RuleServiceTests
     private readonly string actionDump = "MyDump";
     private readonly string actionName = "ValidAction";
     private readonly string actionDescription = "MyDescription";
-    private readonly DomainId ruleId = DomainId.NewGuid();
     private readonly NamedId<DomainId> appId = NamedId.Of(DomainId.NewGuid(), "my-app");
     private readonly TypeRegistry typeRegistry = new TypeRegistry();
     private readonly RuleService sut;

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentSchedulerProcessTests.cs

@@ -114,7 +114,7 @@ public class ContentSchedulerProcessTests : GivenContext
 
         await sut.PublishAsync(CancellationToken);
 
-        A.CallTo(() => contentRepository.ResetScheduledAsync(content1.UniqueId, SearchScope.All, default))
+        A.CallTo(() => contentRepository.ResetScheduledAsync(content1.AppId.Id, content1.Id, SearchScope.All, default))
             .MustHaveHappened();
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs

@@ -16,7 +16,7 @@ public class TokenizerTests
     {
         var source = "The only thing that matters, is time.";
 
-        var parsed = Tokenizer.TokenizeQuery(source);
+        var parsed = Tokenizer.Query(source);
 
         Assert.Equal("only thing matters time", parsed);
     }
@@ -26,7 +26,7 @@ public class TokenizerTests
     {
         var source = "en:The only thing that matters, is time.";
 
-        var parsed = Tokenizer.TokenizeQuery(source);
+        var parsed = Tokenizer.Query(source);
 
         Assert.Equal("only thing matters time", parsed);
     }
@@ -36,7 +36,7 @@ public class TokenizerTests
     {
         var source = "en:when i do this it is pretty slow";
 
-        var parsed = Tokenizer.TokenizeQuery(source);
+        var parsed = Tokenizer.Query(source);
 
         Assert.Equal("when i do pretty slow", parsed);
     }
@@ -46,7 +46,7 @@ public class TokenizerTests
     {
         var source = "iv:The only thing that matters, is time.";
 
-        var parsed = Tokenizer.TokenizeQuery(source);
+        var parsed = Tokenizer.Query(source);
 
         Assert.Equal("the only thing that matters is time", parsed);
     }
@@ -56,7 +56,7 @@ public class TokenizerTests
     {
         var source = "de:Nur die Zeit spielt eine Rolle";
 
-        var parsed = Tokenizer.TokenizeQuery(source);
+        var parsed = Tokenizer.Query(source);
 
         Assert.Equal("zeit spielt rolle", parsed);
     }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/BsonUniqueContentIdSerializerTests.cs

@@ -0,0 +1,90 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Domain.Apps.Core.TestHelpers;
+using Squidex.Domain.Apps.Entities.MongoDb;
+using Squidex.Infrastructure;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text;
+
+public class BsonUniqueContentIdSerializerTests
+{
+    public BsonUniqueContentIdSerializerTests()
+    {
+        BsonUniqueContentIdSerializer.Register();
+    }
+
+    [Fact]
+    public void Should_serialize_and_deserialize_guid_guid()
+    {
+        var source = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid());
+
+        var deserialized = source.SerializeAndDeserializeBson();
+
+        Assert.Equal(source, deserialized);
+    }
+
+    [Fact]
+    public void Should_serialize_and_deserialize_guid_custom()
+    {
+        var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Create("id42"));
+
+        var deserialized = source.SerializeAndDeserializeBson();
+
+        Assert.Equal(source, deserialized);
+    }
+
+    [Fact]
+    public void Should_serialize_and_deserialize_guid_empty()
+    {
+        var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Empty);
+
+        var deserialized = source.SerializeAndDeserializeBson();
+
+        Assert.Equal(source, deserialized);
+    }
+
+    [Fact]
+    public void Should_serialize_and_deserialize_custom_custom()
+    {
+        var source = new UniqueContentId(DomainId.Create("id41"), DomainId.Create("id42"));
+
+        var deserialized = source.SerializeAndDeserializeBson();
+
+        Assert.Equal(source, deserialized);
+    }
+
+    [Fact]
+    public void Should_serialize_and_deserialize_custom_guid()
+    {
+        var source = new UniqueContentId(DomainId.Create("id42"), DomainId.NewGuid());
+
+        var deserialized = source.SerializeAndDeserializeBson();
+
+        Assert.Equal(source, deserialized);
+    }
+
+    [Fact]
+    public void Should_calculate_next_custom_id()
+    {
+        var appId = DomainId.Create("x");
+
+        var actual = BsonUniqueContentIdSerializer.NextAppId(appId);
+
+        Assert.Equal(new UniqueContentId(DomainId.Create("y"), DomainId.Empty), actual);
+    }
+
+    [Fact]
+    public void Should_calculate_next_guid_id()
+    {
+        var appId = DomainId.Create("70fb3772-2ab5-4854-b421-054d2479a0f7");
+
+        var actual = BsonUniqueContentIdSerializer.NextAppId(appId);
+
+        Assert.Equal(new UniqueContentId(DomainId.Create("70fb3773-2ab5-4854-b421-054d2479a0f7"), DomainId.Empty), actual);
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs

@@ -15,11 +15,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text;
 public class CachingTextIndexerStateTests : GivenContext
 {
     private readonly ITextIndexerState inner = A.Fake<ITextIndexerState>();
-    private readonly DomainId contentId = DomainId.NewGuid();
+    private readonly UniqueContentId contentId;
     private readonly CachingTextIndexerState sut;
 
     public CachingTextIndexerStateTests()
     {
+        contentId = new UniqueContentId(AppId.Id, DomainId.NewGuid());
+
         sut = new CachingTextIndexerState(inner);
     }
 
@@ -30,12 +32,12 @@ public class CachingTextIndexerStateTests : GivenContext
 
         var state = new TextContentState { UniqueContentId = contentId };
 
-        var states = new Dictionary<DomainId, TextContentState>
+        var states = new Dictionary<UniqueContentId, TextContentState>
         {
             [contentId] = state
         };
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>.That.Is(contentIds), CancellationToken))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>.That.Is(contentIds), CancellationToken))
             .Returns(states);
 
         var found1 = await sut.GetAsync(HashSet.Of(contentId), CancellationToken);
@@ -44,7 +46,7 @@ public class CachingTextIndexerStateTests : GivenContext
         Assert.Same(state, found1[contentId]);
         Assert.Same(state, found2[contentId]);
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>.That.Is(contentIds), CancellationToken))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>.That.Is(contentIds), CancellationToken))
             .MustHaveHappenedOnceExactly();
     }
 
@@ -53,7 +55,7 @@ public class CachingTextIndexerStateTests : GivenContext
     {
         var contentIds = HashSet.Of(contentId);
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>.That.Is(contentIds), CancellationToken))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>.That.Is(contentIds), CancellationToken))
             .Returns([]);
 
         var found1 = await sut.GetAsync(HashSet.Of(contentId), CancellationToken);
@@ -62,7 +64,7 @@ public class CachingTextIndexerStateTests : GivenContext
         Assert.Empty(found1);
         Assert.Empty(found2);
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>.That.Is(contentIds), CancellationToken))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>.That.Is(contentIds), CancellationToken))
             .MustHaveHappenedOnceExactly();
     }
 
@@ -84,7 +86,7 @@ public class CachingTextIndexerStateTests : GivenContext
         A.CallTo(() => inner.SetAsync(A<List<TextContentState>>.That.IsSameSequenceAs(state), CancellationToken))
             .MustHaveHappenedOnceExactly();
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>._, A<CancellationToken>._))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>._, A<CancellationToken>._))
             .MustNotHaveHappened();
     }
 
@@ -102,7 +104,7 @@ public class CachingTextIndexerStateTests : GivenContext
 
         await sut.SetAsync(
         [
-            new TextContentState { UniqueContentId = contentId, IsDeleted = true }
+            new TextContentState { UniqueContentId = contentId, State = TextState.Deleted }
         ], CancellationToken);
 
         var found1 = await sut.GetAsync(contentIds, CancellationToken);
@@ -111,10 +113,10 @@ public class CachingTextIndexerStateTests : GivenContext
         Assert.Empty(found1);
         Assert.Empty(found2);
 
-        A.CallTo(() => inner.SetAsync(A<List<TextContentState>>.That.Matches(x => x.Count == 1 && x[0].IsDeleted), CancellationToken))
+        A.CallTo(() => inner.SetAsync(A<List<TextContentState>>.That.Matches(x => x.Count == 1 && x[0].State == TextState.Deleted), CancellationToken))
             .MustHaveHappenedOnceExactly();
 
-        A.CallTo(() => inner.GetAsync(A<HashSet<DomainId>>._, A<CancellationToken>._))
+        A.CallTo(() => inner.GetAsync(A<HashSet<UniqueContentId>>._, A<CancellationToken>._))
             .MustNotHaveHappened();
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs

@@ -23,7 +23,7 @@ public sealed class MongoTextIndexFixture : IAsyncLifetime
         var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]!);
         var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]!);
 
-        Index = new MongoTextIndex(mongoDatabase);
+        Index = new MongoTextIndex(mongoDatabase, string.Empty);
     }
 
     public Task InitializeAsync()

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs

@@ -31,22 +31,18 @@ public class MongoTextIndexTests : TextIndexerTestsBase, IClassFixture<MongoText
     [Fact]
     public async Task Should_retrieve_all_stopwords_for_english_query()
     {
-        var both = ids2.Union(ids1).ToList();
-
         await CreateTextAsync(ids1[0], "de", "and und");
         await CreateTextAsync(ids2[0], "en", "and und");
 
-        await SearchText(expected: both, text: "and");
+        await SearchText(expected: ids1, text: "de:and");
     }
 
     [Fact]
     public async Task Should_retrieve_all_stopwords_for_german_query()
     {
-        var both = ids2.Union(ids1).ToList();
-
         await CreateTextAsync(ids1[0], "de", "and und");
         await CreateTextAsync(ids2[0], "en", "and und");
 
-        await SearchText(expected: both, text: "und");
+        await SearchText(expected: ids2, text: "en:und");
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexerStateFixture.cs

@@ -0,0 +1,38 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Domain.Apps.Core.TestHelpers;
+using Squidex.Domain.Apps.Entities.MongoDb.Text;
+using Squidex.Domain.Apps.Entities.TestHelpers;
+using Squidex.Infrastructure.MongoDb;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text;
+
+public sealed class MongoTextIndexerStateFixture : IAsyncLifetime
+{
+    public MongoTextIndexerState State { get; }
+
+    public MongoTextIndexerStateFixture()
+    {
+        TestUtils.SetupBson();
+
+        var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]!);
+        var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]!);
+
+        State = new MongoTextIndexerState(mongoDatabase);
+    }
+
+    public Task InitializeAsync()
+    {
+        return State.InitializeAsync(default);
+    }
+
+    public Task DisposeAsync()
+    {
+        return Task.CompletedTask;
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexerStateTests.cs

@@ -0,0 +1,95 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+#pragma warning disable SA1300 // Element should begin with upper-case letter
+
+using Squidex.Domain.Apps.Core.Apps;
+using Squidex.Domain.Apps.Entities.Contents.Text.State;
+using Squidex.Infrastructure;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text;
+
+[Trait("Category", "Dependencies")]
+public class MongoTextIndexerStateTests : IClassFixture<MongoTextIndexerStateFixture>
+{
+    public MongoTextIndexerStateFixture _ { get; set; }
+
+    public MongoTextIndexerStateTests(MongoTextIndexerStateFixture fixture)
+    {
+        _ = fixture;
+    }
+
+    [Fact]
+    public async Task Should_add_state()
+    {
+        var appId = DomainId.NewGuid();
+        var id1 = new UniqueContentId(appId, DomainId.NewGuid());
+        var id2 = new UniqueContentId(appId, DomainId.NewGuid());
+        var id3 = new UniqueContentId(appId, DomainId.NewGuid());
+
+        await _.State.SetAsync(
+        [
+            new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None },
+            new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft },
+            new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None }
+        ]);
+
+        var actual = await _.State.GetAsync(HashSet.Of(id1, id2));
+
+        actual.Should().BeEquivalentTo(new Dictionary<UniqueContentId, TextContentState>
+        {
+            [id1] = new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None },
+            [id2] = new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft }
+        });
+    }
+
+    [Fact]
+    public async Task Should_remove_state()
+    {
+        var id = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid());
+
+        await _.State.SetAsync(
+        [
+            new TextContentState { UniqueContentId = id, State = TextState.Stage0_Draft__Stage1_None }
+        ]);
+
+        await _.State.SetAsync(
+        [
+            new TextContentState { UniqueContentId = id, State = TextState.Deleted }
+        ]);
+
+        var actual = await _.State.GetAsync(HashSet.Of(id));
+
+        Assert.Empty(actual);
+    }
+
+    [Fact]
+    public async Task Should_remove_by_app_state()
+    {
+        var appId1 = DomainId.NewGuid();
+        var appId2 = DomainId.NewGuid();
+        var id1 = new UniqueContentId(appId1, DomainId.NewGuid());
+        var id2 = new UniqueContentId(appId1, DomainId.NewGuid());
+        var id3 = new UniqueContentId(appId2, DomainId.NewGuid());
+
+        await _.State.SetAsync(
+        [
+            new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None },
+            new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft },
+            new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None }
+        ]);
+
+        await ((IDeleter)_.State).DeleteAppAsync(new App { Id = appId1 }, default);
+
+        var actual = await _.State.GetAsync(HashSet.Of(id1, id2, id3));
+
+        actual.Should().BeEquivalentTo(new Dictionary<UniqueContentId, TextContentState>
+        {
+            [id3] = new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None }
+        });
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs

@@ -127,6 +127,14 @@ public abstract class TextIndexerTestsBase : GivenContext
         await SearchGeo(expected: null, "other.iv", 51.48596429889613, 12.102629469505713);
     }
 
+    [Fact]
+    public async Task Should_search_by_app()
+    {
+        await CreateTextAsync(ids1[0], "iv", "Hello");
+
+        await SearchByAppText(expected: ids1, text: "helo~");
+    }
+
     [Fact]
     public async Task Should_index_invariant_content_and_retrieve()
     {
@@ -183,10 +191,13 @@ public abstract class TextIndexerTestsBase : GivenContext
     [Fact]
     public async Task Should_also_update_published_content()
     {
+        // Create initial content.
         await CreateTextAsync(ids1[0], "iv", "V1");
 
+        // Publish the content.
         await PublishAsync(ids1[0]);
 
+        // Update the published content once.
         await UpdateTextAsync(ids1[0], "iv", "V2");
 
         await SearchText(expected: null, text: "V1", target: SearchScope.All);
@@ -199,10 +210,13 @@ public abstract class TextIndexerTestsBase : GivenContext
     [Fact]
     public async Task Should_also_update_published_content_multiple_times()
     {
+        // Create initial content.
         await CreateTextAsync(ids1[0], "iv", "V1");
 
+        // Publish the content.
         await PublishAsync(ids1[0]);
 
+        // Update the published content twice.
         await UpdateTextAsync(ids1[0], "iv", "V2");
         await UpdateTextAsync(ids1[0], "iv", "V3");
 
@@ -216,6 +230,7 @@ public abstract class TextIndexerTestsBase : GivenContext
     [Fact]
     public async Task Should_simulate_new_version()
     {
+        // Create initial content.
         await CreateTextAsync(ids1[0], "iv", "V1");
 
         // Publish the content.
@@ -467,4 +482,23 @@ public abstract class TextIndexerTestsBase : GivenContext
             actual.Should().BeEmpty();
         }
     }
+
+    protected async Task SearchByAppText(List<DomainId>? expected, string text, SearchScope target = SearchScope.All)
+    {
+        var query = new TextQuery(text, 1000)
+        {
+            PreferredSchemaId = Schema.Id
+        };
+
+        var actual = await Sut.TextIndex.SearchAsync(App, query, target, default);
+
+        if (expected != null)
+        {
+            actual.Should().BeEquivalentTo(expected.ToHashSet());
+        }
+        else
+        {
+            actual.Should().BeEmpty();
+        }
+    }
 }

backend/tests/Squidex.Infrastructure.Tests/MongoDb/DomainIdSerializerTests.cs

@@ -6,8 +6,6 @@
 // ==========================================================================
 
 using MongoDB.Bson;
-using MongoDB.Bson.IO;
-using MongoDB.Bson.Serialization;
 using MongoDB.Bson.Serialization.Attributes;
 using Squidex.Infrastructure.TestHelpers;
 
@@ -21,6 +19,12 @@ public class DomainIdSerializerTests
         public T Id { get; set; }
     }
 
+    private sealed class BinaryEntity<T>
+    {
+        [BsonRepresentation(BsonType.Binary)]
+        public T Id { get; set; }
+    }
+
     private sealed class IdEntity<T>
     {
         public T Id { get; set; }
@@ -34,57 +38,50 @@ public class DomainIdSerializerTests
     [Fact]
     public void Should_deserialize_from_string()
     {
-        var id = Guid.NewGuid();
+        var source = new IdEntity<string> { Id = Guid.NewGuid().ToString() };
-
-        var source = new IdEntity<string> { Id = id.ToString() };
 
-        var actual = SerializeAndDeserializeBson<IdEntity<string>, IdEntity<DomainId>>(source);
+        var actual = TestUtils.SerializeAndDeserializeBson<IdEntity<DomainId>, IdEntity<string>>(source);
 
-        Assert.Equal(actual.Id.ToString(), id.ToString());
+        Assert.Equal(actual.Id.ToString(), source.Id);
     }
 
     [Fact]
     public void Should_deserialize_from_guid_string()
     {
-        var id = Guid.NewGuid();
+        var source = new StringEntity<Guid> { Id = Guid.NewGuid() };
-
-        var source = new StringEntity<Guid> { Id = id };
 
-        var actual = SerializeAndDeserializeBson<StringEntity<Guid>, IdEntity<DomainId>>(source);
+        var actual = TestUtils.SerializeAndDeserializeBson<IdEntity<DomainId>, StringEntity<Guid>>(source);
 
-        Assert.Equal(actual.Id.ToString(), id.ToString());
+        Assert.Equal(actual.Id.ToString(), source.Id.ToString());
     }
 
     [Fact]
     public void Should_deserialize_from_guid_bytes()
     {
-        var id = Guid.NewGuid();
+        var source = new IdEntity<Guid> { Id = Guid.NewGuid() };
 
-        var source = new IdEntity<Guid> { Id = id };
+        var actual = TestUtils.SerializeAndDeserializeBson<IdEntity<DomainId>, IdEntity<Guid>>(source);
 
-        var actual = SerializeAndDeserializeBson<IdEntity<Guid>, IdEntity<DomainId>>(source);
+        Assert.Equal(actual.Id.ToString(), source.Id.ToString());
-
-        Assert.Equal(actual.Id.ToString(), id.ToString());
     }
 
-    private static TOut SerializeAndDeserializeBson<TIn, TOut>(TIn source)
+    [Fact]
+    public void Should_serialize_guid_as_bytes()
     {
-        var stream = new MemoryStream();
+        var source = new BinaryEntity<DomainId> { Id = DomainId.NewGuid() };
 
-        using (var writer = new BsonBinaryWriter(stream))
+        var actual = TestUtils.SerializeAndDeserializeBson(source);
-        {
-            BsonSerializer.Serialize(writer, source);
 
-            writer.Flush();
+        Assert.Equal(actual.Id, source.Id);
-        }
+    }
 
-        stream.Position = 0;
+    [Fact]
+    public void Should_serialize_non_guid_as_bytes()
+    {
+        var source = new BinaryEntity<DomainId> { Id = DomainId.Create("NonGuid") };
 
-        using (var reader = new BsonBinaryReader(stream))
+        var actual = TestUtils.SerializeAndDeserializeBson(source);
-        {
-            var target = BsonSerializer.Deserialize<TOut>(reader);
 
-            return target;
+        Assert.Equal(actual.Id, source.Id);
-        }
     }
 }

backend/tests/Squidex.Infrastructure.Tests/States/ShardedServiceTests.cs

@@ -0,0 +1,82 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using Squidex.Hosting;
+
+namespace Squidex.Infrastructure.States;
+
+public class ShardedServiceTests
+{
+    private readonly IInner inner1 = A.Fake<IInner>();
+    private readonly IInner inner2 = A.Fake<IInner>();
+    private readonly TestSut sut;
+
+    public interface IInner : IInitializable
+    {
+    }
+
+    private class TestSut : ShardedService<IInner>
+    {
+        public TestSut(IShardingStrategy sharding, Func<string, IInner> factory)
+            : base(sharding, factory)
+        {
+        }
+
+        public IInner ExposeShard<TKey>(TKey key) where TKey : notnull
+        {
+            return Shard(key);
+        }
+    }
+
+    public ShardedServiceTests()
+    {
+        sut = new TestSut(new PartitionedSharding(2), key =>
+        {
+            if (key == "_0")
+            {
+                return inner1;
+            }
+            else
+            {
+                return inner2;
+            }
+        });
+
+        sut.InitializeAsync(default).Wait();
+    }
+
+    [Fact]
+    public async Task Should_initialize_shards()
+    {
+        await sut.InitializeAsync(default);
+
+        A.CallTo(() => inner1.InitializeAsync(A<CancellationToken>._))
+            .MustHaveHappened();
+
+        A.CallTo(() => inner2.InitializeAsync(A<CancellationToken>._))
+            .MustHaveHappened();
+    }
+
+    [Fact]
+    public async Task Should_release_shards()
+    {
+        await sut.ReleaseAsync(default);
+
+        A.CallTo(() => inner1.ReleaseAsync(A<CancellationToken>._))
+            .MustHaveHappened();
+
+        A.CallTo(() => inner2.ReleaseAsync(A<CancellationToken>._))
+            .MustHaveHappened();
+    }
+
+    [Fact]
+    public void Should_provide_shards()
+    {
+        Assert.Equal(inner1, sut.ExposeShard(0));
+        Assert.Equal(inner2, sut.ExposeShard(1));
+    }
+}

backend/tests/Squidex.Infrastructure.Tests/States/ShardingTests.cs

@@ -0,0 +1,57 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Infrastructure.States;
+
+public class ShardingTests
+{
+    [Fact]
+    public void Should_provide_single_shard_key()
+    {
+        var strategy = SingleSharding.Instance;
+
+        for (var i = 0; i < 1000; i++)
+        {
+            var shardKey = strategy.GetShardKey(Guid.NewGuid());
+
+            Assert.Equal(string.Empty, shardKey);
+        }
+    }
+
+    [Fact]
+    public void Should_provide_single_shard_keys()
+    {
+        var strategy = SingleSharding.Instance;
+
+        var shardKeys = strategy.GetShardKeys().ToArray();
+
+        Assert.Equal(new[] { string.Empty }, shardKeys);
+    }
+
+    [Fact]
+    public void Should_provide_partitioned_shard_key()
+    {
+        var strategy = new PartitionedSharding(3);
+
+        for (var i = 0; i < 1000; i++)
+        {
+            var shardKey = strategy.GetShardKey(Guid.NewGuid());
+
+            Assert.True(shardKey is "_0" or "_1" or "_2");
+        }
+    }
+
+    [Fact]
+    public void Should_provide_partitioned_shard_keys()
+    {
+        var strategy = new PartitionedSharding(3);
+
+        var shardKeys = strategy.GetShardKeys().ToArray();
+
+        Assert.Equal(new[] { "_0", "_1", "_2" }, shardKeys);
+    }
+}