From 526bf9ec90de63184aeb35a05d114af4441bc8f6 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Sun, 17 Dec 2023 15:48:49 +0100 Subject: [PATCH] Partitioning (#1054) * Sharding V1 * Revert naming. * Revert more changes. * Fix text index. * Full text improvement. * Improved full text. * Update names. * Revert search. --- .../Text/Azure/CommandFactory.cs | 12 +- .../Text/ElasticSearch/CommandFactory.cs | 12 +- .../Migrations/MongoDb/ConvertDocumentIds.cs | 10 +- .../Json/WorkflowTransitionSurrogate.cs | 1 - ...ongoAssetFolderRepository_SnapshotStore.cs | 1 - .../Assets/MongoAssetRepository.cs | 7 +- .../Assets/MongoShardedAssetRepository.cs | 90 ++++++ .../BsonUniqueContentIdSerializer.cs | 183 +++++++++++ .../Contents/MongoContentCollection.cs | 5 +- .../Contents/MongoContentRepository.cs | 10 +- .../Contents/MongoShardedContentRepository.cs | 98 ++++++ .../ShardedSnapshotStore.cs | 101 ++++++ .../Text/AtlasTextIndex.cs | 56 ++-- .../Text/CommandFactory.cs | 39 ++- .../Text/MongoShardedTextIndex.cs | 55 ++++ .../Text/MongoTextIndex.cs | 110 ++++++- .../Text/MongoTextIndexBase.cs | 126 ++------ .../Text/MongoTextIndexEntity.cs | 30 +- .../Text/MongoTextIndexerState.cs | 45 +-- .../Text/Tokenizer.cs | 8 +- .../Contents/ContentSchedulerProcess.cs | 2 +- .../Repositories/IContentRepository.cs | 2 +- .../Contents/Text/IndexCommand.cs | 9 +- .../Text/State/CachingTextIndexerState.cs | 10 +- .../Contents/Text/State/ITextIndexerState.cs | 4 +- .../Text/State/InMemoryTextIndexerState.cs | 8 +- .../Contents/Text/State/TextContentState.cs | 38 +-- .../Contents/Text/State/TextState.cs | 19 ++ .../Contents/Text/TextIndexingProcess.cs | 300 ++++++++++-------- .../Contents/Text/UniqueContentId.cs | 16 + .../Contents/Text/UpsertIndexEntry.cs | 3 - .../Squidex.Domain.Apps.Entities.csproj | 5 + .../MongoDb/BsonDomainIdSerializer.cs | 51 ++- .../src/Squidex.Infrastructure/DomainId.cs | 1 + .../States/ShardedService.cs | 68 ++++ .../Squidex.Infrastructure/States/Sharding.cs | 66 ++++ .../Squidex/Config/Domain/StoreServices.cs | 41 ++- backend/src/Squidex/appsettings.json | 9 + .../HandleRules/RuleServiceTests.cs | 1 - .../Contents/ContentSchedulerProcessTests.cs | 2 +- .../Contents/MongoDb/TokenizerTests.cs | 10 +- .../BsonUniqueContentIdSerializerTests.cs | 90 ++++++ .../Text/CachingTextIndexerStateTests.cs | 22 +- .../Contents/Text/MongoTextIndexFixture.cs | 2 +- .../Contents/Text/MongoTextIndexTests.cs | 8 +- .../Text/MongoTextIndexerStateFixture.cs | 38 +++ .../Text/MongoTextIndexerStateTests.cs | 95 ++++++ .../Contents/Text/TextIndexerTestsBase.cs | 34 ++ .../MongoDb/DomainIdSerializerTests.cs | 57 ++-- .../States/ShardedServiceTests.cs | 82 +++++ .../States/ShardingTests.cs | 57 ++++ 51 files changed, 1651 insertions(+), 498 deletions(-) create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoShardedAssetRepository.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/BsonUniqueContentIdSerializer.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/ShardedSnapshotStore.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextState.cs create mode 100644 backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UniqueContentId.cs create mode 100644 backend/src/Squidex.Infrastructure/States/ShardedService.cs create mode 100644 backend/src/Squidex.Infrastructure/States/Sharding.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/BsonUniqueContentIdSerializerTests.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexerStateFixture.cs create mode 100644 backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexerStateTests.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/States/ShardedServiceTests.cs create mode 100644 backend/tests/Squidex.Infrastructure.Tests/States/ShardingTests.cs diff --git a/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs b/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs index a5c0a9fd9..6a5564556 100644 --- a/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs +++ b/backend/extensions/Squidex.Extensions/Text/Azure/CommandFactory.cs @@ -60,10 +60,10 @@ public static class CommandFactory { var document = new SearchDocument { - ["docId"] = upsert.DocId.ToBase64(), - ["appId"] = upsert.AppId.Id.ToString(), - ["appName"] = upsert.AppId.Name, - ["contentId"] = upsert.ContentId.ToString(), + ["docId"] = upsert.ToDocId(), + ["appId"] = upsert.UniqueContentId.AppId.ToString(), + ["appName"] = string.Empty, + ["contentId"] = upsert.UniqueContentId.ToString(), ["schemaId"] = upsert.SchemaId.Id.ToString(), ["schemaName"] = upsert.SchemaId.Name, ["serveAll"] = upsert.ServeAll, @@ -94,7 +94,7 @@ public static class CommandFactory { var document = new SearchDocument { - ["docId"] = update.DocId.ToBase64(), + ["docId"] = update.ToDocId(), ["serveAll"] = update.ServeAll, ["servePublished"] = update.ServePublished }; @@ -104,7 +104,7 @@ public static class CommandFactory private static void DeleteEntry(DeleteIndexEntry delete, IList> batch) { - batch.Add(IndexDocumentsAction.Delete("docId", delete.DocId.ToBase64())); + batch.Add(IndexDocumentsAction.Delete("docId", delete.ToDocId().ToBase64())); } private static string ToBase64(this string value) diff --git a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs index 71bb9660c..0a9cd9857 100644 --- a/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs +++ b/backend/extensions/Squidex.Extensions/Text/ElasticSearch/CommandFactory.cs @@ -56,7 +56,7 @@ public static class CommandFactory { index = new { - _id = upsert.DocId, + _id = upsert.ToDocId(), _index = indexName } }); @@ -82,9 +82,9 @@ public static class CommandFactory args.Add(new { - appId = upsert.AppId.Id.ToString(), - appName = upsert.AppId.Name, - contentId = upsert.ContentId.ToString(), + appId = upsert.UniqueContentId.AppId.ToString(), + appName = string.Empty, + contentId = upsert.UniqueContentId.ContentId.ToString(), schemaId = upsert.SchemaId.Id.ToString(), schemaName = upsert.SchemaId.Name, serveAll = upsert.ServeAll, @@ -102,7 +102,7 @@ public static class CommandFactory { update = new { - _id = update.DocId, + _id = update.ToDocId(), _index = indexName } }); @@ -123,7 +123,7 @@ public static class CommandFactory { delete = new { - _id = delete.DocId, + _id = delete.ToDocId(), _index = indexName } }); diff --git a/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs b/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs index 35d1aaa30..78ce2c720 100644 --- a/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs +++ b/backend/src/Migrations/Migrations/MongoDb/ConvertDocumentIds.cs @@ -16,7 +16,7 @@ namespace Migrations.Migrations.MongoDb; public sealed class ConvertDocumentIds : MongoBase, IMigration { - private readonly IMongoDatabase database; + private readonly IMongoDatabase databaseDefault; private readonly IMongoDatabase databaseContent; private Scope scope; @@ -27,9 +27,9 @@ public sealed class ConvertDocumentIds : MongoBase, IMigration Contents } - public ConvertDocumentIds(IMongoDatabase database, IMongoDatabase databaseContent) + public ConvertDocumentIds(IMongoDatabase databaseDefault, IMongoDatabase databaseContent) { - this.database = database; + this.databaseDefault = databaseDefault; this.databaseContent = databaseContent; } @@ -58,8 +58,8 @@ public sealed class ConvertDocumentIds : MongoBase, IMigration switch (scope) { case Scope.Assets: - await RebuildAsync(database, ConvertParentId, "States_Assets", ct); - await RebuildAsync(database, ConvertParentId, "States_AssetFolders", ct); + await RebuildAsync(databaseDefault, ConvertParentId, "States_Assets", ct); + await RebuildAsync(databaseDefault, ConvertParentId, "States_AssetFolders", ct); break; case Scope.Contents: await RebuildAsync(databaseContent, null, "State_Contents_All", ct); diff --git a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/Json/WorkflowTransitionSurrogate.cs b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/Json/WorkflowTransitionSurrogate.cs index 7ab17d5e8..5e84c858f 100644 --- a/backend/src/Squidex.Domain.Apps.Core.Model/Contents/Json/WorkflowTransitionSurrogate.cs +++ b/backend/src/Squidex.Domain.Apps.Core.Model/Contents/Json/WorkflowTransitionSurrogate.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using Squidex.Domain.Apps.Core.Apps; using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Core.Contents.Json; diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs index 3a6c794d1..6b128d923 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs @@ -11,7 +11,6 @@ using Squidex.Domain.Apps.Core.Assets; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; using Squidex.Infrastructure.States; -using Squidex.Infrastructure.Translations; #pragma warning disable MA0048 // File name must match type name diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs index 3659caea0..61aa617e4 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository.cs @@ -21,16 +21,19 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets; public sealed partial class MongoAssetRepository : MongoRepositoryBase, IAssetRepository { private readonly MongoCountCollection countCollection; + private readonly string shardKey; static MongoAssetRepository() { MongoAssetEntity.RegisterClassMap(); } - public MongoAssetRepository(IMongoDatabase database, ILogger log) + public MongoAssetRepository(IMongoDatabase database, ILogger log, string shardKey) : base(database) { countCollection = new MongoCountCollection(database, log, CollectionName()); + + this.shardKey = shardKey; } public IMongoCollection GetInternalCollection() @@ -40,7 +43,7 @@ public sealed partial class MongoAssetRepository : MongoRepositoryBase collection, diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoShardedAssetRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoShardedAssetRepository.cs new file mode 100644 index 000000000..f2ebbca15 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoShardedAssetRepository.cs @@ -0,0 +1,90 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using MongoDB.Driver; +using Squidex.Domain.Apps.Core.Assets; +using Squidex.Domain.Apps.Entities.Assets.Repositories; +using Squidex.Infrastructure; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Assets; + +public sealed class MongoShardedAssetRepository : ShardedSnapshotStore, IAssetRepository +{ + public MongoShardedAssetRepository(IShardingStrategy sharding, Func factory) + : base(sharding, factory) + { + } + + public IEnumerable> GetInternalCollections() + { + return Shards.Select(x => x.GetInternalCollection()); + } + + public async Task FindAssetAsync(DomainId id, + CancellationToken ct = default) + { + Asset? result = null; + + foreach (var shard in Shards) + { + if ((result = await shard.FindAssetAsync(id, ct)) != null) + { + return result; + } + } + + return result; + } + + public Task FindAssetAsync(DomainId appId, DomainId id, bool allowDeleted, + CancellationToken ct = default) + { + return Shard(appId).FindAssetAsync(appId, id, allowDeleted, ct); + } + + public Task FindAssetByHashAsync(DomainId appId, string hash, string fileName, long fileSize, + CancellationToken ct = default) + { + return Shard(appId).FindAssetByHashAsync(appId, hash, fileName, fileSize, ct); + } + + public Task FindAssetBySlugAsync(DomainId appId, string slug, bool allowDeleted, + CancellationToken ct = default) + { + return Shard(appId).FindAssetBySlugAsync(appId, slug, allowDeleted, ct); + } + + public Task> QueryAsync(DomainId appId, DomainId? parentId, Q q, + CancellationToken ct = default) + { + return Shard(appId).QueryAsync(appId, parentId, q, ct); + } + + public Task> QueryChildIdsAsync(DomainId appId, DomainId parentId, + CancellationToken ct = default) + { + return Shard(appId).QueryChildIdsAsync(appId, parentId, ct); + } + + public Task> QueryIdsAsync(DomainId appId, HashSet ids, + CancellationToken ct = default) + { + return Shard(appId).QueryIdsAsync(appId, ids, ct); + } + + public IAsyncEnumerable StreamAll(DomainId appId, + CancellationToken ct = default) + { + return Shard(appId).StreamAll(appId, ct); + } + + protected override string GetShardKey(Asset state) + { + return GetShardKey(state.AppId.Id); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/BsonUniqueContentIdSerializer.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/BsonUniqueContentIdSerializer.cs new file mode 100644 index 000000000..7e50ba459 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/BsonUniqueContentIdSerializer.cs @@ -0,0 +1,183 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Text; +using MongoDB.Bson.Serialization; +using MongoDB.Bson.Serialization.Serializers; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.MongoDb; + +public sealed class BsonUniqueContentIdSerializer : SerializerBase +{ + private const byte GuidLength = 16; + private static readonly BsonUniqueContentIdSerializer Instance = new BsonUniqueContentIdSerializer(); + + public static void Register() + { + BsonSerializer.TryRegisterSerializer(Instance); + } + + private BsonUniqueContentIdSerializer() + { + } + + public static UniqueContentId NextAppId(DomainId appId) + { + static void IncrementByteArray(byte[] bytes) + { + for (var i = 0; i < bytes.Length; i++) + { + var value = bytes[i]; + if (value < byte.MaxValue) + { + value += 1; + bytes[i] = value; + break; + } + } + } + + if (Guid.TryParse(appId.ToString(), out var id)) + { + var bytes = id.ToByteArray(); + + IncrementByteArray(bytes); + + return new UniqueContentId(DomainId.Create(new Guid(bytes)), DomainId.Empty); + } + else + { + var bytes = Encoding.UTF8.GetBytes(appId.ToString()); + + IncrementByteArray(bytes); + + return new UniqueContentId(DomainId.Create(Encoding.UTF8.GetString(bytes)), DomainId.Empty); + } + } + + public override UniqueContentId Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) + { + var buffer = context.Reader.ReadBytes()!; + var offset = 0; + + static DomainId ReadId(byte[] buffer, ref int offset) + { + DomainId id; + + // If we have reached the end of the buffer then + if (offset >= buffer.Length) + { + return default; + } + + var length = buffer[offset++]; + // Special length indicator for all guids. + if (length == 0xFF) + { + id = DomainId.Create(new Guid(buffer.AsSpan(offset, GuidLength))); + offset += GuidLength; + } + else + { + id = DomainId.Create(Encoding.UTF8.GetString(buffer.AsSpan(offset, length))); + offset += length; + } + + return id; + } + + return new UniqueContentId(ReadId(buffer, ref offset), ReadId(buffer, ref offset)); + } + + public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, UniqueContentId value) + { + var appId = CheckId(value.AppId); + + var contentId = CheckId(value.ContentId); + + var isEmptyContentId = + contentId.IsGuid && + contentId.Guid == default; + + // Do not write empty Ids to the buffer to allow prefix searches. + var contentLength = !isEmptyContentId ? contentId.Length + 1 : 0; + + var bufferLength = appId.Length + 1 + contentLength; + var bufferArray = new byte[bufferLength]; + + var offset = Write(bufferArray, 0, + appId.IsGuid, + appId.Guid, + appId.Source, + appId.Length); + + if (!isEmptyContentId) + { + // Do not write the empty content id, so we can search for app as well. + Write(bufferArray, offset, + contentId.IsGuid, + contentId.Guid, + contentId.Source, + contentId.Length); + } + + static int Write(byte[] buffer, int offset, bool isGuid, Guid guid, string id, int idLength) + { + if (isGuid) + { + // Special length indicator for all guids. + buffer[offset++] = 0xFF; + WriteGuid(buffer.AsSpan(offset), guid); + + return offset + GuidLength; + } + else + { + // We assume that we use relatively small IDs, not longer than 254 bytes. + buffer[offset++] = (byte)idLength; + WriteString(buffer.AsSpan(offset), id); + + return offset + idLength; + } + } + + context.Writer.WriteBytes(bufferArray); + } + + private static (int Length, bool IsGuid, Guid Guid, string Source) CheckId(DomainId id) + { + var source = id.ToString(); + + var idIsGuid = Guid.TryParse(source, out var idGuid); + var idLength = GuidLength; + + if (!idIsGuid) + { + idLength = (byte)Encoding.UTF8.GetByteCount(source); + + // We only use a single byte to write the length, therefore we do not allow large strings. + if (idLength > 254) + { + ThrowHelper.InvalidOperationException("Cannot write long IDs."); + } + } + + return (idLength, idIsGuid, idGuid, source); + } + + private static void WriteString(Span span, string id) + { + Encoding.UTF8.GetBytes(id, span); + } + + private static void WriteGuid(Span span, Guid guid) + { + guid.TryWriteBytes(span); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs index c768b4d65..59bcf7abb 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentCollection.cs @@ -9,7 +9,6 @@ using Microsoft.Extensions.Logging; using MongoDB.Driver; using NodaTime; using Squidex.Domain.Apps.Core.Apps; -using Squidex.Domain.Apps.Core.Assets; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Core.Schemas; using Squidex.Domain.Apps.Entities.MongoDb.Contents.Operations; @@ -97,9 +96,11 @@ public sealed class MongoContentCollection : MongoRepositoryBase x.CreateIndexes()), ct); } - public Task ResetScheduledAsync(DomainId documentId, + public Task ResetScheduledAsync(DomainId appId, DomainId contentId, CancellationToken ct) { + var documentId = DomainId.Combine(appId, contentId); + return Collection.UpdateOneAsync( x => x.DocumentId == documentId, Update diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 276b37a85..e171ef434 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -42,7 +42,7 @@ public partial class MongoContentRepository : MongoBase, ICo MongoContentEntity.RegisterClassMap(); } - public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider, + public MongoContentRepository(IMongoDatabase database, IAppProvider appProvider, string shardKey, IOptions options, ILogger log) { this.appProvider = appProvider; @@ -50,11 +50,11 @@ public partial class MongoContentRepository : MongoBase, ICo this.options = options.Value; collectionComplete = - new MongoContentCollection("States_Contents_All3", database, log, + new MongoContentCollection($"States_Contents_All3{shardKey}", database, log, ReadPreference.Primary, options.Value.OptimizeForSelfHosting); collectionPublished = - new MongoContentCollection("States_Contents_Published3", database, log, + new MongoContentCollection($"States_Contents_Published3{shardKey}", database, log, ReadPreference.Secondary, options.Value.OptimizeForSelfHosting); } @@ -124,10 +124,10 @@ public partial class MongoContentRepository : MongoBase, ICo return GetCollection(scope).QueryIdsAsync(app, schema, filterNode, ct); } - public Task ResetScheduledAsync(DomainId documentId, SearchScope scope, + public Task ResetScheduledAsync(DomainId appId, DomainId contentId, SearchScope scope, CancellationToken ct = default) { - return GetCollection(SearchScope.All).ResetScheduledAsync(documentId, ct); + return GetCollection(SearchScope.All).ResetScheduledAsync(appId, contentId, ct); } private MongoContentCollection GetCollection(SearchScope scope) diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs new file mode 100644 index 000000000..a55296fdb --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoShardedContentRepository.cs @@ -0,0 +1,98 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Runtime.CompilerServices; +using NodaTime; +using Squidex.Domain.Apps.Core.Apps; +using Squidex.Domain.Apps.Core.Contents; +using Squidex.Domain.Apps.Core.Schemas; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.Repositories; +using Squidex.Infrastructure; +using Squidex.Infrastructure.Queries; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Contents; + +public sealed class MongoShardedContentRepository : ShardedSnapshotStore, IContentRepository +{ + public MongoShardedContentRepository(IShardingStrategy sharding, Func factory) + : base(sharding, factory) + { + } + + public Task FindContentAsync(App app, Schema schema, DomainId id, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).FindContentAsync(app, schema, id, scope, ct); + } + + public Task HasReferrersAsync(App app, DomainId reference, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).HasReferrersAsync(app, reference, scope, ct); + } + + public Task> QueryAsync(App app, List schemas, Q q, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).QueryAsync(app, schemas, q, scope, ct); + } + + public Task> QueryAsync(App app, Schema schema, Q q, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).QueryAsync(app, schema, q, scope, ct); + } + + public Task> QueryIdsAsync(App app, Schema schema, FilterNode filterNode, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).QueryIdsAsync(app, schema, filterNode, scope, ct); + } + + public Task> QueryIdsAsync(App app, HashSet ids, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).QueryIdsAsync(app, ids, scope, ct); + } + + public Task ResetScheduledAsync(DomainId appId, DomainId id, SearchScope scope, + CancellationToken ct = default) + { + return Shard(appId).ResetScheduledAsync(appId, id, scope, ct); + } + + public IAsyncEnumerable StreamAll(DomainId appId, HashSet? schemaIds, SearchScope scope, + CancellationToken ct = default) + { + return Shard(appId).StreamAll(appId, schemaIds, scope, ct); + } + + public IAsyncEnumerable StreamReferencing(DomainId appId, DomainId references, int take, SearchScope scope, + CancellationToken ct = default) + { + return Shard(appId).StreamReferencing(appId, references, take, scope, ct); + } + + public async IAsyncEnumerable StreamScheduledWithoutDataAsync(Instant now, SearchScope scope, + [EnumeratorCancellation] CancellationToken ct = default) + { + foreach (var shard in Shards) + { + await foreach (var content in shard.StreamScheduledWithoutDataAsync(now, scope, ct)) + { + yield return content; + } + } + } + + protected override string GetShardKey(WriteContent state) + { + return GetShardKey(state.AppId.Id); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/ShardedSnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/ShardedSnapshotStore.cs new file mode 100644 index 000000000..398ac08b0 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/ShardedSnapshotStore.cs @@ -0,0 +1,101 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System.Runtime.CompilerServices; +using Squidex.Domain.Apps.Core.Apps; +using Squidex.Infrastructure; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.MongoDb; + +public abstract class ShardedSnapshotStore : ShardedService, ISnapshotStore, IDeleter where T : ISnapshotStore, IDeleter +{ + protected ShardedSnapshotStore(IShardingStrategy sharding, Func factory) + : base(sharding, factory) + { + } + + protected abstract string GetShardKey(TState state); + + public Task WriteAsync(SnapshotWriteJob job, + CancellationToken ct = default) + { + var shard = Shard(GetShardKey(job.Value)); + + return shard.WriteAsync(job, ct); + } + + public Task> ReadAsync(DomainId key, + CancellationToken ct = default) + { + var shard = Shard(GetAppId(key)); + + return shard.ReadAsync(key, ct); + } + + public Task RemoveAsync(DomainId key, + CancellationToken ct = default) + { + var shard = Shard(GetAppId(key)); + + return shard.RemoveAsync(key, ct); + } + + public Task DeleteAppAsync(App app, + CancellationToken ct) + { + var shard = Shard(app.Id); + + return shard.DeleteAppAsync(app, ct); + } + + public async IAsyncEnumerable> ReadAllAsync( + [EnumeratorCancellation] CancellationToken ct = default) + { + foreach (var shard in Shards) + { + await foreach (var asset in shard.ReadAllAsync(ct)) + { + yield return asset; + } + } + } + + public async Task ClearAsync( + CancellationToken ct = default) + { + foreach (var shard in Shards) + { + await shard.ClearAsync(ct); + } + } + + public async Task WriteManyAsync(IEnumerable> jobs, + CancellationToken ct = default) + { + // Some commands might share a shared, therefore we don't group by app id. + foreach (var byShard in jobs.GroupBy(c => GetShardKey(c.Value))) + { + var shard = Shard(byShard.Key); + + await shard.WriteManyAsync(byShard.ToArray(), ct); + } + } + + private static DomainId GetAppId(DomainId key) + { + // This is a leaky abstraction, but the only option to implement that in a fast way. + var parts = key.ToString().Split(DomainId.IdSeparator); + + if (parts.Length != 2) + { + throw new InvalidOperationException("The key does not contain an app id."); + } + + return DomainId.Create(parts[0]); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs index 8afb76d56..389e926b5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/AtlasTextIndex.cs @@ -25,15 +25,15 @@ public sealed class AtlasTextIndex : MongoTextIndexBase options) - : base(database) + public AtlasTextIndex(IMongoDatabase database, IHttpClientFactory atlasClient, IOptions atlasOptions, string shardKey) + : base(database, shardKey, new CommandFactory>(BuildTexts)) { - this.httpClientFactory = httpClientFactory; - this.options = options.Value; + this.atlasClient = atlasClient; + this.atlasOptions = atlasOptions.Value; } protected override async Task SetupCollectionAsync(IMongoCollection>> collection, @@ -41,31 +41,10 @@ public sealed class AtlasTextIndex : MongoTextIndexBase BuildTexts(Dictionary source) - { - var texts = new Dictionary(); - - foreach (var (key, value) in source) - { - var text = value; - - var languageCode = AtlasIndexDefinition.GetFieldName(key); - - if (texts.TryGetValue(languageCode, out var existing)) - { - text = $"{existing} {value}"; - } - - texts[languageCode] = text; - } - - return texts; - } - public override async Task?> SearchAsync(App app, TextQuery query, SearchScope scope, CancellationToken ct = default) { @@ -158,4 +137,25 @@ public sealed class AtlasTextIndex : MongoTextIndexBase x.ContentId).ToList(); } + + private static Dictionary BuildTexts(Dictionary source) + { + var texts = new Dictionary(); + + foreach (var (key, value) in source) + { + var text = value; + + var languageCode = AtlasIndexDefinition.GetFieldName(key); + + if (texts.TryGetValue(languageCode, out var existing)) + { + text = $"{existing} {value}"; + } + + texts[languageCode] = text; + } + + return texts; + } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs index 347be17e1..de161a774 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/CommandFactory.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using MongoDB.Bson; using MongoDB.Driver; using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Infrastructure.MongoDb; @@ -41,17 +42,17 @@ public sealed class CommandFactory : MongoBase> where writes.Add( new UpdateOneModel>( Filter.And( - Filter.Eq(x => x.DocId, upsert.DocId), + FilterByCommand(upsert), Filter.Exists(x => x.GeoField, false), Filter.Exists(x => x.GeoObject, false)), Update .Set(x => x.ServeAll, upsert.ServeAll) .Set(x => x.ServePublished, upsert.ServePublished) .Set(x => x.Texts, BuildTexts(upsert)) - .SetOnInsert(x => x.Id, Guid.NewGuid().ToString()) - .SetOnInsert(x => x.DocId, upsert.DocId) - .SetOnInsert(x => x.AppId, upsert.AppId.Id) - .SetOnInsert(x => x.ContentId, upsert.ContentId) + .SetOnInsert(x => x.Id, ObjectId.GenerateNewId()) + .SetOnInsert(x => x.AppId, upsert.UniqueContentId.AppId) + .SetOnInsert(x => x.ContentId, upsert.UniqueContentId.ContentId) + .SetOnInsert(x => x.Stage, upsert.Stage) .SetOnInsert(x => x.SchemaId, upsert.SchemaId.Id)) { IsUpsert = true @@ -62,9 +63,9 @@ public sealed class CommandFactory : MongoBase> where if (!upsert.IsNew) { writes.Add( - new DeleteOneModel>( + new DeleteManyModel>( Filter.And( - Filter.Eq(x => x.DocId, upsert.DocId), + FilterByCommand(upsert), Filter.Exists(x => x.GeoField), Filter.Exists(x => x.GeoObject)))); } @@ -75,15 +76,15 @@ public sealed class CommandFactory : MongoBase> where new InsertOneModel>( new MongoTextIndexEntity { - Id = Guid.NewGuid().ToString(), - AppId = upsert.AppId.Id, - DocId = upsert.DocId, - ContentId = upsert.ContentId, + Id = ObjectId.GenerateNewId(), + AppId = upsert.UniqueContentId.AppId, + ContentId = upsert.UniqueContentId.ContentId, GeoField = field, GeoObject = geoObject, SchemaId = upsert.SchemaId.Id, ServeAll = upsert.ServeAll, - ServePublished = upsert.ServePublished + ServePublished = upsert.ServePublished, + Stage = upsert.Stage, })); } } @@ -98,7 +99,7 @@ public sealed class CommandFactory : MongoBase> where { writes.Add( new UpdateOneModel>( - Filter.Eq(x => x.DocId, update.DocId), + FilterByCommand(update), Update .Set(x => x.ServeAll, update.ServeAll) .Set(x => x.ServePublished, update.ServePublished))); @@ -107,7 +108,15 @@ public sealed class CommandFactory : MongoBase> where private static void DeleteEntry(DeleteIndexEntry delete, List>> writes) { writes.Add( - new DeleteOneModel>( - Filter.Eq(x => x.DocId, delete.DocId))); + new DeleteManyModel>( + FilterByCommand(delete))); + } + + private static FilterDefinition> FilterByCommand(IndexCommand command) + { + return Filter.And( + Filter.Eq(x => x.AppId, command.UniqueContentId.AppId), + Filter.Eq(x => x.ContentId, command.UniqueContentId.ContentId), + Filter.Eq(x => x.Stage, command.Stage)); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs new file mode 100644 index 000000000..92b0e2f64 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoShardedTextIndex.cs @@ -0,0 +1,55 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Core.Apps; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Infrastructure; +using Squidex.Infrastructure.States; + +namespace Squidex.Domain.Apps.Entities.MongoDb.Text; + +public sealed class MongoShardedTextIndex : ShardedService>, ITextIndex where T : class +{ + public MongoShardedTextIndex(IShardingStrategy sharding, Func> factory) + : base(sharding, factory) + { + } + + public async Task ClearAsync( + CancellationToken ct = default) + { + foreach (var shard in Shards) + { + await shard.ClearAsync(ct); + } + } + + public async Task ExecuteAsync(IndexCommand[] commands, + CancellationToken ct = default) + { + // Some commands might share a shared, therefore we don't group by app id. + foreach (var byShard in commands.GroupBy(c => GetShardKey(c.UniqueContentId.AppId))) + { + var shard = Shard(byShard.Key); + + await shard.ExecuteAsync(byShard.ToArray(), ct); + } + } + + public Task?> SearchAsync(App app, TextQuery query, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).SearchAsync(app, query, scope, ct); + } + + public Task?> SearchAsync(App app, GeoQuery query, SearchScope scope, + CancellationToken ct = default) + { + return Shard(app.Id).SearchAsync(app, query, scope, ct); + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs index 5cd40a069..2b0a07db5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndex.cs @@ -6,13 +6,30 @@ // ========================================================================== using MongoDB.Driver; +using Squidex.Domain.Apps.Core.Apps; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.MongoDb.Text; public sealed class MongoTextIndex : MongoTextIndexBase> { - public MongoTextIndex(IMongoDatabase database) - : base(database) + private record struct SearchOperation + { + required public App App { get; init; } + + required public List<(DomainId Id, double Score)> Results { get; init; } + + required public string SearchTerms { get; init; } + + required public int Take { get; set; } + + required public SearchScope SearchScope { get; init; } + } + + public MongoTextIndex(IMongoDatabase database, string shardKey) + : base(database, shardKey, new CommandFactory>(BuildTexts)) { } @@ -24,19 +41,92 @@ public sealed class MongoTextIndex : MongoTextIndexBase>>( Index - .Text("t.t") .Ascending(x => x.AppId) - .Ascending(x => x.ServeAll) - .Ascending(x => x.ServePublished) - .Ascending(x => x.SchemaId)), + .Text("t.t")), cancellationToken: ct); } - protected override List BuildTexts(Dictionary source) + public override async Task?> SearchAsync(App app, TextQuery query, SearchScope scope, + CancellationToken ct = default) { - return source.Select(x => new MongoTextIndexEntityText + Guard.NotNull(app); + Guard.NotNull(query); + + if (string.IsNullOrWhiteSpace(query.Text)) { - Text = Tokenizer.TokenizerTerms(x.Value, x.Key) - }).ToList(); + return null; + } + + // Use a custom tokenizer to leverage stop words from multiple languages. + var search = new SearchOperation + { + App = app, + SearchTerms = Tokenizer.Query(query.Text), + SearchScope = scope, + Results = [], + Take = query.Take + }; + + if (query.RequiredSchemaIds?.Count > 0) + { + await SearchBySchemaAsync(search, query.RequiredSchemaIds, 1, ct); + } + else if (query.PreferredSchemaId == null) + { + await SearchByAppAsync(search, 1, ct); + } + else + { + // We cannot write queries that prefer results from the same schema, therefore make two queries. + search.Take /= 2; + + // Increasing the scoring of the results from the schema by 10 percent. + await SearchBySchemaAsync(search, Enumerable.Repeat(query.PreferredSchemaId.Value, 1), 1.1, ct); + await SearchByAppAsync(search, 1, ct); + } + + return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); + } + + private Task SearchBySchemaAsync(SearchOperation search, IEnumerable schemaIds, double factor, + CancellationToken ct = default) + { + var filter = + Filter.And( + Filter.Eq(x => x.AppId, search.App.Id), + Filter.Text(search.SearchTerms, "none"), + Filter.In(x => x.SchemaId, schemaIds), + FilterByScope(search.SearchScope)); + + return SearchAsync(search, filter, factor, ct); + } + + private Task SearchByAppAsync(SearchOperation search, double factor, + CancellationToken ct = default) + { + var filter = + Filter.And( + Filter.Eq(x => x.AppId, search.App.Id), + Filter.Text(search.SearchTerms, "none"), + FilterByScope(search.SearchScope)); + + return SearchAsync(search, filter, factor, ct); + } + + private async Task SearchAsync(SearchOperation search, FilterDefinition>> filter, double factor, + CancellationToken ct = default) + { + var byText = + await GetCollection(search.SearchScope).Find(filter).Limit(search.Take) + .Project(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score")) + .ToListAsync(ct); + + search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor))); + } + + private static List BuildTexts(Dictionary source) + { + // Use a custom tokenizer to leverage stop words from multiple languages. + return source.Select(x => MongoTextIndexEntityText.FromText(Tokenizer.Terms(x.Value, x.Key))).ToList(); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs index 8840cbced..80076f1b9 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexBase.cs @@ -18,16 +18,17 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Text; public abstract class MongoTextIndexBase : MongoRepositoryBase>, ITextIndex, IDeleter where T : class { - private readonly CommandFactory commandFactory; + private readonly CommandFactory factory; + private readonly string shardKey; protected sealed class MongoTextResult { [BsonId] [BsonElement] - public string Id { get; set; } + public ObjectId Id { get; set; } [BsonRequired] - [BsonElement("_ci")] + [BsonElement("c")] public DomainId ContentId { get; set; } [BsonIgnoreIfDefault] @@ -35,23 +36,11 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase Results { get; } = []; - - public string SearchTerms { get; set; } - - public App App { get; set; } - - public SearchScope SearchScope { get; set; } - } - - protected MongoTextIndexBase(IMongoDatabase database) + protected MongoTextIndexBase(IMongoDatabase database, string shardKey, CommandFactory factory) : base(database) { -#pragma warning disable MA0056 // Do not call overridable members in constructor - commandFactory = new CommandFactory(BuildTexts); -#pragma warning restore MA0056 // Do not call overridable members in constructor + this.shardKey = shardKey; + this.factory = factory; } protected override Task SetupCollectionAsync(IMongoCollection> collection, @@ -60,13 +49,13 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase>( - Index.Ascending(x => x.DocId)), + Index + .Ascending(x => x.AppId) + .Ascending(x => x.ContentId)), new CreateIndexModel>( Index .Ascending(x => x.AppId) - .Ascending(x => x.ServeAll) - .Ascending(x => x.ServePublished) .Ascending(x => x.SchemaId) .Ascending(x => x.GeoField) .Geo2DSphere(x => x.GeoObject)) @@ -75,11 +64,9 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase source); - async Task IDeleter.DeleteAppAsync(App app, CancellationToken ct) { @@ -93,7 +80,7 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase : MongoRepositoryBase error.Code != MongoDbErrorCodes.Errror16755_InvalidGeoData)) { throw; @@ -121,12 +108,14 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase x.AppId, app.Id), Filter.Eq(x => x.SchemaId, query.SchemaId), - Filter_ByScope(scope), - Filter.GeoWithinCenterSphere(x => x.GeoObject, query.Longitude, query.Latitude, query.Radius / 6378100)); + Filter.Eq(x => x.GeoField, query.Field), + Filter.GeoWithinCenterSphere(x => x.GeoObject, query.Longitude, query.Latitude, query.Radius / 6378100), + FilterByScope(scope)); var byGeo = await GetCollection(scope).Find(findFilter).Limit(query.Take) @@ -136,83 +125,10 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase x.ContentId).ToList(); } - public virtual async Task?> SearchAsync(App app, TextQuery query, SearchScope scope, - CancellationToken ct = default) - { - Guard.NotNull(app); - Guard.NotNull(query); - - if (string.IsNullOrWhiteSpace(query.Text)) - { - return null; - } - - var search = new SearchOperation - { - App = app, - SearchTerms = Tokenizer.TokenizeQuery(query.Text), - SearchScope = scope - }; - - if (query.RequiredSchemaIds?.Count > 0) - { - await SearchBySchemaAsync(search, query.RequiredSchemaIds, query.Take, 1, ct); - } - else if (query.PreferredSchemaId == null) - { - await SearchByAppAsync(search, query.Take, 1, ct); - } - else - { - var halfBucket = query.Take / 2; - - var schemaIds = Enumerable.Repeat(query.PreferredSchemaId.Value, 1); - - await SearchBySchemaAsync(search, schemaIds, halfBucket, 1.1, ct); - await SearchByAppAsync(search, halfBucket, 1, ct); - } - - return search.Results.OrderByDescending(x => x.Score).Select(x => x.Id).Distinct().ToList(); - } - - private Task SearchBySchemaAsync(SearchOperation search, IEnumerable schemaIds, int take, double factor, - CancellationToken ct = default) - { - var filter = - Filter.And( - Filter.Eq(x => x.AppId, search.App.Id), - Filter.In(x => x.SchemaId, schemaIds), - Filter_ByScope(search.SearchScope), - Filter.Text(search.SearchTerms, "none")); - - return SearchAsync(search, filter, take, factor, ct); - } - - private Task SearchByAppAsync(SearchOperation search, int take, double factor, - CancellationToken ct = default) - { - var filter = - Filter.And( - Filter.Eq(x => x.AppId, search.App.Id), - Filter.Exists(x => x.SchemaId), - Filter_ByScope(search.SearchScope), - Filter.Text(search.SearchTerms, "none")); - - return SearchAsync(search, filter, take, factor, ct); - } - - private async Task SearchAsync(SearchOperation search, FilterDefinition> filter, int take, double factor, - CancellationToken ct = default) - { - var byText = - await GetCollection(search.SearchScope).Find(filter).Limit(take) - .Project(Projection.Include(x => x.ContentId).MetaTextScore("score")).Sort(Sort.MetaTextScore("score")) - .ToListAsync(ct); - - search.Results.AddRange(byText.Select(x => (x.ContentId, x.Score * factor))); - } + public abstract Task?> SearchAsync(App app, TextQuery query, SearchScope scope, + CancellationToken ct = default); - private static FilterDefinition> Filter_ByScope(SearchScope scope) + protected static FilterDefinition> FilterByScope(SearchScope scope) { if (scope == SearchScope.All) { @@ -224,7 +140,7 @@ public abstract class MongoTextIndexBase : MongoRepositoryBase> GetCollection(SearchScope scope) + protected IMongoCollection> GetCollection(SearchScope scope) { if (scope == SearchScope.All) { diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs index 165112aeb..526694bfb 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexEntity.cs @@ -16,34 +16,34 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Text; public sealed class MongoTextIndexEntity { [BsonId] - [BsonElement("_id")] - public string Id { get; set; } + public ObjectId Id { get; set; } [BsonRequired] - [BsonElement(nameof(DocId))] - public string DocId { get; set; } + [BsonElement("a")] + [BsonRepresentation(BsonType.String)] + public DomainId AppId { get; set; } [BsonRequired] - [BsonElement("_ci")] + [BsonElement("s")] [BsonRepresentation(BsonType.String)] - public DomainId ContentId { get; set; } + public DomainId SchemaId { get; set; } [BsonRequired] - [BsonElement("_ai")] + [BsonElement("c")] [BsonRepresentation(BsonType.String)] - public DomainId AppId { get; set; } + public DomainId ContentId { get; set; } [BsonRequired] - [BsonElement("_si")] - [BsonRepresentation(BsonType.String)] - public DomainId SchemaId { get; set; } + [BsonElement("x")] + [BsonRepresentation(BsonType.Int32)] + public byte Stage { get; set; } [BsonRequired] - [BsonElement("fa")] + [BsonElement("e")] public bool ServeAll { get; set; } [BsonRequired] - [BsonElement("fp")] + [BsonElement("p")] public bool ServePublished { get; set; } [BsonIgnoreIfNull] @@ -51,11 +51,11 @@ public sealed class MongoTextIndexEntity public T Texts { get; set; } [BsonIgnoreIfNull] - [BsonElement("gf")] + [BsonElement("g")] public string GeoField { get; set; } [BsonIgnoreIfNull] - [BsonElement("go")] + [BsonElement("o")] [BsonJson] [BsonRepresentation(BsonType.Document)] public Geometry GeoObject { get; set; } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs index 85bdc0c24..6102e852f 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/MongoTextIndexerState.cs @@ -8,6 +8,7 @@ using MongoDB.Bson.Serialization; using MongoDB.Driver; using Squidex.Domain.Apps.Core.Apps; +using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text.State; using Squidex.Infrastructure; using Squidex.Infrastructure.MongoDb; @@ -18,21 +19,14 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase(cm => { - cm.MapIdField(x => x.UniqueContentId); - - cm.MapProperty(x => x.AppId) - .SetElementName("a"); - - cm.MapProperty(x => x.DocIdCurrent) - .SetElementName("c"); + cm.MapIdProperty(x => x.UniqueContentId); - cm.MapProperty(x => x.DocIdNew) - .SetElementName("n").SetIgnoreIfNull(true); - - cm.MapProperty(x => x.DocIdForPublished) - .SetElementName("p").SetIgnoreIfNull(true); + cm.MapProperty(x => x.State) + .SetElementName("s"); }); } @@ -41,16 +35,6 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase collection, - CancellationToken ct) - { - return collection.Indexes.CreateManyAsync(new[] - { - new CreateIndexModel( - Index.Ascending(x => x.AppId)) - }, ct); - } - protected override string CollectionName() { return "TextIndexerState"; @@ -59,10 +43,15 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase x.AppId, app.Id), ct); + var filter = + Filter.And( + Filter.Gte(x => x.UniqueContentId, new UniqueContentId(app.Id, DomainId.Empty)), + Filter.Lt(x => x.UniqueContentId, BsonUniqueContentIdSerializer.NextAppId(app.Id))); + + await Collection.DeleteManyAsync(filter, ct); } - public async Task> GetAsync(HashSet ids, + public async Task> GetAsync(HashSet ids, CancellationToken ct = default) { var entities = await Collection.Find(Filter.In(x => x.UniqueContentId, ids)).ToListAsync(ct); @@ -77,7 +66,7 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase( @@ -88,9 +77,9 @@ public sealed class MongoTextIndexerState : MongoRepositoryBase( Filter.Eq(x => x.UniqueContentId, update.UniqueContentId), update) - { - IsUpsert = true - }); + { + IsUpsert = true + }); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs index 8ae968d34..3eaeb61e6 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Text/Tokenizer.cs @@ -59,7 +59,7 @@ public static class Tokenizer } } - public static string TokenizeQuery(string query) + public static string Query(string query) { query = query.Trim(); @@ -73,10 +73,10 @@ public static class Tokenizer textReader.Read(); } - return TokenizeWord(textReader, textLanguage); + return Word(textReader, textLanguage); } - public static string TokenizerTerms(string query, string language) + public static string Terms(string query, string language) { var stopWords = string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ? @@ -87,7 +87,7 @@ public static class Tokenizer return Tokenize(new StringReader(query), stopWords); } - private static string TokenizeWord(TextReader reader, string language) + private static string Word(TextReader reader, string language) { var stopWords = string.Equals(language, InvariantPartitioning.Key, StringComparison.OrdinalIgnoreCase) ? diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentSchedulerProcess.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentSchedulerProcess.cs index e5228d40f..85242dd7c 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentSchedulerProcess.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/ContentSchedulerProcess.cs @@ -93,7 +93,7 @@ public sealed class ContentSchedulerProcess : IBackgroundProcess } catch (DomainObjectNotFoundException) { - await contentRepository.ResetScheduledAsync(content.UniqueId, default); + await contentRepository.ResetScheduledAsync(content.AppId.Id, id, default); } catch (Exception ex) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs index 8b9aa2d39..c3ea96fe5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Repositories/IContentRepository.cs @@ -43,6 +43,6 @@ public interface IContentRepository Task HasReferrersAsync(App app, DomainId reference, SearchScope scope, CancellationToken ct = default); - Task ResetScheduledAsync(DomainId documentId, SearchScope scope, + Task ResetScheduledAsync(DomainId appId, DomainId contentId, SearchScope scope, CancellationToken ct = default); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs index 9e5f31881..d43a90b61 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs @@ -11,9 +11,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text; public abstract class IndexCommand { - public NamedId AppId { get; set; } + public UniqueContentId UniqueContentId { get; set; } public NamedId SchemaId { get; set; } - public string DocId { get; set; } + public byte Stage { get; set; } + + public string ToDocId() + { + return $"{UniqueContentId.AppId}__{UniqueContentId.ContentId}_{Stage}"; + } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs index 9651dd32c..b1ea5d6f5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs @@ -13,7 +13,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State; public sealed class CachingTextIndexerState : ITextIndexerState { private readonly ITextIndexerState inner; - private readonly LRUCache> cache = new LRUCache>(10000); + private readonly LRUCache> cache = new LRUCache>(10000); public CachingTextIndexerState(ITextIndexerState inner) { @@ -30,14 +30,14 @@ public sealed class CachingTextIndexerState : ITextIndexerState cache.Clear(); } - public async Task> GetAsync(HashSet ids, + public async Task> GetAsync(HashSet ids, CancellationToken ct = default) { Guard.NotNull(ids); - var missingIds = new HashSet(); + var missingIds = new HashSet(); - var result = new Dictionary(); + var result = new Dictionary(); foreach (var id in ids) { @@ -81,7 +81,7 @@ public sealed class CachingTextIndexerState : ITextIndexerState foreach (var update in updates) { - if (update.IsDeleted) + if (update.State == TextState.Deleted) { cache.Set(update.UniqueContentId, Tuple.Create(null)); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs index 97f5fb802..2d5d3ff5f 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs @@ -5,13 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using Squidex.Infrastructure; - namespace Squidex.Domain.Apps.Entities.Contents.Text.State; public interface ITextIndexerState { - Task> GetAsync(HashSet ids, + Task> GetAsync(HashSet ids, CancellationToken ct = default); Task SetAsync(List updates, diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs index a733d0b06..20572cd67 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs @@ -11,7 +11,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State; public sealed class InMemoryTextIndexerState : ITextIndexerState { - private readonly Dictionary states = []; + private readonly Dictionary states = []; public Task ClearAsync( CancellationToken ct = default) @@ -21,12 +21,12 @@ public sealed class InMemoryTextIndexerState : ITextIndexerState return Task.CompletedTask; } - public Task> GetAsync(HashSet ids, + public Task> GetAsync(HashSet ids, CancellationToken ct = default) { Guard.NotNull(ids); - var result = new Dictionary(); + var result = new Dictionary(); foreach (var id in ids) { @@ -46,7 +46,7 @@ public sealed class InMemoryTextIndexerState : ITextIndexerState foreach (var update in updates) { - if (update.IsDeleted) + if (update.State == TextState.Deleted) { states.Remove(update.UniqueContentId); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs index 0efba07c8..1a1ad42cc 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs @@ -5,45 +5,11 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using Squidex.Infrastructure; - namespace Squidex.Domain.Apps.Entities.Contents.Text.State; public sealed class TextContentState { - public DomainId AppId { get; set; } - - public DomainId UniqueContentId { get; set; } - - public string DocIdCurrent { get; set; } - - public string? DocIdNew { get; set; } - - public string? DocIdForPublished { get; set; } - - public bool IsDeleted { get; set; } - - public void GenerateDocIdNew() - { - if (DocIdCurrent?.EndsWith("_2", StringComparison.Ordinal) != false) - { - DocIdNew = $"{UniqueContentId}_1"; - } - else - { - DocIdNew = $"{UniqueContentId}_2"; - } - } + public UniqueContentId UniqueContentId { get; set; } - public void GenerateDocIdCurrent() - { - if (DocIdNew?.EndsWith("_2", StringComparison.Ordinal) != false) - { - DocIdCurrent = $"{UniqueContentId}_1"; - } - else - { - DocIdCurrent = $"{UniqueContentId}_2"; - } - } + public TextState State { get; set; } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextState.cs new file mode 100644 index 000000000..ae230fb28 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextState.cs @@ -0,0 +1,19 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.Contents.Text.State; + +public enum TextState +{ + Stage0_Draft__Stage1_None, + Stage0_Published__Stage1_None, + Stage0_Published__Stage1_Draft, + Stage1_Draft__Stage0_None, + Stage1_Published__Stage0_None, + Stage1_Published__Stage0_Draft, + Deleted, +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs index 14e6be846..d3703de56 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using LibGit2Sharp; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Contents.Text.State; using Squidex.Domain.Apps.Events.Contents; @@ -16,7 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text; public sealed class TextIndexingProcess : IEventConsumer { - private const string NotFound = "<404>"; private readonly IJsonSerializer serializer; private readonly ITextIndex textIndex; private readonly ITextIndexerState textIndexerState; @@ -25,7 +25,7 @@ public sealed class TextIndexingProcess : IEventConsumer public int BatchDelay => 1000; - public string Name => "TextIndexer5"; + public string Name => "TextIndexer6"; public StreamFilter EventsFilter { get; } = StreamFilter.Prefix("content-"); @@ -36,14 +36,15 @@ public sealed class TextIndexingProcess : IEventConsumer private sealed class Updates { - private readonly Dictionary states; + private readonly Dictionary currentState; + private readonly Dictionary currentUpdates; + private readonly Dictionary<(UniqueContentId, byte), IndexCommand> commands = []; private readonly IJsonSerializer serializer; - private readonly Dictionary updates = []; - private readonly Dictionary commands = []; - public Updates(Dictionary states, IJsonSerializer serializer) + public Updates(Dictionary states, IJsonSerializer serializer) { - this.states = states; + currentState = states; + currentUpdates = []; this.serializer = serializer; } @@ -54,9 +55,9 @@ public sealed class TextIndexingProcess : IEventConsumer await textIndex.ExecuteAsync(commands.Values.ToArray()); } - if (updates.Count > 0) + if (currentUpdates.Count > 0) { - await textIndexerState.SetAsync(updates.Values.ToList()); + await textIndexerState.SetAsync(currentUpdates.Values.ToList()); } } @@ -98,211 +99,228 @@ public sealed class TextIndexingProcess : IEventConsumer private void Create(ContentEvent @event, ContentData data) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); var state = new TextContentState { - AppId = @event.AppId.Id, UniqueContentId = uniqueId }; - state.GenerateDocIdCurrent(); - Index(@event, new UpsertIndexEntry { - ContentId = @event.ContentId, - DocId = state.DocIdCurrent, + UniqueContentId = uniqueId, GeoObjects = data.ToGeo(serializer), + IsNew = true, + Stage = 0, ServeAll = true, ServePublished = false, Texts = data.ToTexts(), - IsNew = true }); - states[state.UniqueContentId] = state; - - updates[state.UniqueContentId] = state; + currentState[state.UniqueContentId] = state; + currentUpdates[state.UniqueContentId] = state; } private void CreateDraft(ContentEvent @event) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state)) + if (currentState.TryGetValue(uniqueId, out var state)) { - state.GenerateDocIdNew(); + switch (state.State) + { + case TextState.Stage0_Published__Stage1_None: + state.State = TextState.Stage0_Published__Stage1_Draft; + break; + case TextState.Stage1_Published__Stage0_None: + state.State = TextState.Stage1_Published__Stage0_Draft; + break; + } - updates[state.UniqueContentId] = state; + currentUpdates[state.UniqueContentId] = state; } } private void Unpublish(ContentEvent @event) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state) && state.DocIdForPublished != null) + if (currentState.TryGetValue(uniqueId, out var state)) { - Index(@event, - new UpdateIndexEntry - { - DocId = state.DocIdForPublished, - ServeAll = true, - ServePublished = false - }); + switch (state.State) + { + case TextState.Stage0_Published__Stage1_None: + CoreUpdate(@event, uniqueId, 0, true, false); - state.DocIdForPublished = null; + state.State = TextState.Stage0_Draft__Stage1_None; + break; + case TextState.Stage1_Published__Stage0_None: + CoreUpdate(@event, uniqueId, 1, true, false); - updates[state.UniqueContentId] = state; + state.State = TextState.Stage1_Draft__Stage0_None; + break; + } + + currentUpdates[state.UniqueContentId] = state; } } private void Update(ContentEvent @event, ContentData data) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state)) + if (currentState.TryGetValue(uniqueId, out var state)) { - if (state.DocIdNew != null) + switch (state.State) { - Index(@event, - new UpsertIndexEntry - { - ContentId = @event.ContentId, - DocId = state.DocIdNew, - GeoObjects = data.ToGeo(serializer), - ServeAll = true, - ServePublished = false, - Texts = data.ToTexts() - }); - - Index(@event, - new UpdateIndexEntry - { - DocId = state.DocIdCurrent, - ServeAll = false, - ServePublished = true - }); + case TextState.Stage0_Draft__Stage1_None: + CoreUpsert(@event, uniqueId, 0, true, false, data); + break; + case TextState.Stage0_Published__Stage1_None: + CoreUpsert(@event, uniqueId, 0, true, true, data); + break; + case TextState.Stage0_Published__Stage1_Draft: + CoreUpsert(@event, uniqueId, 1, true, false, data); + CoreUpdate(@event, uniqueId, 0, false, true); + break; + case TextState.Stage1_Draft__Stage0_None: + CoreUpsert(@event, uniqueId, 1, true, false, data); + break; + case TextState.Stage1_Published__Stage0_None: + CoreUpsert(@event, uniqueId, 1, true, true, data); + break; + case TextState.Stage1_Published__Stage0_Draft: + CoreUpsert(@event, uniqueId, 0, true, false, data); + CoreUpdate(@event, uniqueId, 1, false, true); + break; } - else - { - var isPublished = state.DocIdCurrent == state.DocIdForPublished; - Index(@event, - new UpsertIndexEntry - { - ContentId = @event.ContentId, - DocId = state.DocIdCurrent, - GeoObjects = data.ToGeo(serializer), - ServeAll = true, - ServePublished = isPublished, - Texts = data.ToTexts() - }); - } + currentUpdates[state.UniqueContentId] = state; } } private void Publish(ContentEvent @event) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state)) + if (currentState.TryGetValue(uniqueId, out var state)) { - if (state.DocIdNew != null) + switch (state.State) { - Index(@event, - new UpdateIndexEntry - { - DocId = state.DocIdNew, - ServeAll = true, - ServePublished = true - }); - - Index(@event, - new DeleteIndexEntry - { - DocId = state.DocIdCurrent - }); - - state.DocIdForPublished = state.DocIdNew; - state.DocIdCurrent = state.DocIdNew; + case TextState.Stage0_Published__Stage1_Draft: + CoreUpdate(@event, uniqueId, 1, true, true); + CoreDelete(@event, uniqueId, 0); + + state.State = TextState.Stage1_Published__Stage0_None; + break; + case TextState.Stage1_Published__Stage0_Draft: + CoreUpdate(@event, uniqueId, 0, true, true); + CoreDelete(@event, uniqueId, 1); + + state.State = TextState.Stage0_Published__Stage1_None; + break; + case TextState.Stage0_Draft__Stage1_None: + CoreUpdate(@event, uniqueId, 0, true, true); + + state.State = TextState.Stage0_Published__Stage1_None; + break; + case TextState.Stage1_Draft__Stage0_None: + CoreUpdate(@event, uniqueId, 1, true, true); + + state.State = TextState.Stage1_Published__Stage0_None; + break; } - else - { - Index(@event, - new UpdateIndexEntry - { - DocId = state.DocIdCurrent, - ServeAll = true, - ServePublished = true - }); - state.DocIdForPublished = state.DocIdCurrent; - } - - state.DocIdNew = null; - - updates[state.UniqueContentId] = state; + currentUpdates[state.UniqueContentId] = state; } } private void DeleteDraft(ContentEvent @event) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state) && state.DocIdNew != null) + if (currentState.TryGetValue(uniqueId, out var state)) { - Index(@event, - new UpdateIndexEntry - { - DocId = state.DocIdCurrent, - ServeAll = true, - ServePublished = true - }); - - Index(@event, - new DeleteIndexEntry - { - DocId = state.DocIdNew - }); - - state.DocIdNew = null; + switch (state.State) + { + case TextState.Stage0_Published__Stage1_Draft: + CoreUpdate(@event, uniqueId, 0, true, true); + CoreDelete(@event, uniqueId, 1); + + state.State = TextState.Stage0_Published__Stage1_None; + break; + case TextState.Stage1_Published__Stage0_Draft: + CoreUpdate(@event, uniqueId, 1, true, true); + CoreDelete(@event, uniqueId, 0); + + state.State = TextState.Stage1_Published__Stage0_None; + break; + } - updates[state.UniqueContentId] = state; + currentUpdates[state.UniqueContentId] = state; } } private void Delete(ContentEvent @event) { - var uniqueId = DomainId.Combine(@event.AppId, @event.ContentId); + var uniqueId = new UniqueContentId(@event.AppId.Id, @event.ContentId); - if (states.TryGetValue(uniqueId, out var state)) + if (currentState.TryGetValue(uniqueId, out var state)) { - Index(@event, - new DeleteIndexEntry - { - DocId = state.DocIdCurrent - }); + CoreDelete(@event, uniqueId, 0); + CoreDelete(@event, uniqueId, 1); - Index(@event, - new DeleteIndexEntry - { - DocId = state.DocIdNew ?? NotFound - }); + state.State = TextState.Deleted; - state.IsDeleted = true; - - updates[state.UniqueContentId] = state; + currentUpdates[state.UniqueContentId] = state; } } + private void CoreUpsert(ContentEvent @event, UniqueContentId uniqueId, byte stage, bool all, bool published, ContentData data) + { + Index(@event, + new UpsertIndexEntry + { + UniqueContentId = uniqueId, + GeoObjects = data.ToGeo(serializer), + Stage = stage, + ServeAll = all, + ServePublished = published, + Texts = data.ToTexts() + }); + } + + private void CoreUpdate(ContentEvent @event, UniqueContentId uniqueId, byte stage, bool all, bool published) + { + Index(@event, + new UpdateIndexEntry + { + UniqueContentId = uniqueId, + Stage = stage, + ServeAll = all, + ServePublished = published, + }); + } + + private void CoreDelete(ContentEvent @event, UniqueContentId uniqueId, byte stage) + { + Index(@event, + new DeleteIndexEntry + { + UniqueContentId = uniqueId, + Stage = stage, + }); + } + private void Index(ContentEvent @event, IndexCommand command) { - command.AppId = @event.AppId; command.SchemaId = @event.SchemaId; + var key = (command.UniqueContentId, command.Stage); + if (command is UpdateIndexEntry update && - commands.TryGetValue(command.DocId, out var existing) && + commands.TryGetValue(key, out var existing) && existing is UpsertIndexEntry upsert) { upsert.ServeAll = update.ServeAll; @@ -310,7 +328,7 @@ public sealed class TextIndexingProcess : IEventConsumer } else { - commands[command.DocId] = command; + commands[key] = command; } } } @@ -345,12 +363,12 @@ public sealed class TextIndexingProcess : IEventConsumer await updates.WriteAsync(textIndex, textIndexerState); } - private Task> QueryStatesAsync(IEnumerable> events) + private Task> QueryStatesAsync(IEnumerable> events) { var ids = events .Select(x => x.Payload).OfType() - .Select(x => DomainId.Combine(x.AppId, x.ContentId)) + .Select(x => new UniqueContentId(x.AppId.Id, x.ContentId)) .ToHashSet(); return textIndexerState.GetAsync(ids); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UniqueContentId.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UniqueContentId.cs new file mode 100644 index 000000000..dd6b0a43f --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UniqueContentId.cs @@ -0,0 +1,16 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Infrastructure; + +#pragma warning disable SA1313 // Parameter names should begin with lower-case letter + +namespace Squidex.Domain.Apps.Entities.Contents.Text; + +public record struct UniqueContentId(DomainId AppId, DomainId ContentId) +{ +} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UpsertIndexEntry.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UpsertIndexEntry.cs index aeb499669..495541123 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UpsertIndexEntry.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/UpsertIndexEntry.cs @@ -6,7 +6,6 @@ // ========================================================================== using NetTopologySuite.Geometries; -using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Contents.Text; @@ -20,7 +19,5 @@ public sealed class UpsertIndexEntry : IndexCommand public bool ServePublished { get; set; } - public DomainId ContentId { get; set; } - public bool IsNew { get; set; } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj index 82513ac9e..68894b28e 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj +++ b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj @@ -10,6 +10,11 @@ full True + + + + + diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDomainIdSerializer.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDomainIdSerializer.cs index b3edb9561..fc960b94e 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDomainIdSerializer.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/BsonDomainIdSerializer.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Text; using MongoDB.Bson; using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Serializers; @@ -13,7 +14,7 @@ namespace Squidex.Infrastructure.MongoDb; public sealed class BsonDomainIdSerializer : SerializerBase, IBsonPolymorphicSerializer, IRepresentationConfigurable { - private static readonly BsonDomainIdSerializer Instance = new BsonDomainIdSerializer(); + private static readonly BsonDomainIdSerializer Instance = new BsonDomainIdSerializer(BsonType.String); public static void Register() { @@ -29,7 +30,12 @@ public sealed class BsonDomainIdSerializer : SerializerBase, IBsonPoly get => true; } - public BsonType Representation { get; } = BsonType.String; + public BsonType Representation { get; } + + public BsonDomainIdSerializer(BsonType representation) + { + Representation = representation; + } public override DomainId Deserialize(BsonDeserializationContext context, BsonDeserializationArgs args) { @@ -45,7 +51,7 @@ public sealed class BsonDomainIdSerializer : SerializerBase, IBsonPoly return DomainId.Create(binary.ToGuid()); } - return DomainId.Create(binary.ToString()); + return DomainId.Create(Encoding.UTF8.GetString(binary.Bytes)); default: ThrowHelper.NotSupportedException(); return default!; @@ -54,17 +60,52 @@ public sealed class BsonDomainIdSerializer : SerializerBase, IBsonPoly public override void Serialize(BsonSerializationContext context, BsonSerializationArgs args, DomainId value) { - context.Writer.WriteString(value.ToString()); + switch (Representation) + { + case BsonType.String: + context.Writer.WriteString(value.ToString()); + break; + case BsonType.Binary: + if (Guid.TryParse(value.ToString(), out var guid)) + { +#pragma warning disable CS0618 // Type or member is obsolete + if (context.Writer.Settings.GuidRepresentation == GuidRepresentation.CSharpLegacy) + { + context.Writer.WriteBinaryData(new BsonBinaryData(guid.ToByteArray(), BsonBinarySubType.UuidLegacy, GuidRepresentation.CSharpLegacy)); + } + else + { + context.Writer.WriteBinaryData(new BsonBinaryData(guid, GuidRepresentation.Standard)); + } +#pragma warning restore CS0618 // Type or member is obsolete + } + else + { + var buffer = Encoding.UTF8.GetBytes(value.ToString()); + + context.Writer.WriteBytes(buffer); + } + + break; + default: + ThrowHelper.NotSupportedException(); + break; + } } public BsonDomainIdSerializer WithRepresentation(BsonType representation) { - if (representation != BsonType.String) + if (representation is not BsonType.String and not BsonType.Binary) { ThrowHelper.NotSupportedException(); return default!; } + if (representation != Representation) + { + return new BsonDomainIdSerializer(representation); + } + return this; } diff --git a/backend/src/Squidex.Infrastructure/DomainId.cs b/backend/src/Squidex.Infrastructure/DomainId.cs index 4c04eea93..d9746aa58 100644 --- a/backend/src/Squidex.Infrastructure/DomainId.cs +++ b/backend/src/Squidex.Infrastructure/DomainId.cs @@ -13,6 +13,7 @@ namespace Squidex.Infrastructure; public readonly struct DomainId : IEquatable, IComparable { private static readonly string EmptyString = Guid.Empty.ToString(); + public static readonly DomainId Empty = default; public static readonly string IdSeparator = "--"; diff --git a/backend/src/Squidex.Infrastructure/States/ShardedService.cs b/backend/src/Squidex.Infrastructure/States/ShardedService.cs new file mode 100644 index 000000000..cdb0814a5 --- /dev/null +++ b/backend/src/Squidex.Infrastructure/States/ShardedService.cs @@ -0,0 +1,68 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Hosting; + +namespace Squidex.Infrastructure.States; + +public abstract class ShardedService : IInitializable +{ + private readonly Dictionary shards = new Dictionary(); + private readonly IShardingStrategy sharding; + private readonly Func factory; + + protected IEnumerable Shards => shards.Values; + + protected ShardedService(IShardingStrategy sharding, Func factory) + { + this.sharding = sharding; + this.factory = factory; + } + + public async Task InitializeAsync( + CancellationToken ct) + { + foreach (var shardKey in sharding.GetShardKeys()) + { + var inner = factory(shardKey); + + if (inner is IInitializable initializable) + { + await initializable.InitializeAsync(ct); + } + + shards[shardKey] = inner; + } + } + + public async Task ReleaseAsync( + CancellationToken ct) + { + foreach (var inner in shards.Values) + { + if (inner is IInitializable initializable) + { + await initializable.ReleaseAsync(ct); + } + } + } + + protected string GetShardKey(TKey key) where TKey : notnull + { + return sharding.GetShardKey(key); + } + + protected T Shard(TKey key) where TKey : notnull + { + return shards[GetShardKey(key)]; + } + + protected string GetShardKey(DomainId appId) + { + return sharding.GetShardKey(appId); + } +} diff --git a/backend/src/Squidex.Infrastructure/States/Sharding.cs b/backend/src/Squidex.Infrastructure/States/Sharding.cs new file mode 100644 index 000000000..028e63394 --- /dev/null +++ b/backend/src/Squidex.Infrastructure/States/Sharding.cs @@ -0,0 +1,66 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +#pragma warning disable MA0048 // File name must match type name + +namespace Squidex.Infrastructure.States; + +public interface IShardingStrategy +{ + string GetShardKey(T key) where T : notnull; + + IEnumerable GetShardKeys(); +} + +public sealed class SingleSharding : IShardingStrategy +{ + public static readonly IShardingStrategy Instance = new SingleSharding(); + + private SingleSharding() + { + } + + public string GetShardKey(T key) where T : notnull + { + return string.Empty; + } + + public IEnumerable GetShardKeys() + { + yield return string.Empty; + } +} + +public sealed class PartitionedSharding : IShardingStrategy +{ + private readonly int numPartitions; + + public PartitionedSharding(int numPartitions) + { + this.numPartitions = numPartitions; + } + + public string GetShardKey(T key) where T : notnull + { + var partition = Math.Abs(key.GetHashCode()) % numPartitions; + + return GetShardKey(partition); + } + + public IEnumerable GetShardKeys() + { + for (var i = 0; i < numPartitions; i++) + { + yield return GetShardKey(i); + } + } + + private static string GetShardKey(int partition) + { + return $"_{partition}"; + } +} diff --git a/backend/src/Squidex/Config/Domain/StoreServices.cs b/backend/src/Squidex/Config/Domain/StoreServices.cs index b2f6a7987..de5ebfb43 100644 --- a/backend/src/Squidex/Config/Domain/StoreServices.cs +++ b/backend/src/Squidex/Config/Domain/StoreServices.cs @@ -99,9 +99,6 @@ public static class StoreServices services.AddTransientAs(c => new ConvertDocumentIds(GetDatabase(c, mongoDatabaseName), GetDatabase(c, mongoContentDatabaseName))) .As(); - services.AddSingletonAs(c => ActivatorUtilities.CreateInstance(c, GetDatabase(c, mongoContentDatabaseName))) - .As().As>().As(); - services.AddTransientAs() .As(); @@ -138,9 +135,6 @@ public static class StoreServices services.AddSingletonAs() .As>().As(); - services.AddSingletonAs() - .As().As>().As(); - services.AddSingletonAs() .As().As>().As(); @@ -162,6 +156,20 @@ public static class StoreServices services.AddSingletonAs() .As().As(); + services.AddSingletonAs(c => + { + return new MongoShardedAssetRepository(GetSharding(config, "store:mongoDB:assetShardCount"), + shardKey => ActivatorUtilities.CreateInstance(c, shardKey)); + }).As().As>().As(); + + services.AddSingletonAs(c => + { + var contentDatabase = GetDatabase(c, mongoContentDatabaseName); + + return new MongoShardedContentRepository(GetSharding(config, "store:mongoDB:contentShardCount"), + shardKey => ActivatorUtilities.CreateInstance(c, shardKey, contentDatabase)); + }).As().As>().As(); + services.AddOpenIddict() .AddCore(builder => { @@ -191,13 +199,19 @@ public static class StoreServices }; }); - services.AddSingletonAs() - .AsOptional().As(); + services.AddSingletonAs(c => + { + return new MongoShardedTextIndex>(GetSharding(config, "store:mongoDB:textShardCount"), + shardKey => ActivatorUtilities.CreateInstance(c, shardKey)); + }).AsOptional().As(); } else { - services.AddSingletonAs() - .AsOptional().As(); + services.AddSingletonAs(c => + { + return new MongoShardedTextIndex>(GetSharding(config, "store:mongoDB:textShardCount"), + shardKey => ActivatorUtilities.CreateInstance(c, shardKey)); + }).AsOptional().As(); } services.AddInitializer("Serializer (BSON)", jsonSerializerOptions => @@ -230,6 +244,13 @@ public static class StoreServices }); } + private static IShardingStrategy GetSharding(IConfiguration config, string name) + { + var shardCount = config.GetValue(name); + + return shardCount > 0 && shardCount <= 100 ? new PartitionedSharding(shardCount) : SingleSharding.Instance; + } + private static IMongoDatabase GetDatabase(IServiceProvider serviceProvider, string name) { return serviceProvider.GetRequiredService().GetDatabase(name); diff --git a/backend/src/Squidex/appsettings.json b/backend/src/Squidex/appsettings.json index 349e0b2c9..77b8451cf 100644 --- a/backend/src/Squidex/appsettings.json +++ b/backend/src/Squidex/appsettings.json @@ -530,6 +530,15 @@ // The database for all your content collections (one collection per app). "contentDatabase": "SquidexContent", + // Defines the number of collections for contents. + "contentShardCount": 0, + + // Defines the number of collections for assets. + "assetShardCount": 0, + + // Defines the number of collections for texts. + "textShardCount": 0, + // The database for all your other read collections. "database": "Squidex", diff --git a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs index 5e8e2de5a..b30e08086 100644 --- a/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/HandleRules/RuleServiceTests.cs @@ -31,7 +31,6 @@ public class RuleServiceTests private readonly string actionDump = "MyDump"; private readonly string actionName = "ValidAction"; private readonly string actionDescription = "MyDescription"; - private readonly DomainId ruleId = DomainId.NewGuid(); private readonly NamedId appId = NamedId.Of(DomainId.NewGuid(), "my-app"); private readonly TypeRegistry typeRegistry = new TypeRegistry(); private readonly RuleService sut; diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentSchedulerProcessTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentSchedulerProcessTests.cs index 5f348db7a..85edecee2 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentSchedulerProcessTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/ContentSchedulerProcessTests.cs @@ -114,7 +114,7 @@ public class ContentSchedulerProcessTests : GivenContext await sut.PublishAsync(CancellationToken); - A.CallTo(() => contentRepository.ResetScheduledAsync(content1.UniqueId, SearchScope.All, default)) + A.CallTo(() => contentRepository.ResetScheduledAsync(content1.AppId.Id, content1.Id, SearchScope.All, default)) .MustHaveHappened(); } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs index 7da20e7e8..11b8f9aa2 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/MongoDb/TokenizerTests.cs @@ -16,7 +16,7 @@ public class TokenizerTests { var source = "The only thing that matters, is time."; - var parsed = Tokenizer.TokenizeQuery(source); + var parsed = Tokenizer.Query(source); Assert.Equal("only thing matters time", parsed); } @@ -26,7 +26,7 @@ public class TokenizerTests { var source = "en:The only thing that matters, is time."; - var parsed = Tokenizer.TokenizeQuery(source); + var parsed = Tokenizer.Query(source); Assert.Equal("only thing matters time", parsed); } @@ -36,7 +36,7 @@ public class TokenizerTests { var source = "en:when i do this it is pretty slow"; - var parsed = Tokenizer.TokenizeQuery(source); + var parsed = Tokenizer.Query(source); Assert.Equal("when i do pretty slow", parsed); } @@ -46,7 +46,7 @@ public class TokenizerTests { var source = "iv:The only thing that matters, is time."; - var parsed = Tokenizer.TokenizeQuery(source); + var parsed = Tokenizer.Query(source); Assert.Equal("the only thing that matters is time", parsed); } @@ -56,7 +56,7 @@ public class TokenizerTests { var source = "de:Nur die Zeit spielt eine Rolle"; - var parsed = Tokenizer.TokenizeQuery(source); + var parsed = Tokenizer.Query(source); Assert.Equal("zeit spielt rolle", parsed); } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/BsonUniqueContentIdSerializerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/BsonUniqueContentIdSerializerTests.cs new file mode 100644 index 000000000..0a6bb118d --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/BsonUniqueContentIdSerializerTests.cs @@ -0,0 +1,90 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Domain.Apps.Core.TestHelpers; +using Squidex.Domain.Apps.Entities.MongoDb; +using Squidex.Infrastructure; + +namespace Squidex.Domain.Apps.Entities.Contents.Text; + +public class BsonUniqueContentIdSerializerTests +{ + public BsonUniqueContentIdSerializerTests() + { + BsonUniqueContentIdSerializer.Register(); + } + + [Fact] + public void Should_serialize_and_deserialize_guid_guid() + { + var source = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid()); + + var deserialized = source.SerializeAndDeserializeBson(); + + Assert.Equal(source, deserialized); + } + + [Fact] + public void Should_serialize_and_deserialize_guid_custom() + { + var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Create("id42")); + + var deserialized = source.SerializeAndDeserializeBson(); + + Assert.Equal(source, deserialized); + } + + [Fact] + public void Should_serialize_and_deserialize_guid_empty() + { + var source = new UniqueContentId(DomainId.NewGuid(), DomainId.Empty); + + var deserialized = source.SerializeAndDeserializeBson(); + + Assert.Equal(source, deserialized); + } + + [Fact] + public void Should_serialize_and_deserialize_custom_custom() + { + var source = new UniqueContentId(DomainId.Create("id41"), DomainId.Create("id42")); + + var deserialized = source.SerializeAndDeserializeBson(); + + Assert.Equal(source, deserialized); + } + + [Fact] + public void Should_serialize_and_deserialize_custom_guid() + { + var source = new UniqueContentId(DomainId.Create("id42"), DomainId.NewGuid()); + + var deserialized = source.SerializeAndDeserializeBson(); + + Assert.Equal(source, deserialized); + } + + [Fact] + public void Should_calculate_next_custom_id() + { + var appId = DomainId.Create("x"); + + var actual = BsonUniqueContentIdSerializer.NextAppId(appId); + + Assert.Equal(new UniqueContentId(DomainId.Create("y"), DomainId.Empty), actual); + } + + [Fact] + public void Should_calculate_next_guid_id() + { + var appId = DomainId.Create("70fb3772-2ab5-4854-b421-054d2479a0f7"); + + var actual = BsonUniqueContentIdSerializer.NextAppId(appId); + + Assert.Equal(new UniqueContentId(DomainId.Create("70fb3773-2ab5-4854-b421-054d2479a0f7"), DomainId.Empty), actual); + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs index 4ec240f59..f18780ba7 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs @@ -15,11 +15,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text; public class CachingTextIndexerStateTests : GivenContext { private readonly ITextIndexerState inner = A.Fake(); - private readonly DomainId contentId = DomainId.NewGuid(); + private readonly UniqueContentId contentId; private readonly CachingTextIndexerState sut; public CachingTextIndexerStateTests() { + contentId = new UniqueContentId(AppId.Id, DomainId.NewGuid()); + sut = new CachingTextIndexerState(inner); } @@ -30,12 +32,12 @@ public class CachingTextIndexerStateTests : GivenContext var state = new TextContentState { UniqueContentId = contentId }; - var states = new Dictionary + var states = new Dictionary { [contentId] = state }; - A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) .Returns(states); var found1 = await sut.GetAsync(HashSet.Of(contentId), CancellationToken); @@ -44,7 +46,7 @@ public class CachingTextIndexerStateTests : GivenContext Assert.Same(state, found1[contentId]); Assert.Same(state, found2[contentId]); - A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) .MustHaveHappenedOnceExactly(); } @@ -53,7 +55,7 @@ public class CachingTextIndexerStateTests : GivenContext { var contentIds = HashSet.Of(contentId); - A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) .Returns([]); var found1 = await sut.GetAsync(HashSet.Of(contentId), CancellationToken); @@ -62,7 +64,7 @@ public class CachingTextIndexerStateTests : GivenContext Assert.Empty(found1); Assert.Empty(found2); - A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds), CancellationToken)) .MustHaveHappenedOnceExactly(); } @@ -84,7 +86,7 @@ public class CachingTextIndexerStateTests : GivenContext A.CallTo(() => inner.SetAsync(A>.That.IsSameSequenceAs(state), CancellationToken)) .MustHaveHappenedOnceExactly(); - A.CallTo(() => inner.GetAsync(A>._, A._)) + A.CallTo(() => inner.GetAsync(A>._, A._)) .MustNotHaveHappened(); } @@ -102,7 +104,7 @@ public class CachingTextIndexerStateTests : GivenContext await sut.SetAsync( [ - new TextContentState { UniqueContentId = contentId, IsDeleted = true } + new TextContentState { UniqueContentId = contentId, State = TextState.Deleted } ], CancellationToken); var found1 = await sut.GetAsync(contentIds, CancellationToken); @@ -111,10 +113,10 @@ public class CachingTextIndexerStateTests : GivenContext Assert.Empty(found1); Assert.Empty(found2); - A.CallTo(() => inner.SetAsync(A>.That.Matches(x => x.Count == 1 && x[0].IsDeleted), CancellationToken)) + A.CallTo(() => inner.SetAsync(A>.That.Matches(x => x.Count == 1 && x[0].State == TextState.Deleted), CancellationToken)) .MustHaveHappenedOnceExactly(); - A.CallTo(() => inner.GetAsync(A>._, A._)) + A.CallTo(() => inner.GetAsync(A>._, A._)) .MustNotHaveHappened(); } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs index 8ea0db9fd..62eaa1df3 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexFixture.cs @@ -23,7 +23,7 @@ public sealed class MongoTextIndexFixture : IAsyncLifetime var mongoClient = MongoClientFactory.Create(TestConfig.Configuration["mongoDb:configuration"]!); var mongoDatabase = mongoClient.GetDatabase(TestConfig.Configuration["mongodb:database"]!); - Index = new MongoTextIndex(mongoDatabase); + Index = new MongoTextIndex(mongoDatabase, string.Empty); } public Task InitializeAsync() diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs index fec0f883e..9a6864941 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/MongoTextIndexTests.cs @@ -31,22 +31,18 @@ public class MongoTextIndexTests : TextIndexerTestsBase, IClassFixture +{ + public MongoTextIndexerStateFixture _ { get; set; } + + public MongoTextIndexerStateTests(MongoTextIndexerStateFixture fixture) + { + _ = fixture; + } + + [Fact] + public async Task Should_add_state() + { + var appId = DomainId.NewGuid(); + var id1 = new UniqueContentId(appId, DomainId.NewGuid()); + var id2 = new UniqueContentId(appId, DomainId.NewGuid()); + var id3 = new UniqueContentId(appId, DomainId.NewGuid()); + + await _.State.SetAsync( + [ + new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, + new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft }, + new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } + ]); + + var actual = await _.State.GetAsync(HashSet.Of(id1, id2)); + + actual.Should().BeEquivalentTo(new Dictionary + { + [id1] = new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, + [id2] = new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft } + }); + } + + [Fact] + public async Task Should_remove_state() + { + var id = new UniqueContentId(DomainId.NewGuid(), DomainId.NewGuid()); + + await _.State.SetAsync( + [ + new TextContentState { UniqueContentId = id, State = TextState.Stage0_Draft__Stage1_None } + ]); + + await _.State.SetAsync( + [ + new TextContentState { UniqueContentId = id, State = TextState.Deleted } + ]); + + var actual = await _.State.GetAsync(HashSet.Of(id)); + + Assert.Empty(actual); + } + + [Fact] + public async Task Should_remove_by_app_state() + { + var appId1 = DomainId.NewGuid(); + var appId2 = DomainId.NewGuid(); + var id1 = new UniqueContentId(appId1, DomainId.NewGuid()); + var id2 = new UniqueContentId(appId1, DomainId.NewGuid()); + var id3 = new UniqueContentId(appId2, DomainId.NewGuid()); + + await _.State.SetAsync( + [ + new TextContentState { UniqueContentId = id1, State = TextState.Stage0_Draft__Stage1_None }, + new TextContentState { UniqueContentId = id2, State = TextState.Stage0_Published__Stage1_Draft }, + new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } + ]); + + await ((IDeleter)_.State).DeleteAppAsync(new App { Id = appId1 }, default); + + var actual = await _.State.GetAsync(HashSet.Of(id1, id2, id3)); + + actual.Should().BeEquivalentTo(new Dictionary + { + [id3] = new TextContentState { UniqueContentId = id3, State = TextState.Stage0_Published__Stage1_None } + }); + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs index 1d70dd0b0..f27b2ac8e 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs @@ -127,6 +127,14 @@ public abstract class TextIndexerTestsBase : GivenContext await SearchGeo(expected: null, "other.iv", 51.48596429889613, 12.102629469505713); } + [Fact] + public async Task Should_search_by_app() + { + await CreateTextAsync(ids1[0], "iv", "Hello"); + + await SearchByAppText(expected: ids1, text: "helo~"); + } + [Fact] public async Task Should_index_invariant_content_and_retrieve() { @@ -183,10 +191,13 @@ public abstract class TextIndexerTestsBase : GivenContext [Fact] public async Task Should_also_update_published_content() { + // Create initial content. await CreateTextAsync(ids1[0], "iv", "V1"); + // Publish the content. await PublishAsync(ids1[0]); + // Update the published content once. await UpdateTextAsync(ids1[0], "iv", "V2"); await SearchText(expected: null, text: "V1", target: SearchScope.All); @@ -199,10 +210,13 @@ public abstract class TextIndexerTestsBase : GivenContext [Fact] public async Task Should_also_update_published_content_multiple_times() { + // Create initial content. await CreateTextAsync(ids1[0], "iv", "V1"); + // Publish the content. await PublishAsync(ids1[0]); + // Update the published content twice. await UpdateTextAsync(ids1[0], "iv", "V2"); await UpdateTextAsync(ids1[0], "iv", "V3"); @@ -216,6 +230,7 @@ public abstract class TextIndexerTestsBase : GivenContext [Fact] public async Task Should_simulate_new_version() { + // Create initial content. await CreateTextAsync(ids1[0], "iv", "V1"); // Publish the content. @@ -467,4 +482,23 @@ public abstract class TextIndexerTestsBase : GivenContext actual.Should().BeEmpty(); } } + + protected async Task SearchByAppText(List? expected, string text, SearchScope target = SearchScope.All) + { + var query = new TextQuery(text, 1000) + { + PreferredSchemaId = Schema.Id + }; + + var actual = await Sut.TextIndex.SearchAsync(App, query, target, default); + + if (expected != null) + { + actual.Should().BeEquivalentTo(expected.ToHashSet()); + } + else + { + actual.Should().BeEmpty(); + } + } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/DomainIdSerializerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/DomainIdSerializerTests.cs index 903216b52..ae5646675 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/DomainIdSerializerTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/DomainIdSerializerTests.cs @@ -6,8 +6,6 @@ // ========================================================================== using MongoDB.Bson; -using MongoDB.Bson.IO; -using MongoDB.Bson.Serialization; using MongoDB.Bson.Serialization.Attributes; using Squidex.Infrastructure.TestHelpers; @@ -21,6 +19,12 @@ public class DomainIdSerializerTests public T Id { get; set; } } + private sealed class BinaryEntity + { + [BsonRepresentation(BsonType.Binary)] + public T Id { get; set; } + } + private sealed class IdEntity { public T Id { get; set; } @@ -34,57 +38,50 @@ public class DomainIdSerializerTests [Fact] public void Should_deserialize_from_string() { - var id = Guid.NewGuid(); - - var source = new IdEntity { Id = id.ToString() }; + var source = new IdEntity { Id = Guid.NewGuid().ToString() }; - var actual = SerializeAndDeserializeBson, IdEntity>(source); + var actual = TestUtils.SerializeAndDeserializeBson, IdEntity>(source); - Assert.Equal(actual.Id.ToString(), id.ToString()); + Assert.Equal(actual.Id.ToString(), source.Id); } [Fact] public void Should_deserialize_from_guid_string() { - var id = Guid.NewGuid(); - - var source = new StringEntity { Id = id }; + var source = new StringEntity { Id = Guid.NewGuid() }; - var actual = SerializeAndDeserializeBson, IdEntity>(source); + var actual = TestUtils.SerializeAndDeserializeBson, StringEntity>(source); - Assert.Equal(actual.Id.ToString(), id.ToString()); + Assert.Equal(actual.Id.ToString(), source.Id.ToString()); } [Fact] public void Should_deserialize_from_guid_bytes() { - var id = Guid.NewGuid(); + var source = new IdEntity { Id = Guid.NewGuid() }; - var source = new IdEntity { Id = id }; + var actual = TestUtils.SerializeAndDeserializeBson, IdEntity>(source); - var actual = SerializeAndDeserializeBson, IdEntity>(source); - - Assert.Equal(actual.Id.ToString(), id.ToString()); + Assert.Equal(actual.Id.ToString(), source.Id.ToString()); } - private static TOut SerializeAndDeserializeBson(TIn source) + [Fact] + public void Should_serialize_guid_as_bytes() { - var stream = new MemoryStream(); + var source = new BinaryEntity { Id = DomainId.NewGuid() }; - using (var writer = new BsonBinaryWriter(stream)) - { - BsonSerializer.Serialize(writer, source); + var actual = TestUtils.SerializeAndDeserializeBson(source); - writer.Flush(); - } + Assert.Equal(actual.Id, source.Id); + } - stream.Position = 0; + [Fact] + public void Should_serialize_non_guid_as_bytes() + { + var source = new BinaryEntity { Id = DomainId.Create("NonGuid") }; - using (var reader = new BsonBinaryReader(stream)) - { - var target = BsonSerializer.Deserialize(reader); + var actual = TestUtils.SerializeAndDeserializeBson(source); - return target; - } + Assert.Equal(actual.Id, source.Id); } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/ShardedServiceTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/ShardedServiceTests.cs new file mode 100644 index 000000000..c265e24aa --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/States/ShardedServiceTests.cs @@ -0,0 +1,82 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using Squidex.Hosting; + +namespace Squidex.Infrastructure.States; + +public class ShardedServiceTests +{ + private readonly IInner inner1 = A.Fake(); + private readonly IInner inner2 = A.Fake(); + private readonly TestSut sut; + + public interface IInner : IInitializable + { + } + + private class TestSut : ShardedService + { + public TestSut(IShardingStrategy sharding, Func factory) + : base(sharding, factory) + { + } + + public IInner ExposeShard(TKey key) where TKey : notnull + { + return Shard(key); + } + } + + public ShardedServiceTests() + { + sut = new TestSut(new PartitionedSharding(2), key => + { + if (key == "_0") + { + return inner1; + } + else + { + return inner2; + } + }); + + sut.InitializeAsync(default).Wait(); + } + + [Fact] + public async Task Should_initialize_shards() + { + await sut.InitializeAsync(default); + + A.CallTo(() => inner1.InitializeAsync(A._)) + .MustHaveHappened(); + + A.CallTo(() => inner2.InitializeAsync(A._)) + .MustHaveHappened(); + } + + [Fact] + public async Task Should_release_shards() + { + await sut.ReleaseAsync(default); + + A.CallTo(() => inner1.ReleaseAsync(A._)) + .MustHaveHappened(); + + A.CallTo(() => inner2.ReleaseAsync(A._)) + .MustHaveHappened(); + } + + [Fact] + public void Should_provide_shards() + { + Assert.Equal(inner1, sut.ExposeShard(0)); + Assert.Equal(inner2, sut.ExposeShard(1)); + } +} diff --git a/backend/tests/Squidex.Infrastructure.Tests/States/ShardingTests.cs b/backend/tests/Squidex.Infrastructure.Tests/States/ShardingTests.cs new file mode 100644 index 000000000..810f6c05a --- /dev/null +++ b/backend/tests/Squidex.Infrastructure.Tests/States/ShardingTests.cs @@ -0,0 +1,57 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Infrastructure.States; + +public class ShardingTests +{ + [Fact] + public void Should_provide_single_shard_key() + { + var strategy = SingleSharding.Instance; + + for (var i = 0; i < 1000; i++) + { + var shardKey = strategy.GetShardKey(Guid.NewGuid()); + + Assert.Equal(string.Empty, shardKey); + } + } + + [Fact] + public void Should_provide_single_shard_keys() + { + var strategy = SingleSharding.Instance; + + var shardKeys = strategy.GetShardKeys().ToArray(); + + Assert.Equal(new[] { string.Empty }, shardKeys); + } + + [Fact] + public void Should_provide_partitioned_shard_key() + { + var strategy = new PartitionedSharding(3); + + for (var i = 0; i < 1000; i++) + { + var shardKey = strategy.GetShardKey(Guid.NewGuid()); + + Assert.True(shardKey is "_0" or "_1" or "_2"); + } + } + + [Fact] + public void Should_provide_partitioned_shard_keys() + { + var strategy = new PartitionedSharding(3); + + var shardKeys = strategy.GetShardKeys().ToArray(); + + Assert.Equal(new[] { "_0", "_1", "_2" }, shardKeys); + } +}