diff --git a/backend/src/Migrations/Migrations/ConvertEventStore.cs b/backend/src/Migrations/Migrations/ConvertEventStore.cs index 26b291617..58683e33c 100644 --- a/backend/src/Migrations/Migrations/ConvertEventStore.cs +++ b/backend/src/Migrations/Migrations/ConvertEventStore.cs @@ -32,20 +32,20 @@ namespace Migrations.Migrations var filter = Builders.Filter; - var writesBatches = new List>(); + var writes = new List>(); async Task WriteAsync(WriteModel? model, bool force) { if (model != null) { - writesBatches.Add(model); + writes.Add(model); } - if (writesBatches.Count == 1000 || (force && writesBatches.Count > 0)) + if (writes.Count == 1000 || (force && writes.Count > 0)) { - await collection.BulkWriteAsync(writesBatches); + await collection.BulkWriteAsync(writes); - writesBatches.Clear(); + writes.Clear(); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs index b4c671464..417eed915 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs @@ -55,7 +55,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { using (Profiler.TraceMethod()) { - await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(Map(x), x.Version), ct); + await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(Map(x), x.Version), ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs index ea27c74ab..7d481d855 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs @@ -55,7 +55,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets { using (Profiler.TraceMethod()) { - await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(Map(x), x.Version), ct); + await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(Map(x), x.Version), ct); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs deleted file mode 100644 index fca521983..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs +++ /dev/null @@ -1,124 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.IO; -using System.IO.Compression; -using System.Threading.Tasks; -using Lucene.Net.Index; -using Lucene.Net.Store; -using MongoDB.Driver.GridFS; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; -using Squidex.Infrastructure; -using LuceneDirectory = Lucene.Net.Store.Directory; - -namespace Squidex.Domain.Apps.Entities.MongoDb.FullText -{ - public sealed class MongoIndexStorage : IIndexStorage - { - private readonly IGridFSBucket bucket; - - public MongoIndexStorage(IGridFSBucket bucket) - { - Guard.NotNull(bucket, nameof(bucket)); - - this.bucket = bucket; - } - - public async Task CreateDirectoryAsync(Guid ownerId) - { - var fileId = $"index_{ownerId}"; - - var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), fileId)); - - if (directoryInfo.Exists) - { - directoryInfo.Delete(true); - } - - directoryInfo.Create(); - - try - { - using (var stream = await bucket.OpenDownloadStreamAsync(fileId)) - { - using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, true)) - { - foreach (var entry in zipArchive.Entries) - { - var file = new FileInfo(Path.Combine(directoryInfo.FullName, entry.Name)); - - using (var entryStream = entry.Open()) - { - using (var fileStream = file.OpenWrite()) - { - await entryStream.CopyToAsync(fileStream); - } - } - } - } - } - } - catch (GridFSFileNotFoundException) - { - } - - var directory = FSDirectory.Open(directoryInfo); - - return directory; - } - - public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) - { - var directoryInfo = ((FSDirectory)directory).Directory; - - var commit = snapshotter.Snapshot(); - try - { - var fileId = directoryInfo.Name; - - try - { - await bucket.DeleteAsync(fileId); - } - catch (GridFSFileNotFoundException) - { - } - - using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId)) - { - using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) - { - foreach (var fileName in commit.FileNames) - { - var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); - - using (var fileStream = file.OpenRead()) - { - var entry = zipArchive.CreateEntry(fileStream.Name); - - using (var entryStream = entry.Open()) - { - await fileStream.CopyToAsync(entryStream); - } - } - } - } - } - } - finally - { - snapshotter.Release(commit); - } - } - - public Task ClearAsync() - { - return bucket.DropAsync(); - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs new file mode 100644 index 000000000..b62c1d59e --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs @@ -0,0 +1,168 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using MongoDB.Driver; +using Squidex.Domain.Apps.Entities.Apps; +using Squidex.Domain.Apps.Entities.Contents; +using Squidex.Domain.Apps.Entities.Contents.Text; +using Squidex.Infrastructure.MongoDb; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +{ + public sealed class MongoTextIndex : MongoRepositoryBase, ITextIndex + { + private static readonly List EmptyResults = new List(); + + public MongoTextIndex(IMongoDatabase database, bool setup = false) + : base(database, setup) + { + } + + protected override Task SetupCollectionAsync(IMongoCollection collection, CancellationToken ct = default) + { + return collection.Indexes.CreateManyAsync(new[] + { + new CreateIndexModel( + Index + .Text("t.t") + .Ascending(x => x.AppId) + .Ascending(x => x.ServeAll) + .Ascending(x => x.ServePublished) + .Ascending(x => x.SchemaId)) + }, ct); + } + + protected override string CollectionName() + { + return "TextIndex"; + } + + public Task ExecuteAsync(params IndexCommand[] commands) + { + var writes = new List>(commands.Length); + + foreach (var command in commands) + { + switch (command) + { + case DeleteIndexEntry delete: + writes.Add( + new DeleteOneModel( + Filter.Eq(x => x.DocId, command.DocId))); + break; + case UpdateIndexEntry update: + writes.Add( + new UpdateOneModel( + Filter.Eq(x => x.DocId, command.DocId), + Update + .Set(x => x.ServeAll, update.ServeAll) + .Set(x => x.ServePublished, update.ServePublished))); + break; + case UpsertIndexEntry upsert when upsert.Texts.Count > 0: + writes.Add( + new ReplaceOneModel( + Filter.Eq(x => x.DocId, command.DocId), + new MongoTextIndexEntity + { + DocId = upsert.DocId, + ContentId = upsert.ContentId, + SchemaId = upsert.SchemaId.Id, + ServeAll = upsert.ServeAll, + ServePublished = upsert.ServePublished, + Texts = upsert.Texts.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList(), + AppId = upsert.AppId.Id + }) + { + IsUpsert = true + }); + break; + } + } + + if (writes.Count == 0) + { + return Task.CompletedTask; + } + + return Collection.BulkWriteAsync(writes); + } + + public async Task?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope) + { + if (string.IsNullOrWhiteSpace(queryText)) + { + return EmptyResults; + } + + if (filter == null) + { + return await SearchByAppAsync(queryText, app, scope, 2000); + } + else if (filter.Must) + { + return await SearchBySchemaAsync(queryText, app, filter, scope, 2000); + } + else + { + var (bySchema, byApp) = + await AsyncHelper.WhenAll( + SearchBySchemaAsync(queryText, app, filter, scope, 1000), + SearchByAppAsync(queryText, app, scope, 1000)); + + return bySchema.Union(byApp).Distinct().ToList(); + } + } + + private async Task> SearchBySchemaAsync(string queryText, IAppEntity app, SearchFilter filter, SearchScope scope, int limit) + { + var bySchema = + await Collection.Find( + Filter.And( + Filter.Eq(x => x.AppId, app.Id), + Filter.In(x => x.SchemaId, filter.SchemaIds), + Filter_ByScope(scope), + Filter.Text(queryText))) + .Only(x => x.ContentId).Limit(limit) + .ToListAsync(); + + return bySchema.Select(x => Guid.Parse(x["_ci"].AsString)).Distinct().ToList(); + } + + private async Task> SearchByAppAsync(string queryText, IAppEntity app, SearchScope scope, int limit) + { + var bySchema = + await Collection.Find( + Filter.And( + Filter.Eq(x => x.AppId, app.Id), + Filter.Exists(x => x.SchemaId), + Filter_ByScope(scope), + Filter.Text(queryText))) + .Only(x => x.ContentId).Limit(limit) + .ToListAsync(); + + return bySchema.Select(x => Guid.Parse(x["_ci"].AsString)).Distinct().ToList(); + } + + private static FilterDefinition Filter_ByScope(SearchScope scope) + { + if (scope == SearchScope.All) + { + return Filter.Eq(x => x.ServeAll, true); + } + else + { + return Filter.Eq(x => x.ServePublished, true); + } + } + } +} diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs new file mode 100644 index 000000000..650616f31 --- /dev/null +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs @@ -0,0 +1,49 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using MongoDB.Bson; +using MongoDB.Bson.Serialization.Attributes; + +namespace Squidex.Domain.Apps.Entities.MongoDb.FullText +{ + public sealed class MongoTextIndexEntity + { + [BsonId] + [BsonElement] + [BsonRepresentation(BsonType.String)] + public string DocId { get; set; } + + [BsonRequired] + [BsonElement("_ci")] + [BsonRepresentation(BsonType.String)] + public Guid ContentId { get; set; } + + [BsonRequired] + [BsonElement("_ai")] + [BsonRepresentation(BsonType.String)] + public Guid AppId { get; set; } + + [BsonRequired] + [BsonElement("_si")] + [BsonRepresentation(BsonType.String)] + public Guid SchemaId { get; set; } + + [BsonRequired] + [BsonElement("fa")] + public bool ServeAll { get; set; } + + [BsonRequired] + [BsonElement("fp")] + public bool ServePublished { get; set; } + + [BsonIgnoreIfNull] + [BsonElement("t")] + public List Texts { get; set; } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Assets.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs similarity index 52% rename from backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Assets.cs rename to backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs index 768c0e6eb..dca5d7d8a 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Assets.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs @@ -5,10 +5,18 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -namespace Squidex.Domain.Apps.Entities.Contents.Text +using MongoDB.Bson.Serialization.Attributes; + +namespace Squidex.Domain.Apps.Entities.MongoDb.FullText { - public class TextIndexerTests_Assets : TextIndexerTestsBase + public sealed class MongoTextIndexEntityText { - public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.Assets()); + [BsonRequired] + [BsonElement("t")] + public string Text { get; set; } + + [BsonIgnoreIfNull] + [BsonElement("language")] + public string Language { get; set; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs index 95dede9c9..c66ec3db5 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs @@ -6,6 +6,8 @@ // ========================================================================== using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using MongoDB.Bson.Serialization; using MongoDB.Driver; @@ -57,5 +59,38 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText { return Collection.ReplaceOneAsync(x => x.ContentId == state.ContentId, state, UpsertReplace); } + + public async Task> GetAsync(HashSet ids) + { + var entities = await Collection.Find(Filter.In(x => x.ContentId, ids)).ToListAsync(); + + return entities.ToDictionary(x => x.ContentId); + } + + public Task SetAsync(List updates) + { + var writes = new List>(); + + foreach (var update in updates) + { + if (update.IsDeleted) + { + writes.Add( + new DeleteOneModel( + Filter.Eq(x => x.ContentId, update.ContentId))); + } + else + { + writes.Add( + new ReplaceOneModel( + Filter.Eq(x => x.ContentId, update.ContentId), update) + { + IsUpsert = true + }); + } + } + + return Collection.BulkWriteAsync(writes); + } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs index a1109021c..60d4c3db0 100644 --- a/backend/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using MongoDB.Bson.Serialization; @@ -64,9 +65,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.History } } - public Task InsertAsync(HistoryEvent item) + public async Task InsertManyAsync(IEnumerable historyEvents) { - return Collection.ReplaceOneAsync(x => x.Id == item.Id, item, UpsertReplace); + var writes = historyEvents + .Select(x => + new ReplaceOneModel(Filter.Eq(y => y.Id, x.Id), x) + { + IsUpsert = true + }) + .ToList(); + + if (writes.Count > 0) + { + await Collection.BulkWriteAsync(writes); + } } public Task RemoveAsync(Guid appId) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Apps/Invitation/InvitationEventConsumer.cs b/backend/src/Squidex.Domain.Apps.Entities/Apps/Invitation/InvitationEventConsumer.cs index a2f6d47b8..5a994c13a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Apps/Invitation/InvitationEventConsumer.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Apps/Invitation/InvitationEventConsumer.cs @@ -45,16 +45,6 @@ namespace Squidex.Domain.Apps.Entities.Apps.Invitation this.log = log; } - public bool Handles(StoredEvent @event) - { - return true; - } - - public Task ClearAsync() - { - return Task.CompletedTask; - } - public async Task On(Envelope @event) { if (!emailSender.IsActive) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs index d721979ec..540c1c635 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs @@ -9,14 +9,13 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using Squidex.Infrastructure; -using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.UsageTracking; #pragma warning disable CS0649 namespace Squidex.Domain.Apps.Entities.Assets { - public partial class AssetUsageTracker : IAssetUsageTracker, IEventConsumer + public partial class AssetUsageTracker : IAssetUsageTracker { private const string CounterTotalCount = "TotalAssets"; private const string CounterTotalSize = "TotalSize"; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs index a00abd5f6..9dba562e2 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs @@ -13,26 +13,26 @@ using Squidex.Infrastructure.UsageTracking; namespace Squidex.Domain.Apps.Entities.Assets { - public partial class AssetUsageTracker + public partial class AssetUsageTracker : IEventConsumer { - public string Name + public int BatchSize { - get { return GetType().Name; } + get { return 1000; } } - public string EventsFilter + public int BatchDelay { - get { return "^asset-"; } + get { return 1000; } } - public bool Handles(StoredEvent @event) + public string Name { - return true; + get { return GetType().Name; } } - public Task ClearAsync() + public string EventsFilter { - return Task.CompletedTask; + get { return "^asset-"; } } public Task On(Envelope @event) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs b/backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs index fbe4d92c6..4a1a48288 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs @@ -57,11 +57,6 @@ namespace Squidex.Domain.Apps.Entities.Assets folderDeletedType = typeNameRegistry.GetName(); } - public Task ClearAsync() - { - return Task.CompletedTask; - } - public bool Handles(StoredEvent @event) { return @event.Data.Type == folderDeletedType; diff --git a/backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs index a9cf736a5..87bdd60d3 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs @@ -223,7 +223,7 @@ namespace Squidex.Domain.Apps.Entities.Backup log.LogError(ex, logContext, (ctx, w) => { - w.WriteProperty("action", "retore"); + w.WriteProperty("action", "restore"); w.WriteProperty("status", "failed"); w.WriteProperty("operationId", ctx.jobId); w.WriteProperty("url", ctx.jobUrl); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs index 5d916a770..c031b44e4 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs @@ -18,14 +18,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic [ExcludeFromCodeCoverage] public sealed class ElasticSearchTextIndex : ITextIndex { - private const string IndexName = "contents"; private readonly ElasticLowLevelClient client; + private readonly string indexName; + private readonly bool waitForTesting; - public ElasticSearchTextIndex() + public ElasticSearchTextIndex(string configurationString, string indexName, bool waitForTesting = false) { - var config = new ConnectionConfiguration(new Uri("http://localhost:9200")); + Guard.NotNull(configurationString, nameof(configurationString)); + Guard.NotNull(indexName, nameof(indexName)); + + var config = new ConnectionConfiguration(new Uri(configurationString)); client = new ElasticLowLevelClient(config); + + this.indexName = indexName; + + this.waitForTesting = waitForTesting; } public Task ClearAsync() @@ -33,14 +41,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic return Task.CompletedTask; } - public async Task ExecuteAsync(NamedId appId, NamedId schemaId, params IndexCommand[] commands) + public async Task ExecuteAsync(params IndexCommand[] commands) { foreach (var command in commands) { switch (command) { case UpsertIndexEntry upsert: - await UpsertAsync(appId, schemaId, upsert); + await UpsertAsync(upsert); break; case UpdateIndexEntry update: await UpdateAsync(update); @@ -50,23 +58,28 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic break; } } + + if (waitForTesting) + { + await Task.Delay(1000); + } } - private async Task UpsertAsync(NamedId appId, NamedId schemaId, UpsertIndexEntry upsert) + private async Task UpsertAsync(UpsertIndexEntry upsert) { var data = new { - appId = appId.Id, - appName = appId.Name, + appId = upsert.AppId.Id, + appName = upsert.AppId.Name, contentId = upsert.ContentId, - schemaId = schemaId.Id, - schemaName = schemaId.Name, + schemaId = upsert.SchemaId.Id, + schemaName = upsert.SchemaId.Name, serveAll = upsert.ServeAll, servePublished = upsert.ServePublished, texts = upsert.Texts, }; - var result = await client.IndexAsync(IndexName, upsert.DocId, CreatePost(data)); + var result = await client.IndexAsync(indexName, upsert.DocId, CreatePost(data)); if (!result.Success) { @@ -80,12 +93,12 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic { doc = new { - update.ServeAll, - update.ServePublished + serveAll = update.ServeAll, + servePublished = update.ServePublished } }; - var result = await client.UpdateAsync(IndexName, update.DocId, CreatePost(data)); + var result = await client.UpdateAsync(indexName, update.DocId, CreatePost(data)); if (!result.Success) { @@ -95,7 +108,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic private Task DeleteAsync(DeleteIndexEntry delete) { - return client.DeleteAsync(IndexName, delete.DocId); + return client.DeleteAsync(indexName, delete.DocId); } private static PostData CreatePost(T data) @@ -155,7 +168,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic { var bySchema = new { - term = new Dictionary + terms = new Dictionary { ["schemaId.keyword"] = filter.SchemaIds } @@ -171,7 +184,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic } } - var result = await client.SearchAsync(IndexName, CreatePost(query)); + var result = await client.SearchAsync(indexName, CreatePost(query)); if (!result.Success) { diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs index fd4d8c7e9..c8a01dde4 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs @@ -9,7 +9,6 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; using Squidex.Domain.Apps.Entities.Apps; -using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Contents.Text { @@ -19,6 +18,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text Task ClearAsync(); - Task ExecuteAsync(NamedId appId, NamedId schemaId, params IndexCommand[] commands); + Task ExecuteAsync(params IndexCommand[] commands); } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs index ce2658e47..c40da634d 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs @@ -5,10 +5,17 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using Squidex.Infrastructure; + namespace Squidex.Domain.Apps.Entities.Contents.Text { public abstract class IndexCommand { + public NamedId AppId { get; set; } + + public NamedId SchemaId { get; set; } + public string DocId { get; set; } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IIndex.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IIndex.cs deleted file mode 100644 index 78aba59bf..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IIndex.cs +++ /dev/null @@ -1,28 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Lucene.Net.Search; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public interface IIndex - { - Analyzer? Analyzer { get; } - - IndexReader? Reader { get; } - - IndexSearcher? Searcher { get; } - - IndexWriter Writer { get; } - - void EnsureReader(); - - void MarkStale(); - } -} \ No newline at end of file diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/ILuceneTextIndexGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/ILuceneTextIndexGrain.cs deleted file mode 100644 index 964a8fa37..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/ILuceneTextIndexGrain.cs +++ /dev/null @@ -1,23 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans; -using Orleans.Concurrency; -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public interface ILuceneTextIndexGrain : IGrainWithGuidKey - { - Task IndexAsync(NamedId schemaId, Immutable updates); - - Task> SearchAsync(string queryText, SearchFilter? filter, SearchContext context); - } -} \ No newline at end of file diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager.cs deleted file mode 100644 index 2408b3b7f..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager.cs +++ /dev/null @@ -1,151 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Log; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public sealed partial class IndexManager : DisposableObjectBase - { - private readonly Dictionary indices = new Dictionary(); - private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); - private readonly IIndexStorage indexStorage; - private readonly ISemanticLog log; - - public IndexManager(IIndexStorage indexStorage, ISemanticLog log) - { - Guard.NotNull(indexStorage, nameof(indexStorage)); - Guard.NotNull(log, nameof(log)); - - this.indexStorage = indexStorage; - - this.log = log; - } - - protected override void DisposeObject(bool disposing) - { - if (disposing) - { - ReleaseAllAsync().Wait(); - } - } - - public Task ClearAsync() - { - return indexStorage.ClearAsync(); - } - - public async Task AcquireAsync(Guid ownerId) - { - IndexHolder? indexHolder; - - try - { - await lockObject.WaitAsync(); - - if (indices.TryGetValue(ownerId, out indexHolder)) - { - log.LogWarning(w => w - .WriteProperty("message", "Unreleased index found.") - .WriteProperty("ownerId", ownerId.ToString())); - - await CommitInternalAsync(indexHolder, true); - } - - var directory = await indexStorage.CreateDirectoryAsync(ownerId); - - indexHolder = new IndexHolder(ownerId, directory); - indices[ownerId] = indexHolder; - } - finally - { - lockObject.Release(); - } - - return indexHolder; - } - - public async Task ReleaseAsync(IIndex index) - { - Guard.NotNull(index, nameof(index)); - - var indexHolder = (IndexHolder)index; - - try - { - lockObject.Wait(); - - indexHolder.Dispose(); - indices.Remove(indexHolder.Id); - } - finally - { - lockObject.Release(); - } - - await CommitInternalAsync(indexHolder, true); - } - - public Task CommitAsync(IIndex index) - { - Guard.NotNull(index, nameof(index)); - - return CommitInternalAsync(index, false); - } - - private async Task CommitInternalAsync(IIndex index, bool dispose) - { - if (index is IndexHolder holder) - { - if (dispose) - { - holder.Dispose(); - } - else - { - holder.Commit(); - } - - await indexStorage.WriteAsync(holder.Directory, holder.Snapshotter); - } - } - - private async Task ReleaseAllAsync() - { - var current = indices.Values.ToList(); - - try - { - lockObject.Wait(); - - indices.Clear(); - } - finally - { - lockObject.Release(); - } - - if (current.Count > 0) - { - log.LogWarning(w => w - .WriteProperty("message", "Unreleased indices found.") - .WriteProperty("count", indices.Count)); - - foreach (var index in current) - { - await CommitInternalAsync(index, true); - } - } - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager_Impl.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager_Impl.cs deleted file mode 100644 index edb5f0732..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager_Impl.cs +++ /dev/null @@ -1,179 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using Lucene.Net.Analysis; -using Lucene.Net.Index; -using Lucene.Net.Search; -using Lucene.Net.Store; -using Lucene.Net.Util; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public sealed partial class IndexManager - { - private sealed class IndexHolder : IDisposable, IIndex - { - private const LuceneVersion Version = LuceneVersion.LUCENE_48; - private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); - private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); - private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); - private readonly Directory directory; - private IndexWriter indexWriter; - private IndexSearcher? indexSearcher; - private DirectoryReader? indexReader; - private bool isDisposed; - - public Analyzer Analyzer - { - get { return SharedAnalyzer; } - } - - public SnapshotDeletionPolicy Snapshotter - { - get { return snapshotter; } - } - - public Directory Directory - { - get { return directory; } - } - - public IndexWriter Writer - { - get - { - ThrowIfReleased(); - - if (indexWriter == null) - { - throw new InvalidOperationException("Index writer has not been created yet. Call Open()"); - } - - return indexWriter; - } - } - - public IndexReader? Reader - { - get - { - ThrowIfReleased(); - - return indexReader; - } - } - - public IndexSearcher? Searcher - { - get - { - ThrowIfReleased(); - - return indexSearcher; - } - } - - public Guid Id { get; } - - public IndexHolder(Guid id, Directory directory) - { - Id = id; - - this.directory = directory; - - RecreateIndexWriter(); - - if (indexWriter.NumDocs > 0) - { - EnsureReader(); - } - } - - public void Dispose() - { - if (!isDisposed) - { - indexReader?.Dispose(); - indexReader = null; - - indexWriter.Dispose(); - - isDisposed = true; - } - } - - private IndexWriter RecreateIndexWriter() - { - var config = new IndexWriterConfig(Version, Analyzer) - { - IndexDeletionPolicy = snapshotter, - MergePolicy = new TieredMergePolicy(), - MergeScheduler = MergeScheduler - }; - - indexWriter = new IndexWriter(directory, config); - - MarkStale(); - - return indexWriter; - } - - public void EnsureReader() - { - ThrowIfReleased(); - - if (indexReader == null && indexWriter != null) - { - indexReader = indexWriter.GetReader(true); - indexSearcher = new IndexSearcher(indexReader); - } - } - - public void MarkStale() - { - ThrowIfReleased(); - - MarkStaleInternal(); - } - - private void MarkStaleInternal() - { - if (indexReader != null) - { - indexReader.Dispose(); - indexReader = null; - indexSearcher = null; - } - } - - internal void Commit() - { - try - { - MarkStaleInternal(); - - indexWriter.Commit(); - } - catch (OutOfMemoryException) - { - RecreateIndexWriter(); - - throw; - } - } - - private void ThrowIfReleased() - { - if (indexWriter == null) - { - throw new InvalidOperationException("Index is already released or not open yet."); - } - } - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneExtensions.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneExtensions.cs deleted file mode 100644 index 422ee97e3..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneExtensions.cs +++ /dev/null @@ -1,53 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using Lucene.Net.Index; -using Lucene.Net.Util; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public static class LuceneExtensions - { - public static BytesRef GetBinaryValue(this IndexReader? reader, string field, int docId, BytesRef? result = null) - { - if (result != null) - { - Array.Clear(result.Bytes, 0, result.Bytes.Length); - } - else - { - result = new BytesRef(); - } - - if (reader == null || docId < 0) - { - return result; - } - - var leaves = reader.Leaves; - - if (leaves.Count == 1) - { - var docValues = leaves[0].AtomicReader.GetBinaryDocValues(field); - - docValues.Get(docId, result); - } - else if (leaves.Count > 1) - { - var subIndex = ReaderUtil.SubIndex(docId, leaves); - - var subLeave = leaves[subIndex]; - var subValues = subLeave.AtomicReader.GetBinaryDocValues(field); - - subValues.Get(docId - subLeave.DocBase, result); - } - - return result; - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndex.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndex.cs deleted file mode 100644 index 623beebbf..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndex.cs +++ /dev/null @@ -1,70 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Threading.Tasks; -using Orleans; -using Orleans.Concurrency; -using Squidex.Domain.Apps.Entities.Apps; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Log; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public sealed class LuceneTextIndex : ITextIndex - { - private readonly IGrainFactory grainFactory; - private readonly IndexManager indexManager; - - public LuceneTextIndex(IGrainFactory grainFactory, IndexManager indexManager) - { - Guard.NotNull(grainFactory, nameof(grainFactory)); - Guard.NotNull(indexManager, nameof(indexManager)); - - this.grainFactory = grainFactory; - - this.indexManager = indexManager; - } - - public Task ClearAsync() - { - return indexManager.ClearAsync(); - } - - public async Task?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope) - { - if (string.IsNullOrWhiteSpace(queryText)) - { - return null; - } - - var index = grainFactory.GetGrain(app.Id); - - using (Profiler.TraceMethod()) - { - var context = CreateContext(app, scope); - - return await index.SearchAsync(queryText, filter, context); - } - } - - private static SearchContext CreateContext(IAppEntity app, SearchScope scope) - { - var languages = new HashSet(app.LanguagesConfig.AllKeys); - - return new SearchContext { Languages = languages, Scope = scope }; - } - - public Task ExecuteAsync(NamedId appId, NamedId schemaId, params IndexCommand[] commands) - { - var index = grainFactory.GetGrain(appId.Id); - - return index.IndexAsync(schemaId, commands.AsImmutable()); - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs deleted file mode 100644 index 57cbd81fd..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs +++ /dev/null @@ -1,262 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Lucene.Net.Documents; -using Lucene.Net.Index; -using Lucene.Net.QueryParsers.Classic; -using Lucene.Net.Search; -using Lucene.Net.Util; -using Orleans.Concurrency; -using Squidex.Domain.Apps.Core; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Orleans; -using Squidex.Infrastructure.Validation; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public sealed class LuceneTextIndexGrain : GrainOfGuid, ILuceneTextIndexGrain - { - private const LuceneVersion Version = LuceneVersion.LUCENE_48; - private const int MaxResults = 2000; - private const int MaxUpdates = 400; - private const string MetaId = "_id"; - private const string MetaFor = "_fd"; - private const string MetaContentId = "_cid"; - private const string MetaSchemaId = "_si"; - private const string MetaSchemaName = "_sn"; - private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(10); - private static readonly string[] Invariant = { InvariantPartitioning.Key }; - private readonly IndexManager indexManager; - private IDisposable? timer; - private IIndex index; - private QueryParser? queryParser; - private HashSet? currentLanguages; - private int updates; - - public LuceneTextIndexGrain(IndexManager indexManager) - { - Guard.NotNull(indexManager, nameof(indexManager)); - - this.indexManager = indexManager; - } - - public override async Task OnDeactivateAsync() - { - if (index != null) - { - await CommitAsync(); - - await indexManager.ReleaseAsync(index); - } - } - - protected override async Task OnActivateAsync(Guid key) - { - index = await indexManager.AcquireAsync(key); - } - - public Task> SearchAsync(string queryText, SearchFilter? filter, SearchContext context) - { - var result = new List(); - - if (!string.IsNullOrWhiteSpace(queryText)) - { - index.EnsureReader(); - - if (index.Searcher != null) - { - var query = BuildQuery(queryText, filter, context); - - var hits = index.Searcher.Search(query, MaxResults).ScoreDocs; - - if (hits.Length > 0) - { - var buffer = new BytesRef(2); - - var found = new HashSet(); - - foreach (var hit in hits) - { - var forValue = index.Reader.GetBinaryValue(MetaFor, hit.Doc, buffer); - - if (context.Scope == SearchScope.All && forValue.Bytes[0] != 1) - { - continue; - } - - if (context.Scope == SearchScope.Published && forValue.Bytes[1] != 1) - { - continue; - } - - var document = index.Searcher.Doc(hit.Doc); - - if (document != null) - { - var idString = document.Get(MetaContentId); - - if (Guid.TryParse(idString, out var id)) - { - if (found.Add(id)) - { - result.Add(id); - } - } - } - } - } - } - } - - return Task.FromResult(result); - } - - private Query BuildQuery(string query, SearchFilter? filter, SearchContext context) - { - if (queryParser == null || currentLanguages == null || !currentLanguages.SetEquals(context.Languages)) - { - var fields = context.Languages.Union(Invariant).ToArray(); - - queryParser = new MultiFieldQueryParser(Version, fields, index.Analyzer); - - currentLanguages = context.Languages; - } - - try - { - var byQuery = queryParser.Parse(query); - - if (filter?.SchemaIds.Count > 0) - { - var bySchemas = new BooleanQuery - { - Boost = 2f - }; - - foreach (var schemaId in filter.SchemaIds) - { - var term = new Term(MetaSchemaId, schemaId.ToString()); - - bySchemas.Add(new TermQuery(term), Occur.SHOULD); - } - - var occur = filter.Must ? Occur.MUST : Occur.SHOULD; - - return new BooleanQuery - { - { byQuery, Occur.MUST }, - { bySchemas, occur } - }; - } - - return byQuery; - } - catch (ParseException ex) - { - throw new ValidationException(ex.Message); - } - } - - private async Task TryCommitAsync() - { - timer?.Dispose(); - - updates++; - - if (updates >= MaxUpdates) - { - await CommitAsync(); - - return true; - } - else - { - index.MarkStale(); - - try - { - timer = RegisterTimer(_ => CommitAsync(), null, CommitDelay, CommitDelay); - } - catch (InvalidOperationException) - { - return false; - } - } - - return false; - } - - public async Task CommitAsync() - { - if (updates > 0) - { - await indexManager.CommitAsync(index); - - updates = 0; - } - } - - public Task IndexAsync(NamedId schemaId, Immutable updates) - { - foreach (var command in updates.Value) - { - switch (command) - { - case DeleteIndexEntry delete: - index.Writer.DeleteDocuments(new Term(MetaId, delete.DocId)); - break; - case UpdateIndexEntry update: - try - { - var values = GetValue(update.ServeAll, update.ServePublished); - - index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, values); - } - catch (ArgumentException) - { - } - - break; - case UpsertIndexEntry upsert: - { - var document = new Document(); - - document.AddStringField(MetaId, upsert.DocId, Field.Store.YES); - document.AddStringField(MetaContentId, upsert.ContentId.ToString(), Field.Store.YES); - document.AddStringField(MetaSchemaId, schemaId.Id.ToString(), Field.Store.YES); - document.AddStringField(MetaSchemaName, schemaId.Name, Field.Store.YES); - document.AddBinaryDocValuesField(MetaFor, GetValue(upsert.ServeAll, upsert.ServePublished)); - - foreach (var (key, value) in upsert.Texts) - { - document.AddTextField(key, value, Field.Store.NO); - } - - index.Writer.UpdateDocument(new Term(MetaId, upsert.DocId), document); - - break; - } - } - } - - return TryCommitAsync(); - } - - private static BytesRef GetValue(bool forDraft, bool forPublished) - { - return new BytesRef(new[] - { - (byte)(forDraft ? 1 : 0), - (byte)(forPublished ? 1 : 0) - }); - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/MultiLanguageAnalyzer.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/MultiLanguageAnalyzer.cs deleted file mode 100644 index 1bf08329e..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/MultiLanguageAnalyzer.cs +++ /dev/null @@ -1,65 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Collections.Generic; -using System.Linq; -using Lucene.Net.Analysis; -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Util; -using Squidex.Infrastructure; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public sealed class MultiLanguageAnalyzer : AnalyzerWrapper - { - private readonly StandardAnalyzer fallbackAnalyzer; - private readonly Dictionary analyzers = new Dictionary(StringComparer.OrdinalIgnoreCase); - - public MultiLanguageAnalyzer(LuceneVersion version) - : base(PER_FIELD_REUSE_STRATEGY) - { - fallbackAnalyzer = new StandardAnalyzer(version); - - foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) - { - if (typeof(Analyzer).IsAssignableFrom(type)) - { - var language = type.Namespace!.Split('.').Last(); - - if (language.Length == 2) - { - try - { - var analyzer = Activator.CreateInstance(type, version)!; - - analyzers[language] = (Analyzer)analyzer; - } - catch (MissingMethodException) - { - continue; - } - } - } - } - } - - protected override Analyzer GetWrappedAnalyzer(string fieldName) - { - if (fieldName.Length >= 2) - { - var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; - - return analyzer; - } - else - { - return fallbackAnalyzer; - } - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs deleted file mode 100644 index 2d6fde2c2..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs +++ /dev/null @@ -1,114 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.IO; -using System.IO.Compression; -using System.Threading.Tasks; -using Lucene.Net.Index; -using Lucene.Net.Store; -using Squidex.Infrastructure; -using Squidex.Infrastructure.Assets; -using LuceneDirectory = Lucene.Net.Store.Directory; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage -{ - public sealed class AssetIndexStorage : IIndexStorage - { - private const string ArchiveFile = "Archive.zip"; - private readonly IAssetStore assetStore; - - public AssetIndexStorage(IAssetStore assetStore) - { - Guard.NotNull(assetStore, nameof(assetStore)); - - this.assetStore = assetStore; - } - - public async Task CreateDirectoryAsync(Guid ownerId) - { - var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "LocalIndices", ownerId.ToString())); - - if (directoryInfo.Exists) - { - directoryInfo.Delete(true); - } - - directoryInfo.Create(); - - using (var fileStream = GetArchiveStream(directoryInfo)) - { - try - { - await assetStore.DownloadAsync(directoryInfo.Name, fileStream); - - fileStream.Position = 0; - - using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) - { - zipArchive.ExtractToDirectory(directoryInfo.FullName); - } - } - catch (AssetNotFoundException) - { - } - } - - var directory = FSDirectory.Open(directoryInfo); - - return directory; - } - - public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) - { - var directoryInfo = ((FSDirectory)directory).Directory; - - var commit = snapshotter.Snapshot(); - try - { - using (var fileStream = GetArchiveStream(directoryInfo)) - { - using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) - { - foreach (var fileName in commit.FileNames) - { - var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); - - zipArchive.CreateEntryFromFile(file.FullName, file.Name); - } - } - - fileStream.Position = 0; - - await assetStore.UploadAsync(directoryInfo.Name, fileStream, true); - } - } - finally - { - snapshotter.Release(commit); - } - } - - private static FileStream GetArchiveStream(DirectoryInfo directoryInfo) - { - var path = Path.Combine(directoryInfo.FullName, ArchiveFile); - - return new FileStream( - path, - FileMode.Create, - FileAccess.ReadWrite, - FileShare.None, - 4096, - FileOptions.DeleteOnClose); - } - - public Task ClearAsync() - { - return Task.CompletedTask; - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs deleted file mode 100644 index b90926c32..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs +++ /dev/null @@ -1,37 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.IO; -using System.Threading.Tasks; -using Lucene.Net.Index; -using Lucene.Net.Store; -using LuceneDirectory = Lucene.Net.Store.Directory; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage -{ - public sealed class FileIndexStorage : IIndexStorage - { - public Task CreateDirectoryAsync(Guid ownerId) - { - var folderName = $"Indexes/{ownerId}"; - var folderPath = Path.Combine(Path.GetTempPath(), folderName); - - return Task.FromResult(FSDirectory.Open(folderPath)); - } - - public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) - { - return Task.CompletedTask; - } - - public Task ClearAsync() - { - return Task.CompletedTask; - } - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs deleted file mode 100644 index 37036c469..000000000 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs +++ /dev/null @@ -1,23 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using Lucene.Net.Index; -using Lucene.Net.Store; - -namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene -{ - public interface IIndexStorage - { - Task CreateDirectoryAsync(Guid ownerId); - - Task WriteAsync(Directory directory, SnapshotDeletionPolicy snapshotter); - - Task ClearAsync(); - } -} diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs index 7d057b006..e9dacb3be 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs @@ -6,6 +6,7 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.Threading.Tasks; using Squidex.Infrastructure; using Squidex.Infrastructure.Caching; @@ -15,7 +16,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State public sealed class CachingTextIndexerState : ITextIndexerState { private readonly ITextIndexerState inner; - private LRUCache> cache = new LRUCache>(1000); + private readonly LRUCache> cache = new LRUCache>(10000); public CachingTextIndexerState(ITextIndexerState inner) { @@ -28,37 +29,69 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State { await inner.ClearAsync(); - cache = new LRUCache>(1000); + cache.Clear(); } - public async Task GetAsync(Guid contentId) + public async Task> GetAsync(HashSet ids) { - if (cache.TryGetValue(contentId, out var value)) + Guard.NotNull(ids, nameof(ids)); + + var missingIds = new HashSet(); + + var result = new Dictionary(); + + foreach (var id in ids) { - return value.Item1; + if (cache.TryGetValue(id, out var state)) + { + if (state.Item1 != null) + { + result[id] = state.Item1; + } + } + else + { + missingIds.Add(id); + } } - var result = await inner.GetAsync(contentId); + if (missingIds.Count > 0) + { + var fromInner = await inner.GetAsync(missingIds); - cache.Set(contentId, Tuple.Create(result)); + foreach (var (id, state) in fromInner) + { + result[id] = state; + } - return result; - } + foreach (var id in missingIds) + { + var state = fromInner.GetOrDefault(id); - public Task SetAsync(TextContentState state) - { - Guard.NotNull(state, nameof(state)); - - cache.Set(state.ContentId, Tuple.Create(state)); + cache.Set(id, Tuple.Create(state)); + } + } - return inner.SetAsync(state); + return result; } - public Task RemoveAsync(Guid contentId) + public Task SetAsync(List updates) { - cache.Set(contentId, Tuple.Create(null)); + Guard.NotNull(updates, nameof(updates)); + + foreach (var update in updates) + { + if (update.IsDeleted) + { + cache.Set(update.ContentId, Tuple.Create(null)); + } + else + { + cache.Set(update.ContentId, Tuple.Create(update)); + } + } - return inner.RemoveAsync(contentId); + return inner.SetAsync(updates); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs index b1693a10b..772f19c9a 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs @@ -6,17 +6,16 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace Squidex.Domain.Apps.Entities.Contents.Text.State { public interface ITextIndexerState { - Task GetAsync(Guid contentId); + Task> GetAsync(HashSet ids); - Task SetAsync(TextContentState state); - - Task RemoveAsync(Guid contentId); + Task SetAsync(List updates); Task ClearAsync(); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs index 017102b36..9c1278768 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Squidex.Infrastructure; namespace Squidex.Domain.Apps.Entities.Contents.Text.State { @@ -22,26 +23,38 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State return Task.CompletedTask; } - public Task GetAsync(Guid contentId) + public Task> GetAsync(HashSet ids) { - if (states.TryGetValue(contentId, out var result)) + Guard.NotNull(ids, nameof(ids)); + + var result = new Dictionary(); + + foreach (var id in ids) { - return Task.FromResult(result); + if (states.TryGetValue(id, out var state)) + { + result.Add(id, state); + } } - return Task.FromResult(null); + return Task.FromResult(result); } - public Task RemoveAsync(Guid contentId) + public Task SetAsync(List updates) { - states.Remove(contentId); - - return Task.CompletedTask; - } + Guard.NotNull(updates, nameof(updates)); - public Task SetAsync(TextContentState state) - { - states[state.ContentId] = state; + foreach (var update in updates) + { + if (update.IsDeleted) + { + states.Remove(update.ContentId); + } + else + { + states[update.ContentId] = update; + } + } return Task.CompletedTask; } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs index 3f2bbe07e..150ee5d64 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs @@ -19,6 +19,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State public string? DocIdForPublished { get; set; } + public bool IsDeleted { get; set; } + public void GenerateDocIdNew() { if (DocIdCurrent?.EndsWith("_2") != false) diff --git a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs index ca4c64fc4..4c00f7053 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs @@ -5,6 +5,9 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Contents.Text.State; @@ -17,266 +20,332 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public sealed class TextIndexingProcess : IEventConsumer { private const string NotFound = "<404>"; - private readonly ITextIndex textIndexer; + private readonly ITextIndex textIndex; private readonly ITextIndexerState textIndexerState; - public string Name + public int BatchSize { - get { return "TextIndexer4"; } + get { return 1000; } } - public string EventsFilter + public int BatchDelay { - get { return "^content-"; } + get { return 1000; } } - public ITextIndex TextIndexer + public string Name { - get { return textIndexer; } + get { return "TextIndexer5"; } } - public TextIndexingProcess(ITextIndex textIndexer, ITextIndexerState textIndexerState) + public string EventsFilter { - Guard.NotNull(textIndexer, nameof(textIndexer)); - Guard.NotNull(textIndexerState, nameof(textIndexerState)); - - this.textIndexer = textIndexer; - this.textIndexerState = textIndexerState; + get { return "^content-"; } } - public bool Handles(StoredEvent @event) + public ITextIndex TextIndex { - return true; + get { return textIndex; } } - public async Task ClearAsync() + private sealed class Updates { - await textIndexer.ClearAsync(); - await textIndexerState.ClearAsync(); - } + private readonly Dictionary states; + private readonly Dictionary updates = new Dictionary(); + private readonly Dictionary commands = new Dictionary(); - public async Task On(Envelope @event) - { - switch (@event.Payload) + public Updates(Dictionary states) { - case ContentCreated created: - await CreateAsync(created, created.Data); - break; - case ContentUpdated updated: - await UpdateAsync(updated, updated.Data); - break; - case ContentStatusChanged statusChanged when statusChanged.Status == Status.Published: - await PublishAsync(statusChanged); - break; - case ContentStatusChanged statusChanged: - await UnpublishAsync(statusChanged); - break; - case ContentDraftDeleted draftDelted: - await DeleteDraftAsync(draftDelted); - break; - case ContentDeleted deleted: - await DeleteAsync(deleted); - break; - case ContentDraftCreated draftCreated: - { - await CreateDraftAsync(draftCreated); + this.states = states; + } - if (draftCreated.MigratedData != null) + public async Task WriteAsync(ITextIndex textIndex, ITextIndexerState textIndexerState) + { + if (commands.Count > 0) + { + await textIndex.ExecuteAsync(commands.Values.ToArray()); + } + + if (updates.Count > 0) + { + await textIndexerState.SetAsync(updates.Values.ToList()); + } + } + + public void On(Envelope @event) + { + switch (@event.Payload) + { + case ContentCreated created: + Create(created, created.Data); + break; + case ContentUpdated updated: + Update(updated, updated.Data); + break; + case ContentStatusChanged statusChanged when statusChanged.Status == Status.Published: + Publish(statusChanged); + break; + case ContentStatusChanged statusChanged: + Unpublish(statusChanged); + break; + case ContentDraftDeleted draftDelted: + DeleteDraft(draftDelted); + break; + case ContentDeleted deleted: + Delete(deleted); + break; + case ContentDraftCreated draftCreated: { - await UpdateAsync(draftCreated, draftCreated.MigratedData); + CreateDraft(draftCreated); + + if (draftCreated.MigratedData != null) + { + Update(draftCreated, draftCreated.MigratedData); + } } - } - break; + break; + } } - } - private async Task CreateAsync(ContentEvent @event, NamedContentData data) - { - var state = new TextContentState + private void Create(ContentEvent @event, NamedContentData data) { - ContentId = @event.ContentId - }; + var state = new TextContentState + { + ContentId = @event.ContentId + }; - state.GenerateDocIdCurrent(); + state.GenerateDocIdCurrent(); - await IndexAsync(@event, - new UpsertIndexEntry - { - ContentId = @event.ContentId, - DocId = state.DocIdCurrent, - ServeAll = true, - ServePublished = false, - Texts = data.ToTexts() - }); - - await textIndexerState.SetAsync(state); - } + Index(@event, + new UpsertIndexEntry + { + ContentId = @event.ContentId, + DocId = state.DocIdCurrent, + ServeAll = true, + ServePublished = false, + Texts = data.ToTexts() + }); - private async Task CreateDraftAsync(ContentEvent @event) - { - var state = await textIndexerState.GetAsync(@event.ContentId); + states[state.ContentId] = state; + + updates[state.ContentId] = state; + } - if (state != null) + private void CreateDraft(ContentEvent @event) { - state.GenerateDocIdNew(); + if (states.TryGetValue(@event.ContentId, out var state)) + { + state.GenerateDocIdNew(); - await textIndexerState.SetAsync(state); + updates[state.ContentId] = state; + } } - } - - private async Task UpdateAsync(ContentEvent @event, NamedContentData data) - { - var state = await textIndexerState.GetAsync(@event.ContentId); - if (state != null) + private void Unpublish(ContentEvent @event) { - if (state.DocIdNew != null) + if (states.TryGetValue(@event.ContentId, out var state) && state.DocIdForPublished != null) { - await IndexAsync(@event, - new UpsertIndexEntry - { - ContentId = @event.ContentId, - DocId = state.DocIdNew, - ServeAll = true, - ServePublished = false, - Texts = data.ToTexts() - }, + Index(@event, new UpdateIndexEntry { - DocId = state.DocIdCurrent, - ServeAll = false, - ServePublished = true - }); - } - else - { - var isPublished = state.DocIdCurrent == state.DocIdForPublished; - - await IndexAsync(@event, - new UpsertIndexEntry - { - ContentId = @event.ContentId, - DocId = state.DocIdCurrent, + DocId = state.DocIdForPublished, ServeAll = true, - ServePublished = isPublished, - Texts = data.ToTexts() + ServePublished = false }); - } - await textIndexerState.SetAsync(state); + state.DocIdForPublished = null; + + updates[state.ContentId] = state; + } } - } - private async Task UnpublishAsync(ContentEvent @event) - { - var state = await textIndexerState.GetAsync(@event.ContentId); + private void Update(ContentEvent @event, NamedContentData data) + { + if (states.TryGetValue(@event.ContentId, out var state)) + { + if (state.DocIdNew != null) + { + Index(@event, + new UpsertIndexEntry + { + ContentId = @event.ContentId, + DocId = state.DocIdNew, + ServeAll = true, + ServePublished = false, + Texts = data.ToTexts() + }); + + Index(@event, + new UpdateIndexEntry + { + DocId = state.DocIdCurrent, + ServeAll = false, + ServePublished = true + }); + } + else + { + var isPublished = state.DocIdCurrent == state.DocIdForPublished; + + Index(@event, + new UpsertIndexEntry + { + ContentId = @event.ContentId, + DocId = state.DocIdCurrent, + ServeAll = true, + ServePublished = isPublished, + Texts = data.ToTexts() + }); + } + } + } - if (state != null && state.DocIdForPublished != null) + private void Publish(ContentEvent @event) { - await IndexAsync(@event, - new UpdateIndexEntry + if (states.TryGetValue(@event.ContentId, out var state)) + { + if (state.DocIdNew != null) { - DocId = state.DocIdForPublished, - ServeAll = true, - ServePublished = false - }); + Index(@event, + new UpdateIndexEntry + { + DocId = state.DocIdNew, + ServeAll = true, + ServePublished = true + }); + + Index(@event, + new DeleteIndexEntry + { + DocId = state.DocIdCurrent + }); + + state.DocIdForPublished = state.DocIdNew; + state.DocIdCurrent = state.DocIdNew; + } + else + { + Index(@event, + new UpdateIndexEntry + { + DocId = state.DocIdCurrent, + ServeAll = true, + ServePublished = true + }); + + state.DocIdForPublished = state.DocIdCurrent; + } - state.DocIdForPublished = null; + state.DocIdNew = null; - await textIndexerState.SetAsync(state); + updates[state.ContentId] = state; + } } - } - private async Task PublishAsync(ContentEvent @event) - { - var state = await textIndexerState.GetAsync(@event.ContentId); - - if (state != null) + private void DeleteDraft(ContentEvent @event) { - if (state.DocIdNew != null) + if (states.TryGetValue(@event.ContentId, out var state) && state.DocIdNew != null) { - await IndexAsync(@event, + Index(@event, new UpdateIndexEntry { - DocId = state.DocIdNew, + DocId = state.DocIdCurrent, ServeAll = true, ServePublished = true - }, + }); + + Index(@event, new DeleteIndexEntry { - DocId = state.DocIdCurrent + DocId = state.DocIdNew }); - state.DocIdForPublished = state.DocIdNew; - state.DocIdCurrent = state.DocIdNew; + state.DocIdNew = null; + + updates[state.ContentId] = state; } - else + } + + private void Delete(ContentEvent @event) + { + if (states.TryGetValue(@event.ContentId, out var state)) { - await IndexAsync(@event, - new UpdateIndexEntry + Index(@event, + new DeleteIndexEntry { - DocId = state.DocIdCurrent, - ServeAll = true, - ServePublished = true + DocId = state.DocIdCurrent }); - state.DocIdForPublished = state.DocIdCurrent; + Index(@event, + new DeleteIndexEntry + { + DocId = state.DocIdNew ?? NotFound + }); + + state.IsDeleted = true; + + updates[state.ContentId] = state; } + } - state.DocIdNew = null; + private void Index(ContentEvent @event, IndexCommand command) + { + command.AppId = @event.AppId; + command.SchemaId = @event.SchemaId; - await textIndexerState.SetAsync(state); + if (command is UpdateIndexEntry update && + commands.TryGetValue(command.DocId, out var existing) && + existing is UpsertIndexEntry upsert) + { + upsert.ServeAll = update.ServeAll; + upsert.ServePublished = update.ServePublished; + } + else + { + commands[command.DocId] = command; + } } } - private async Task DeleteDraftAsync(ContentEvent @event) + public TextIndexingProcess(ITextIndex textIndexer, ITextIndexerState textIndexerState) { - var state = await textIndexerState.GetAsync(@event.ContentId); - - if (state != null && state.DocIdNew != null) - { - await IndexAsync(@event, - new UpdateIndexEntry - { - DocId = state.DocIdCurrent, - ServeAll = true, - ServePublished = true - }, - new DeleteIndexEntry - { - DocId = state.DocIdNew - }); + Guard.NotNull(textIndexer, nameof(textIndexer)); + Guard.NotNull(textIndexerState, nameof(textIndexerState)); - state.DocIdNew = null; + this.textIndex = textIndexer; + this.textIndexerState = textIndexerState; + } - await textIndexerState.SetAsync(state); - } + public async Task ClearAsync() + { + await textIndex.ClearAsync(); + await textIndexerState.ClearAsync(); } - private async Task DeleteAsync(ContentEvent @event) + public async Task On(IEnumerable> @events) { - var state = await textIndexerState.GetAsync(@event.ContentId); + var states = await QueryStatesAsync(events); - if (state != null) - { - await IndexAsync(@event, - new DeleteIndexEntry - { - DocId = state.DocIdCurrent - }, - new DeleteIndexEntry - { - DocId = state.DocIdNew ?? NotFound - }); + var updates = new Updates(states); - await textIndexerState.RemoveAsync(state.ContentId); + foreach (var @event in events) + { + updates.On(@event); } + + await updates.WriteAsync(textIndex, textIndexerState); } - private Task IndexAsync(ContentEvent @event, params IndexCommand[] commands) + private Task> QueryStatesAsync(IEnumerable> events) { - return textIndexer.ExecuteAsync(@event.AppId, @event.SchemaId, commands); + var ids = + events + .Select(x => x.Payload).OfType() + .Select(x => x.ContentId) + .ToHashSet(); + + return textIndexerState.GetAsync(ids); } } } diff --git a/backend/src/Squidex.Domain.Apps.Entities/History/HistoryService.cs b/backend/src/Squidex.Domain.Apps.Entities/History/HistoryService.cs index 81048ab6d..1c7512862 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/History/HistoryService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/History/HistoryService.cs @@ -23,14 +23,19 @@ namespace Squidex.Domain.Apps.Entities.History private readonly IHistoryEventRepository repository; private readonly NotifoService notifo; - public string Name + public int BatchSize { - get { return GetType().Name; } + get { return 1000; } } - public string EventsFilter + public int BatchDelay { - get { return ".*"; } + get { return 1000; } + } + + public string Name + { + get { return GetType().Name; } } public HistoryService(IHistoryEventRepository repository, IEnumerable creators, NotifoService notifo) @@ -54,38 +59,48 @@ namespace Squidex.Domain.Apps.Entities.History this.notifo = notifo; } - public bool Handles(StoredEvent @event) - { - return true; - } - public Task ClearAsync() { return repository.ClearAsync(); } - public async Task On(Envelope @event) + public async Task On(IEnumerable> @events) { - await notifo.HandleEventAsync(@event); + var targets = new List<(Envelope Event, HistoryEvent? HistoryEvent)>(); - foreach (var creator in creators) + foreach (var @event in events) { - var historyEvent = await creator.CreateEventAsync(@event); - - if (historyEvent != null) + if (@event.Payload is AppEvent) { var appEvent = @event.To(); - await notifo.HandleHistoryEventAsync(appEvent, historyEvent); + HistoryEvent? historyEvent = null; + + foreach (var creator in creators) + { + historyEvent = await creator.CreateEventAsync(@event); + + if (historyEvent != null) + { + historyEvent.Actor = appEvent.Payload.Actor; + historyEvent.AppId = appEvent.Payload.AppId.Id; + historyEvent.Created = @event.Headers.Timestamp(); + historyEvent.Version = @event.Headers.EventStreamNumber(); - historyEvent.Actor = appEvent.Payload.Actor; - historyEvent.AppId = appEvent.Payload.AppId.Id; - historyEvent.Created = @event.Headers.Timestamp(); - historyEvent.Version = @event.Headers.EventStreamNumber(); + break; + } + } - await repository.InsertAsync(historyEvent); + targets.Add((appEvent, historyEvent)); } } + + if (targets.Count > 0) + { + await notifo.HandleEventsAsync(targets); + + await repository.InsertManyAsync(targets.NotNull(x => x.HistoryEvent)); + } } public async Task> QueryByChannelAsync(Guid appId, string channelPrefix, int count) diff --git a/backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs b/backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs index 2a97562d5..2acd77b40 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs @@ -5,6 +5,8 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Grpc.Core; @@ -117,32 +119,33 @@ namespace Squidex.Domain.Apps.Entities.History await userResolver.SetClaimAsync(user.Id, SquidexClaimTypes.NotifoKey, response.User.ApiKey); } - public async Task HandleEventAsync(Envelope @event) + public async Task HandleEventsAsync(IEnumerable<(Envelope AppEvent, HistoryEvent? HistoryEvent)> events) { - Guard.NotNull(@event, nameof(@event)); + Guard.NotNull(events, nameof(events)); if (client == null) { return; } - switch (@event.Payload) + var now = clock.GetCurrentInstant(); + + var publishedEvents = events + .Where(x => IsTooOld(x.AppEvent.Headers, now) == false) + .Where(x => IsComment(x.AppEvent.Payload) || x.HistoryEvent != null) + .ToList(); + + if (publishedEvents.Any()) { - case CommentCreated comment: + using (var stream = client.PublishMany()) + { + foreach (var @event in publishedEvents) { - if (IsTooOld(@event.Headers)) - { - return; - } + var payload = @event.AppEvent.Payload; - if (comment.Mentions == null || comment.Mentions.Length == 0) + if (payload is CommentCreated comment && IsComment(payload)) { - break; - } - - using (var stream = client.PublishMany()) - { - foreach (var userId in comment.Mentions) + foreach (var userId in comment.Mentions!) { var publishRequest = new PublishRequest { @@ -164,52 +167,87 @@ namespace Squidex.Domain.Apps.Entities.History await stream.RequestStream.WriteAsync(publishRequest); } - - await stream.RequestStream.CompleteAsync(); - await stream.ResponseAsync; } + else if (@event.HistoryEvent != null) + { + var historyEvent = @event.HistoryEvent; - break; - } + var publishRequest = new PublishRequest + { + AppId = options.AppId + }; - case AppContributorAssigned contributorAssigned: - { - var user = await userResolver.FindByIdAsync(contributorAssigned.ContributorId); + foreach (var (key, value) in historyEvent.Parameters) + { + publishRequest.Properties.Add(key, value); + } - if (user != null) - { - await UpsertUserAsync(user); - } + publishRequest.Properties["SquidexApp"] = payload.AppId.Name; - var request = BuildAllowedTopicRequest(contributorAssigned, contributorAssigned.ContributorId); + if (payload is ContentEvent c && !(payload is ContentDeleted)) + { + var url = urlGenerator.ContentUI(c.AppId, c.SchemaId, c.ContentId); - try - { - await client.AddAllowedTopicAsync(request); - } - catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) - { - break; - } + publishRequest.Properties["SquidexUrl"] = url; + } + + publishRequest.TemplateCode = @event.HistoryEvent.EventType; - break; + SetUser(payload, publishRequest); + SetTopic(payload, publishRequest, historyEvent); + + await stream.RequestStream.WriteAsync(publishRequest); + } } - case AppContributorRemoved contributorRemoved: - { - var request = BuildAllowedTopicRequest(contributorRemoved, contributorRemoved.ContributorId); + await stream.RequestStream.CompleteAsync(); + await stream.ResponseAsync; + } + } - try + foreach (var @event in events) + { + switch (@event.AppEvent.Payload) + { + case AppContributorAssigned contributorAssigned: { - await client.RemoveAllowedTopicAsync(request); + var user = await userResolver.FindByIdAsync(contributorAssigned.ContributorId); + + if (user != null) + { + await UpsertUserAsync(user); + } + + var request = BuildAllowedTopicRequest(contributorAssigned, contributorAssigned.ContributorId); + + try + { + await client.AddAllowedTopicAsync(request); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + { + break; + } + + break; } - catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + + case AppContributorRemoved contributorRemoved: { + var request = BuildAllowedTopicRequest(contributorRemoved, contributorRemoved.ContributorId); + + try + { + await client.RemoveAllowedTopicAsync(request); + } + catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound) + { + break; + } + break; } - - break; - } + } } } @@ -226,52 +264,14 @@ namespace Squidex.Domain.Apps.Entities.History return topicRequest; } - public async Task HandleHistoryEventAsync(Envelope @event, HistoryEvent historyEvent) + private static bool IsTooOld(EnvelopeHeaders headers, Instant now) { - if (client == null) - { - return; - } - - if (IsTooOld(@event.Headers)) - { - return; - } - - var appEvent = @event.Payload; - - var publishRequest = new PublishRequest - { - AppId = options.AppId - }; - - foreach (var (key, value) in historyEvent.Parameters) - { - publishRequest.Properties.Add(key, value); - } - - publishRequest.Properties["SquidexApp"] = appEvent.AppId.Name; - - if (appEvent is ContentEvent c && !(appEvent is ContentDeleted)) - { - var url = urlGenerator.ContentUI(c.AppId, c.SchemaId, c.ContentId); - - publishRequest.Properties["SquidexUrl"] = url; - } - - publishRequest.TemplateCode = historyEvent.EventType; - - SetUser(appEvent, publishRequest); - SetTopic(appEvent, publishRequest, historyEvent); - - await client.PublishAsync(publishRequest); + return now - headers.Timestamp() > MaxAge; } - private bool IsTooOld(EnvelopeHeaders headers) + private static bool IsComment(AppEvent appEvent) { - var now = clock.GetCurrentInstant(); - - return now - headers.Timestamp() > MaxAge; + return appEvent is CommentCreated comment && comment.Mentions?.Length > 0; } private static void SetUser(AppEvent appEvent, PublishRequest publishRequest) diff --git a/backend/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs b/backend/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs index d9fb04e2b..10fe79e45 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs @@ -15,7 +15,7 @@ namespace Squidex.Domain.Apps.Entities.History.Repositories { Task> QueryByChannelAsync(Guid appId, string channelPrefix, int count); - Task InsertAsync(HistoryEvent item); + Task InsertManyAsync(IEnumerable historyEvents); Task ClearAsync(); } diff --git a/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs b/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs index d55045c6b..aa883e68c 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs +++ b/backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs @@ -33,11 +33,6 @@ namespace Squidex.Domain.Apps.Entities.Rules get { return GetType().Name; } } - public string EventsFilter - { - get { return ".*"; } - } - public RuleEnqueuer(IAppProvider appProvider, IMemoryCache cache, ILocalCache localCache, IRuleEventRepository ruleEventRepository, RuleService ruleService) { @@ -55,16 +50,6 @@ namespace Squidex.Domain.Apps.Entities.Rules this.localCache = localCache; } - public bool Handles(StoredEvent @event) - { - return true; - } - - public Task ClearAsync() - { - return Task.CompletedTask; - } - public async Task Enqueue(Rule rule, Guid ruleId, Envelope @event) { Guard.NotNull(rule, nameof(rule)); diff --git a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj index 34b9440cf..7ee7c2063 100644 --- a/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj +++ b/backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj @@ -25,10 +25,6 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - - - - all diff --git a/backend/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbSubscription.cs b/backend/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbSubscription.cs index 6f9c3b813..02be09f86 100644 --- a/backend/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbSubscription.cs +++ b/backend/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbSubscription.cs @@ -24,10 +24,9 @@ namespace Squidex.Infrastructure.EventSourcing internal sealed class CosmosDbSubscription : IEventSubscription, IChangeFeedObserverFactory, IChangeFeedObserver { private readonly TaskCompletionSource processorStopRequested = new TaskCompletionSource(); - private readonly Task processorTask; private readonly CosmosDbEventStore store; private readonly Regex regex; - private readonly string? hostName; + private readonly string hostName; private readonly IEventSubscriber subscriber; public CosmosDbSubscription(CosmosDbEventStore store, IEventSubscriber subscriber, string? streamFilter, string? position = null) @@ -42,7 +41,7 @@ namespace Squidex.Infrastructure.EventSourcing } else { - hostName = position; + hostName = position ?? "none"; } if (!StreamFilter.IsAll(streamFilter)) @@ -52,7 +51,7 @@ namespace Squidex.Infrastructure.EventSourcing this.subscriber = subscriber; - processorTask = Task.Run(async () => + Task.Run(async () => { try { @@ -128,7 +127,7 @@ namespace Squidex.Infrastructure.EventSourcing var eventData = @event.ToEventData(); - await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName ?? "None", eventStreamOffset, eventData)); + await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName, eventStreamOffset, eventData)); } } } @@ -136,15 +135,9 @@ namespace Squidex.Infrastructure.EventSourcing } } - public void WakeUp() - { - } - - public Task StopAsync() + public void Unsubscribe() { processorStopRequested.SetResult(true); - - return processorTask; } } } diff --git a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStoreSubscription.cs b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStoreSubscription.cs index 0d73af252..afcdafe94 100644 --- a/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStoreSubscription.cs +++ b/backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStoreSubscription.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.Threading.Tasks; using EventStore.ClientAPI; using EventStore.ClientAPI.Exceptions; using Squidex.Infrastructure.Json; @@ -44,15 +43,9 @@ namespace Squidex.Infrastructure.EventSourcing subscription = SubscribeToStream(streamName); } - public Task StopAsync() + public void Unsubscribe() { subscription.Stop(); - - return Task.CompletedTask; - } - - public void WakeUp() - { } private EventStoreCatchUpSubscription SubscribeToStream(string streamName) diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStoreSubscription.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStoreSubscription.cs index 136ed13a0..cdaae9def 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStoreSubscription.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStoreSubscription.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using MongoDB.Bson; using MongoDB.Driver; using NodaTime; +using Squidex.Infrastructure.Tasks; namespace Squidex.Infrastructure.EventSourcing { @@ -19,14 +20,13 @@ namespace Squidex.Infrastructure.EventSourcing private readonly MongoEventStore eventStore; private readonly IEventSubscriber eventSubscriber; private readonly CancellationTokenSource stopToken = new CancellationTokenSource(); - private readonly Task task; public MongoEventStoreSubscription(MongoEventStore eventStore, IEventSubscriber eventSubscriber, string? streamFilter, string? position) { this.eventStore = eventStore; this.eventSubscriber = eventSubscriber; - task = QueryAsync(streamFilter, position); + QueryAsync(streamFilter, position).Forget(); } private async Task QueryAsync(string? streamFilter, string? position) @@ -155,15 +155,9 @@ namespace Squidex.Infrastructure.EventSourcing return result; } - public Task StopAsync() + public void Unsubscribe() { stopToken.Cancel(); - - return task; - } - - public void WakeUp() - { } } } diff --git a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs index e2ba2fd03..fc8def0b5 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs @@ -138,7 +138,7 @@ namespace Squidex.Infrastructure.EventSourcing { using (Profiler.TraceMethod()) { - await Collection.Find(filter, options: Batching.Options).Sort(Sort.Ascending(TimestampField)).ForEachPipelineAsync(async commit => + await Collection.Find(filter, options: Batching.Options).Sort(Sort.Ascending(TimestampField)).ForEachPipedAsync(async commit => { foreach (var @event in commit.Filtered(position, predicate)) { diff --git a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs index f6e45c1fb..ac190e8f1 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs @@ -163,15 +163,15 @@ namespace Squidex.Infrastructure.MongoDb return BsonClassMap.LookupClassMap(typeof(TEntity)).GetMemberMap(nameof(IVersionedEntity.Version)).ElementName; } - public static async Task ForEachPipelineAsync(this IAsyncCursorSource source, Func processor, CancellationToken cancellationToken = default) + public static async Task ForEachPipedAsync(this IAsyncCursorSource source, Func processor, CancellationToken cancellationToken = default) { using (var cursor = await source.ToCursorAsync(cancellationToken)) { - await cursor.ForEachPipelineAsync(processor, cancellationToken); + await cursor.ForEachPipedAsync(processor, cancellationToken); } } - public static async Task ForEachPipelineAsync(this IAsyncCursor source, Func processor, CancellationToken cancellationToken = default) + public static async Task ForEachPipedAsync(this IAsyncCursor source, Func processor, CancellationToken cancellationToken = default) { using (var selfToken = new CancellationTokenSource()) { diff --git a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs index d235d7d85..4cdb44e92 100644 --- a/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs +++ b/backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs @@ -71,7 +71,7 @@ namespace Squidex.Infrastructure.States { using (Profiler.TraceMethod>()) { - await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(x.Doc, x.Version), ct); + await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(x.Doc, x.Version), ct); } } diff --git a/backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs b/backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs index 4075f8dc1..afb56c463 100644 --- a/backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs +++ b/backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs @@ -80,16 +80,6 @@ namespace Squidex.Infrastructure.CQRS.Events } } - public bool Handles(StoredEvent @event) - { - return true; - } - - public Task ClearAsync() - { - return Task.CompletedTask; - } - public Task On(Envelope @event) { var jsonString = jsonSerializer.Serialize(@event); diff --git a/backend/src/Squidex.Infrastructure/CollectionExtensions.cs b/backend/src/Squidex.Infrastructure/CollectionExtensions.cs index a69c38716..c25e56fca 100644 --- a/backend/src/Squidex.Infrastructure/CollectionExtensions.cs +++ b/backend/src/Squidex.Infrastructure/CollectionExtensions.cs @@ -85,6 +85,11 @@ namespace Squidex.Infrastructure return source.Where(x => x != null)!; } + public static IEnumerable NotNull(this IEnumerable source, Func selector) where TOut : class + { + return source.Select(selector).Where(x => x != null)!; + } + public static IEnumerable Concat(this IEnumerable source, T value) { return source.Concat(Enumerable.Repeat(value, 1)); diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/CompoundEventConsumer.cs b/backend/src/Squidex.Infrastructure/EventSourcing/CompoundEventConsumer.cs deleted file mode 100644 index 974711988..000000000 --- a/backend/src/Squidex.Infrastructure/EventSourcing/CompoundEventConsumer.cs +++ /dev/null @@ -1,77 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschränkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.Linq; -using System.Threading.Tasks; - -namespace Squidex.Infrastructure.EventSourcing -{ - public sealed class CompoundEventConsumer : IEventConsumer - { - private readonly IEventConsumer[] inners; - - public string Name { get; } - - public string EventsFilter { get; } - - public CompoundEventConsumer(IEventConsumer first, params IEventConsumer[] inners) - : this(first?.Name!, first!, inners) - { - } - - public CompoundEventConsumer(IEventConsumer[] inners) - { - Guard.NotNull(inners, nameof(inners)); - Guard.NotEmpty(inners, nameof(inners)); - - this.inners = inners; - - Name = inners.First().Name; - - var innerFilters = - this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter)) - .Select(x => $"({x.EventsFilter})"); - - EventsFilter = string.Join("|", innerFilters); - } - - public CompoundEventConsumer(string name, IEventConsumer first, params IEventConsumer[] inners) - { - Guard.NotNull(first, nameof(first)); - Guard.NotNull(inners, nameof(inners)); - Guard.NotNullOrEmpty(name, nameof(name)); - - this.inners = new[] { first }.Union(inners).ToArray(); - - Name = name; - - var innerFilters = - this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter)) - .Select(x => $"({x.EventsFilter})"); - - EventsFilter = string.Join("|", innerFilters); - } - - public bool Handles(StoredEvent @event) - { - return inners.Any(x => x.Handles(@event)); - } - - public Task ClearAsync() - { - return Task.WhenAll(inners.Select(i => i.ClearAsync())); - } - - public async Task On(Envelope @event) - { - foreach (var inner in inners) - { - await inner.On(@event); - } - } - } -} diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs new file mode 100644 index 000000000..01ff81ab3 --- /dev/null +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs @@ -0,0 +1,193 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; +using Squidex.Infrastructure.Reflection; +using Squidex.Infrastructure.Tasks; + +namespace Squidex.Infrastructure.EventSourcing.Grains +{ + internal sealed class BatchSubscriber : IEventSubscriber + { + private readonly ITargetBlock pipelineStart; + private readonly IEventDataFormatter eventDataFormatter; + private readonly IEventSubscription eventSubscription; + private readonly IDataflowBlock pipelineEnd; + + private sealed class Job + { + public StoredEvent? StoredEvent { get; set; } + + public Exception? Exception { get; set; } + + public Envelope? Event { get; set; } + + public bool ShouldHandle { get; set; } + + public object Sender { get; set; } + } + + public BatchSubscriber( + EventConsumerGrain grain, + IEventDataFormatter eventDataFormatter, + IEventConsumer eventConsumer, + Func factory, + TaskScheduler scheduler) + { + this.eventDataFormatter = eventDataFormatter; + + var batchSize = Math.Max(1, eventConsumer!.BatchSize); + var batchDelay = Math.Max(100, eventConsumer.BatchDelay); + + var parse = new TransformBlock(job => + { + if (job.StoredEvent != null) + { + job.ShouldHandle = eventConsumer.Handles(job.StoredEvent); + } + + if (job.ShouldHandle) + { + try + { + job.Event = ParseKnownEvent(job.StoredEvent!); + } + catch (Exception ex) + { + job.Exception = ex; + } + } + + return job; + }, new ExecutionDataflowBlockOptions + { + BoundedCapacity = batchSize, + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1 + }); + + var buffer = AsyncHelper.CreateBatchBlock(batchSize, batchDelay, new GroupingDataflowBlockOptions + { + BoundedCapacity = batchSize * 2 + }); + + var handle = new ActionBlock>(async jobs => + { + foreach (var jobsBySender in jobs.GroupBy(x => x.Sender)) + { + var sender = jobsBySender.Key; + + if (ReferenceEquals(sender, eventSubscription.Sender)) + { + var exception = jobs.FirstOrDefault(x => x.Exception != null)?.Exception; + + if (exception != null) + { + await grain.OnErrorAsync(exception); + } + else + { + await grain.OnEventsAsync(GetEvents(jobsBySender), GetPosition(jobsBySender)); + } + } + } + }, + new ExecutionDataflowBlockOptions + { + BoundedCapacity = 2, + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = 1, + TaskScheduler = scheduler + }); + + parse.LinkTo(buffer, new DataflowLinkOptions + { + PropagateCompletion = true + }); + + buffer.LinkTo(handle, new DataflowLinkOptions + { + PropagateCompletion = true + }); + + pipelineStart = parse; + pipelineEnd = handle; + + eventSubscription = factory(this); + } + + private static List> GetEvents(IEnumerable jobsBySender) + { + return jobsBySender.NotNull(x => x.Event).ToList(); + } + + private static string GetPosition(IEnumerable jobsBySender) + { + return jobsBySender.Last().StoredEvent!.EventPosition; + } + + public Task CompleteAsync() + { + pipelineStart.Complete(); + + return pipelineEnd.Completion; + } + + public void WakeUp() + { + eventSubscription.WakeUp(); + } + + public void Unsubscribe() + { + eventSubscription.Unsubscribe(); + } + + private Envelope? ParseKnownEvent(StoredEvent storedEvent) + { + try + { + var @event = eventDataFormatter.Parse(storedEvent.Data); + + @event.SetEventPosition(storedEvent.EventPosition); + @event.SetEventStreamNumber(storedEvent.EventStreamNumber); + + return @event; + } + catch (TypeNameNotFoundException) + { + return null; + } + } + + public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) + { + var job = new Job + { + Sender = subscription, + StoredEvent = storedEvent + }; + + return pipelineStart.SendAsync(job); + } + + public Task OnErrorAsync(IEventSubscription subscription, Exception exception) + { + var job = new Job + { + Sender = subscription, + Exception = exception + }; + + return pipelineStart.SendAsync(job); + } + } +} diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs index 0321a2d61..84cf9fd80 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs @@ -6,15 +6,13 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Orleans; using Orleans.Concurrency; using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; -using Squidex.Infrastructure.Reflection; -using Squidex.Infrastructure.Tasks; namespace Squidex.Infrastructure.EventSourcing.Grains { @@ -26,7 +24,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains private readonly IEventStore eventStore; private readonly ISemanticLog log; private TaskScheduler? scheduler; - private IEventSubscription? currentSubscription; + private BatchSubscriber? currentSubscriber; private IEventConsumer? eventConsumer; private EventConsumerState State @@ -65,6 +63,14 @@ namespace Squidex.Infrastructure.EventSourcing.Grains return Task.CompletedTask; } + public async Task CompleteAsync() + { + if (currentSubscriber != null) + { + await currentSubscriber.CompleteAsync(); + } + } + public Task> GetStateAsync() { return Task.FromResult(CreateInfo()); @@ -75,41 +81,23 @@ namespace Squidex.Infrastructure.EventSourcing.Grains return State.ToInfo(eventConsumer!.Name).AsImmutable(); } - public Task OnEventAsync(Immutable subscription, Immutable storedEvent) + public Task OnEventsAsync(IReadOnlyList> events, string position) { - if (subscription.Value != currentSubscription) - { - return Task.CompletedTask; - } - return DoAndUpdateStateAsync(async () => { - if (eventConsumer!.Handles(storedEvent.Value)) - { - var @event = ParseKnownEvent(storedEvent.Value); - - if (@event != null) - { - await DispatchConsumerAsync(@event); - } - } + await DispatchAsync(events); - State = State.Handled(storedEvent.Value.EventPosition); + State = State.Handled(position, events.Count); }); } - public Task OnErrorAsync(Immutable subscription, Immutable exception) + public Task OnErrorAsync(Exception exception) { - if (subscription.Value != currentSubscription) - { - return Task.CompletedTask; - } - return DoAndUpdateStateAsync(() => { Unsubscribe(); - State = State.Stopped(exception.Value); + State = State.Stopped(exception); }); } @@ -119,14 +107,14 @@ namespace Squidex.Infrastructure.EventSourcing.Grains { await DoAndUpdateStateAsync(() => { - Subscribe(State.Position); + Subscribe(); State = State.Started(); }); } else if (!State.IsStopped) { - Subscribe(State.Position); + Subscribe(); } } @@ -139,7 +127,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await DoAndUpdateStateAsync(() => { - Subscribe(State.Position); + Subscribe(); State = State.Started(); }); @@ -172,21 +160,36 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await ClearAsync(); - Subscribe(null); - State = State.Reset(); + + Subscribe(); }); return CreateInfo(); } + private async Task DispatchAsync(IReadOnlyList> events) + { + if (events.Count > 0) + { + await eventConsumer!.On(events); + } + } + private Task DoAndUpdateStateAsync(Action action, [CallerMemberName] string? caller = null) { - return DoAndUpdateStateAsync(() => { action(); return Task.CompletedTask; }, caller); + return DoAndUpdateStateAsync(() => + { + action(); + + return Task.CompletedTask; + }, caller); } private async Task DoAndUpdateStateAsync(Func action, [CallerMemberName] string? caller = null) { + var previousState = State; + try { await action(); @@ -207,10 +210,13 @@ namespace Squidex.Infrastructure.EventSourcing.Grains .WriteProperty("status", "Failed") .WriteProperty("eventConsumer", eventConsumer!.Name)); - State = State.Stopped(ex); + State = previousState.Stopped(ex); } - await state.WriteAsync(); + if (State != previousState) + { + await state.WriteAsync(); + } } private async Task ClearAsync() @@ -233,92 +239,43 @@ namespace Squidex.Infrastructure.EventSourcing.Grains } } - private async Task DispatchConsumerAsync(Envelope @event) - { - var eventId = @event.Headers.EventId().ToString(); - var eventType = @event.Payload.GetType().Name; - - var logContext = (eventId, eventType, consumer: eventConsumer!.Name); - - log.LogDebug(logContext, (ctx, w) => w - .WriteProperty("action", "HandleEvent") - .WriteProperty("actionId", ctx.eventId) - .WriteProperty("status", "Started") - .WriteProperty("eventId", ctx.eventId) - .WriteProperty("eventType", ctx.eventType) - .WriteProperty("eventConsumer", ctx.consumer)); - - using (log.MeasureInformation(logContext, (ctx, w) => w - .WriteProperty("action", "HandleEvent") - .WriteProperty("actionId", ctx.eventId) - .WriteProperty("status", "Completed") - .WriteProperty("eventId", ctx.eventId) - .WriteProperty("eventType", ctx.eventType) - .WriteProperty("eventConsumer", ctx.consumer))) - { - await eventConsumer.On(@event); - } - } - private void Unsubscribe() { - var subscription = Interlocked.Exchange(ref currentSubscription, null); + var subscription = Interlocked.Exchange(ref currentSubscriber, null); - if (subscription != null) - { - subscription.StopAsync().Forget(); - } + subscription?.Unsubscribe(); } - private void Subscribe(string? position) + private void Subscribe() { - if (currentSubscription == null) + if (currentSubscriber == null) { - currentSubscription = CreateSubscription(eventConsumer!.EventsFilter, position); + currentSubscriber = CreateSubscription(); } else { - currentSubscription.WakeUp(); + currentSubscriber.WakeUp(); } } - private Envelope? ParseKnownEvent(StoredEvent storedEvent) - { - try - { - var @event = eventDataFormatter.Parse(storedEvent.Data); - - @event.SetEventPosition(storedEvent.EventPosition); - @event.SetEventStreamNumber(storedEvent.EventStreamNumber); - - return @event; - } - catch (TypeNameNotFoundException) - { - log.LogDebug(w => w.WriteProperty("oldEventFound", storedEvent.Data.Type)); - - return null; - } - } - - protected virtual IEventConsumerGrain GetSelf() + protected virtual TaskScheduler GetScheduler() { - return this.AsReference(); + return scheduler!; } - protected virtual IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string filter, string? position) + private BatchSubscriber CreateSubscription() { - return new RetrySubscription(store, subscriber, filter, position); + return new BatchSubscriber(this, eventDataFormatter, eventConsumer!, CreateRetrySubscription, GetScheduler()); } - protected virtual TaskScheduler GetScheduler() + protected virtual IEventSubscription CreateRetrySubscription(IEventSubscriber subscriber) { - return scheduler!; + return new RetrySubscription(subscriber, CreateSubscription); } - private IEventSubscription CreateSubscription(string streamFilter, string? position) + protected virtual IEventSubscription CreateSubscription(IEventSubscriber subscriber) { - return CreateSubscription(eventStore, new WrapperSubscription(GetSelf(), GetScheduler()), streamFilter, position); + return eventStore.CreateSubscription(subscriber, eventConsumer!.EventsFilter, State.Position); } } } \ No newline at end of file diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerState.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerState.cs index f772a0a3b..acddcbec8 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerState.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerState.cs @@ -46,9 +46,9 @@ namespace Squidex.Infrastructure.EventSourcing.Grains return new EventConsumerState(); } - public EventConsumerState Handled(string position) + public EventConsumerState Handled(string position, int offset = 1) { - return new EventConsumerState(position, Count + 1); + return new EventConsumerState(position, Count + offset); } public EventConsumerState Stopped(Exception? ex = null) diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/IEventConsumerGrain.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/IEventConsumerGrain.cs index fb7d82811..590daea27 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/IEventConsumerGrain.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/IEventConsumerGrain.cs @@ -5,7 +5,6 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System; using System.Threading.Tasks; using Orleans.Concurrency; using Squidex.Infrastructure.Orleans; @@ -21,9 +20,5 @@ namespace Squidex.Infrastructure.EventSourcing.Grains Task> StartAsync(); Task> ResetAsync(); - - Task OnEventAsync(Immutable subscription, Immutable storedEvent); - - Task OnErrorAsync(Immutable subscription, Immutable exception); } } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/WrapperSubscription.cs b/backend/src/Squidex.Infrastructure/EventSourcing/Grains/WrapperSubscription.cs deleted file mode 100644 index 6862a1504..000000000 --- a/backend/src/Squidex.Infrastructure/EventSourcing/Grains/WrapperSubscription.cs +++ /dev/null @@ -1,42 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschränkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading; -using System.Threading.Tasks; -using Orleans.Concurrency; - -namespace Squidex.Infrastructure.EventSourcing.Grains -{ - internal sealed class WrapperSubscription : IEventSubscriber - { - private readonly IEventConsumerGrain grain; - private readonly TaskScheduler scheduler; - - public WrapperSubscription(IEventConsumerGrain grain, TaskScheduler scheduler) - { - this.grain = grain; - - this.scheduler = scheduler ?? TaskScheduler.Default; - } - - public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) - { - return Dispatch(() => grain.OnEventAsync(subscription.AsImmutable(), storedEvent.AsImmutable())); - } - - public Task OnErrorAsync(IEventSubscription subscription, Exception exception) - { - return Dispatch(() => grain.OnErrorAsync(subscription.AsImmutable(), exception.AsImmutable())); - } - - private Task Dispatch(Func task) - { - return Task.Factory.StartNew(task, CancellationToken.None, TaskCreationOptions.None, scheduler).Unwrap(); - } - } -} diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs b/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs index 2822c744d..078cc039a 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs @@ -5,6 +5,7 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System.Collections.Generic; using System.Threading.Tasks; namespace Squidex.Infrastructure.EventSourcing @@ -13,14 +14,35 @@ namespace Squidex.Infrastructure.EventSourcing public interface IEventConsumer { + int BatchDelay => 500; + + int BatchSize => 1; + string Name { get; } - string EventsFilter { get; } + string EventsFilter => ".*"; + + bool Handles(StoredEvent @event) + { + return true; + } - bool Handles(StoredEvent @event); + Task ClearAsync() + { + return Task.CompletedTask; + } - Task ClearAsync(); + Task On(Envelope @event) + { + return Task.CompletedTask; + } - Task On(Envelope @event); + async Task On(IEnumerable> @events) + { + foreach (var @event in events) + { + await On(@event); + } + } } } \ No newline at end of file diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/IEventSubscription.cs b/backend/src/Squidex.Infrastructure/EventSourcing/IEventSubscription.cs index 48ead1da9..9eb3d732d 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/IEventSubscription.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/IEventSubscription.cs @@ -5,14 +5,18 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== -using System.Threading.Tasks; - namespace Squidex.Infrastructure.EventSourcing { public interface IEventSubscription { - void WakeUp(); + object? Sender => this; + + void Unsubscribe() + { + } - Task StopAsync(); + void WakeUp() + { + } } } \ No newline at end of file diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/PollingSubscription.cs b/backend/src/Squidex.Infrastructure/EventSourcing/PollingSubscription.cs index 8cf9a7fb3..401ca6ce4 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/PollingSubscription.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/PollingSubscription.cs @@ -6,7 +6,7 @@ // ========================================================================== using System; -using System.Threading.Tasks; +using Squidex.Infrastructure.Tasks; using Squidex.Infrastructure.Timers; namespace Squidex.Infrastructure.EventSourcing @@ -47,9 +47,9 @@ namespace Squidex.Infrastructure.EventSourcing timer.SkipCurrentDelay(); } - public Task StopAsync() + public void Unsubscribe() { - return timer.StopAsync(); + timer.StopAsync().Forget(); } } } diff --git a/backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs b/backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs index 3ee7e9f49..1c52d9e42 100644 --- a/backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs +++ b/backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs @@ -8,7 +8,6 @@ using System; using System.Threading; using System.Threading.Tasks; -using Squidex.Infrastructure.Tasks; #pragma warning disable RECS0002 // Convert anonymous method to method group @@ -16,109 +15,78 @@ namespace Squidex.Infrastructure.EventSourcing { public sealed class RetrySubscription : IEventSubscription, IEventSubscriber { - private readonly SingleThreadedDispatcher dispatcher = new SingleThreadedDispatcher(10); - private readonly CancellationTokenSource timerCancellation = new CancellationTokenSource(); private readonly RetryWindow retryWindow = new RetryWindow(TimeSpan.FromMinutes(5), 5); - private readonly IEventStore eventStore; private readonly IEventSubscriber eventSubscriber; - private readonly string? streamFilter; + private readonly Func eventSubscriptionFactory; + private CancellationTokenSource timerCancellation = new CancellationTokenSource(); private IEventSubscription? currentSubscription; - private string? position; public int ReconnectWaitMs { get; set; } = 5000; - public RetrySubscription(IEventStore eventStore, IEventSubscriber eventSubscriber, string? streamFilter, string? position) + public object? Sender => currentSubscription?.Sender; + + public RetrySubscription(IEventSubscriber eventSubscriber, Func eventSubscriptionFactory) { - Guard.NotNull(eventStore, nameof(eventStore)); Guard.NotNull(eventSubscriber, nameof(eventSubscriber)); - Guard.NotNull(streamFilter, nameof(streamFilter)); - - this.position = position; + Guard.NotNull(eventSubscriptionFactory, nameof(eventSubscriptionFactory)); - this.eventStore = eventStore; this.eventSubscriber = eventSubscriber; - - this.streamFilter = streamFilter; + this.eventSubscriptionFactory = eventSubscriptionFactory; Subscribe(); } private void Subscribe() { - if (currentSubscription == null) - { - currentSubscription = eventStore.CreateSubscription(this, streamFilter, position); - } - } - - private void Unsubscribe() - { - var subscription = Interlocked.Exchange(ref currentSubscription, null); - - if (subscription != null) + lock (this) { - subscription.StopAsync().Forget(); + if (currentSubscription == null) + { + currentSubscription = eventSubscriptionFactory(this); + } } } - public void WakeUp() - { - currentSubscription?.WakeUp(); - } - - private async Task HandleEventAsync(IEventSubscription subscription, StoredEvent storedEvent) + public void Unsubscribe() { - if (subscription == currentSubscription) + lock (this) { - await eventSubscriber.OnEventAsync(this, storedEvent); - - position = storedEvent.EventPosition; - } - } + if (currentSubscription != null) + { + timerCancellation.Cancel(); + timerCancellation.Dispose(); - private async Task HandleErrorAsync(IEventSubscription subscription, Exception exception) - { - if (subscription == currentSubscription) - { - Unsubscribe(); + currentSubscription.Unsubscribe(); + currentSubscription = null; - if (retryWindow.CanRetryAfterFailure()) - { - RetryAsync().Forget(); - } - else - { - await eventSubscriber.OnErrorAsync(this, exception); + timerCancellation = new CancellationTokenSource(); } } } - private async Task RetryAsync() + public void WakeUp() { - await Task.Delay(ReconnectWaitMs, timerCancellation.Token); - - await dispatcher.DispatchAsync(Subscribe); + currentSubscription?.WakeUp(); } - Task IEventSubscriber.OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) + public async Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) { - return dispatcher.DispatchAsync(() => HandleEventAsync(subscription, storedEvent)); + await eventSubscriber.OnEventAsync(subscription, storedEvent); } - Task IEventSubscriber.OnErrorAsync(IEventSubscription subscription, Exception exception) + public async Task OnErrorAsync(IEventSubscription subscription, Exception exception) { - return dispatcher.DispatchAsync(() => HandleErrorAsync(subscription, exception)); - } + Unsubscribe(); - public async Task StopAsync() - { - await dispatcher.DispatchAsync(Unsubscribe); - await dispatcher.StopAndWaitAsync(); + if (retryWindow.CanRetryAfterFailure()) + { + await Task.Delay(ReconnectWaitMs, timerCancellation.Token); - if (!timerCancellation.IsCancellationRequested) + Subscribe(); + } + else { - timerCancellation.Cancel(); - timerCancellation.Dispose(); + await eventSubscriber.OnErrorAsync(subscription, exception); } } } diff --git a/backend/src/Squidex.Infrastructure/Tasks/AsyncHelper.cs b/backend/src/Squidex.Infrastructure/Tasks/AsyncHelper.cs index ce347c372..236a15859 100644 --- a/backend/src/Squidex.Infrastructure/Tasks/AsyncHelper.cs +++ b/backend/src/Squidex.Infrastructure/Tasks/AsyncHelper.cs @@ -8,6 +8,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using System.Threading.Tasks.Dataflow; namespace Squidex.Infrastructure.Tasks { @@ -50,5 +51,37 @@ namespace Squidex.Infrastructure.Tasks .GetAwaiter() .GetResult(); } + + public static IPropagatorBlock CreateBatchBlock(int batchSize, int timeout, GroupingDataflowBlockOptions? dataflowBlockOptions = null) + { + dataflowBlockOptions ??= new GroupingDataflowBlockOptions(); + + var batchBlock = new BatchBlock(batchSize, dataflowBlockOptions); + + var timer = new Timer(_ => batchBlock.TriggerBatch()); + + var timerBlock = new TransformBlock((T value) => + { + timer.Change(timeout, Timeout.Infinite); + + return value; + }, new ExecutionDataflowBlockOptions() + { + BoundedCapacity = 1, + CancellationToken = dataflowBlockOptions.CancellationToken, + EnsureOrdered = dataflowBlockOptions.EnsureOrdered, + MaxDegreeOfParallelism = 1, + MaxMessagesPerTask = dataflowBlockOptions.MaxMessagesPerTask, + NameFormat = dataflowBlockOptions.NameFormat, + TaskScheduler = dataflowBlockOptions.TaskScheduler + }); + + timerBlock.LinkTo(batchBlock, new DataflowLinkOptions() + { + PropagateCompletion = true + }); + + return DataflowBlock.Encapsulate(timerBlock, batchBlock); + } } } diff --git a/backend/src/Squidex.Infrastructure/Tasks/SingleThreadedDispatcher.cs b/backend/src/Squidex.Infrastructure/Tasks/SingleThreadedDispatcher.cs deleted file mode 100644 index c37997e25..000000000 --- a/backend/src/Squidex.Infrastructure/Tasks/SingleThreadedDispatcher.cs +++ /dev/null @@ -1,67 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschränkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using System.Threading.Tasks.Dataflow; - -namespace Squidex.Infrastructure.Tasks -{ - public sealed class SingleThreadedDispatcher - { - private readonly ActionBlock> block; - private bool isStopped; - - public SingleThreadedDispatcher(int capacity = 1) - { - var options = new ExecutionDataflowBlockOptions - { - BoundedCapacity = capacity, - MaxMessagesPerTask = 1, - MaxDegreeOfParallelism = 1 - }; - - block = new ActionBlock>(Handle, options); - } - - public Task DispatchAsync(Func action) - { - Guard.NotNull(action, nameof(action)); - - return block.SendAsync(action); - } - - public Task DispatchAsync(Action action) - { - Guard.NotNull(action, nameof(action)); - - return block.SendAsync(() => { action(); return Task.CompletedTask; }); - } - - public async Task StopAndWaitAsync() - { - await DispatchAsync(() => - { - isStopped = true; - - block.Complete(); - }); - - await block.Completion; - } - - private Task Handle(Func action) - { - if (isStopped) - { - return Task.CompletedTask; - } - - return action(); - } - } -} diff --git a/backend/src/Squidex.Infrastructure/Translations/ResourcesLocalizer.cs b/backend/src/Squidex.Infrastructure/Translations/ResourcesLocalizer.cs index 5b4d3c59a..d785758c5 100644 --- a/backend/src/Squidex.Infrastructure/Translations/ResourcesLocalizer.cs +++ b/backend/src/Squidex.Infrastructure/Translations/ResourcesLocalizer.cs @@ -113,11 +113,16 @@ namespace Squidex.Infrastructure.Translations { try { - variableValue = Convert.ToString(property.GetValue(args), culture); + var value = property.GetValue(args); + + if (value != null) + { + variableValue = Convert.ToString(value, culture) ?? variableName; + } } catch { - variableValue = null; + variableValue = variableName; } } diff --git a/backend/src/Squidex/Config/Domain/ContentsServices.cs b/backend/src/Squidex/Config/Domain/ContentsServices.cs index f3498fb66..122c98e7c 100644 --- a/backend/src/Squidex/Config/Domain/ContentsServices.cs +++ b/backend/src/Squidex/Config/Domain/ContentsServices.cs @@ -13,7 +13,7 @@ using Squidex.Domain.Apps.Entities.Contents; using Squidex.Domain.Apps.Entities.Contents.Queries; using Squidex.Domain.Apps.Entities.Contents.Queries.Steps; using Squidex.Domain.Apps.Entities.Contents.Text; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; +using Squidex.Domain.Apps.Entities.Contents.Text.Elastic; using Squidex.Domain.Apps.Entities.Contents.Validation; using Squidex.Domain.Apps.Entities.History; using Squidex.Domain.Apps.Entities.Search; @@ -86,20 +86,27 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .AsOptional(); - services.AddSingletonAs() - .As(); - services.AddSingletonAs() .As(); services.AddSingletonAs() .As(); - services.AddSingletonAs() - .AsSelf(); - services.AddSingletonAs>() .AsSelf(); + + config.ConfigureByOption("fullText:type", new Alternatives + { + ["Elastic"] = () => + { + var elasticConfiguration = config.GetRequiredValue("fullText:elastic:configuration"); + var elasticIndexName = config.GetRequiredValue("fullText:elastic:indexName"); + + services.AddSingletonAs(c => new ElasticSearchTextIndex(elasticConfiguration, elasticIndexName)) + .As(); + }, + ["Default"] = () => { } + }); } } } \ No newline at end of file diff --git a/backend/src/Squidex/Config/Domain/StoreServices.cs b/backend/src/Squidex/Config/Domain/StoreServices.cs index ff70fdbab..bd1dfe18d 100644 --- a/backend/src/Squidex/Config/Domain/StoreServices.cs +++ b/backend/src/Squidex/Config/Domain/StoreServices.cs @@ -14,12 +14,11 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Migrations.Migrations.MongoDb; using MongoDB.Driver; -using MongoDB.Driver.GridFS; using Squidex.Domain.Apps.Entities.Assets.Repositories; using Squidex.Domain.Apps.Entities.Assets.State; using Squidex.Domain.Apps.Entities.Contents.Repositories; using Squidex.Domain.Apps.Entities.Contents.State; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; +using Squidex.Domain.Apps.Entities.Contents.Text; using Squidex.Domain.Apps.Entities.Contents.Text.State; using Squidex.Domain.Apps.Entities.History.Repositories; using Squidex.Domain.Apps.Entities.MongoDb.Assets; @@ -118,6 +117,9 @@ namespace Squidex.Config.Domain services.AddSingletonAs(c => ActivatorUtilities.CreateInstance(c, GetDatabase(c, mongoContentDatabaseName))) .As().As>(); + services.AddSingletonAs() + .AsOptional(); + services.AddSingletonAs() .As(); @@ -128,18 +130,6 @@ namespace Squidex.Config.Domain services.AddSingletonAs() .As(); } - - services.AddSingletonAs(c => - { - var database = c.GetRequiredService(); - - var mongoBucket = new GridFSBucket(database, new GridFSBucketOptions - { - BucketName = "fullText" - }); - - return new MongoIndexStorage(mongoBucket); - }).As(); } }); diff --git a/backend/src/Squidex/appsettings.json b/backend/src/Squidex/appsettings.json index ac96050d3..f60ea555a 100644 --- a/backend/src/Squidex/appsettings.json +++ b/backend/src/Squidex/appsettings.json @@ -23,6 +23,27 @@ "enableXForwardedHost": false }, + "fullText": { + /* + * Define the type of the full text store. + * + * Supported: elastic (ElasticSearch). Default: MongoDB + */ + "type": "default", + "elastic": { + /* + * The configuration to your elastic search cluster. + * + * Read More: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-configuration.html + */ + "configuration": "http://localhost:9200", + /* + * The name of the index. + */ + "indexName": "squidex" + } + }, + /* * Define optional paths to plugins. */ diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs index 6b55daf0a..8c6f8ae29 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs @@ -38,10 +38,28 @@ namespace Squidex.Domain.Apps.Entities.Assets sut = new RecursiveDeleter(commandBus, assetRepository, assetFolderRepository, typeNameRegistry, log); } + [Fact] + public void Should_return_assets_filter_for_events_filter() + { + IEventConsumer consumer = sut; + + Assert.Equal("^assetFolder\\-", consumer.EventsFilter); + } + [Fact] public async Task Should_do_nothing_on_clear() { - await sut.ClearAsync(); + IEventConsumer consumer = sut; + + await consumer.ClearAsync(); + } + + [Fact] + public void Should_return_type_name_for_name() + { + IEventConsumer consumer = sut; + + Assert.Equal(typeof(RecursiveDeleter).Name, consumer.Name); } [Fact] diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs index 0ea5d3574..e02ae0e20 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs @@ -6,9 +6,12 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.Threading.Tasks; using FakeItEasy; using Squidex.Domain.Apps.Entities.Contents.Text.State; +using Squidex.Domain.Apps.Entities.TestHelpers; +using Squidex.Infrastructure; using Xunit; namespace Squidex.Domain.Apps.Entities.Contents.Text @@ -27,19 +30,46 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public async Task Should_retrieve_from_inner_when_not_cached() { var contentId = Guid.NewGuid(); + var contentIds = HashSet.Of(contentId); var state = new TextContentState { ContentId = contentId }; - A.CallTo(() => inner.GetAsync(contentId)) - .Returns(state); + var states = new Dictionary + { + [contentId] = state + }; - var found1 = await sut.GetAsync(contentId); - var found2 = await sut.GetAsync(contentId); + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds))) + .Returns(states); - Assert.Same(state, found1); - Assert.Same(state, found2); + var found1 = await sut.GetAsync(HashSet.Of(contentId)); + var found2 = await sut.GetAsync(HashSet.Of(contentId)); - A.CallTo(() => inner.GetAsync(contentId)) + Assert.Same(state, found1[contentId]); + Assert.Same(state, found2[contentId]); + + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds))) + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_retrieve_from_inner_when_not_cached_and_not_found() + { + var contentId = Guid.NewGuid(); + var contentIds = HashSet.Of(contentId); + + var state = new TextContentState { ContentId = contentId }; + + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds))) + .Returns(new Dictionary()); + + var found1 = await sut.GetAsync(HashSet.Of(contentId)); + var found2 = await sut.GetAsync(HashSet.Of(contentId)); + + Assert.Empty(found1); + Assert.Empty(found2); + + A.CallTo(() => inner.GetAsync(A>.That.Is(contentIds))) .MustHaveHappenedOnceExactly(); } @@ -47,21 +77,25 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public async Task Should_not_retrieve_from_inner_when_cached() { var contentId = Guid.NewGuid(); + var contentIds = HashSet.Of(contentId); var state = new TextContentState { ContentId = contentId }; - await sut.SetAsync(state); + await sut.SetAsync(new List + { + state + }); - var found1 = await sut.GetAsync(contentId); - var found2 = await sut.GetAsync(contentId); + var found1 = await sut.GetAsync(contentIds); + var found2 = await sut.GetAsync(contentIds); - Assert.Same(state, found1); - Assert.Same(state, found2); + Assert.Same(state, found1[contentId]); + Assert.Same(state, found2[contentId]); - A.CallTo(() => inner.SetAsync(state)) + A.CallTo(() => inner.SetAsync(A>.That.IsSameSequenceAs(state))) .MustHaveHappenedOnceExactly(); - A.CallTo(() => inner.GetAsync(contentId)) + A.CallTo(() => inner.GetAsync(A>._)) .MustNotHaveHappened(); } @@ -69,22 +103,30 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public async Task Should_not_retrieve_from_inner_when_removed() { var contentId = Guid.NewGuid(); + var contentIds = HashSet.Of(contentId); var state = new TextContentState { ContentId = contentId }; - await sut.SetAsync(state); - await sut.RemoveAsync(contentId); + await sut.SetAsync(new List + { + state + }); + + await sut.SetAsync(new List + { + new TextContentState { ContentId = contentId, IsDeleted = true } + }); - var found1 = await sut.GetAsync(contentId); - var found2 = await sut.GetAsync(contentId); + var found1 = await sut.GetAsync(contentIds); + var found2 = await sut.GetAsync(contentIds); - Assert.Null(found1); - Assert.Null(found2); + Assert.Empty(found1); + Assert.Empty(found2); - A.CallTo(() => inner.RemoveAsync(contentId)) + A.CallTo(() => inner.SetAsync(A>.That.Matches(x => x.Count == 1 && x[0].IsDeleted))) .MustHaveHappenedOnceExactly(); - A.CallTo(() => inner.GetAsync(contentId)) + A.CallTo(() => inner.GetAsync(A>._)) .MustNotHaveHappened(); } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/DocValuesTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/DocValuesTests.cs deleted file mode 100644 index 0b8c7a419..000000000 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/DocValuesTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using Lucene.Net.Analysis.Standard; -using Lucene.Net.Documents; -using Lucene.Net.Index; -using Lucene.Net.Store; -using Lucene.Net.Util; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; -using Xunit; - -namespace Squidex.Domain.Apps.Entities.Contents.Text -{ - public class DocValuesTests - { - [Fact] - public void Should_read_and_write_doc_values() - { - var version = LuceneVersion.LUCENE_48; - - var indexWriter = - new IndexWriter(new RAMDirectory(), - new IndexWriterConfig(version, new StandardAnalyzer(version))); - - using (indexWriter) - { - for (byte i = 0; i < 255; i++) - { - var document = new Document(); - - document.AddBinaryDocValuesField("field", new BytesRef(new[] { i })); - - indexWriter.AddDocument(document); - } - - indexWriter.Commit(); - - using (var reader = indexWriter.GetReader(true)) - { - var bytesRef = new BytesRef(1); - - for (byte i = 0; i < 255; i++) - { - reader.GetBinaryValue("field", i, bytesRef); - - Assert.Equal(i, bytesRef.Bytes[0]); - } - } - } - } - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/LuceneIndexFactory.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/LuceneIndexFactory.cs deleted file mode 100644 index 02cfa9ce4..000000000 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/LuceneIndexFactory.cs +++ /dev/null @@ -1,50 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using System.Threading.Tasks; -using FakeItEasy; -using Orleans; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; -using Squidex.Infrastructure.Log; - -namespace Squidex.Domain.Apps.Entities.Contents.Text -{ - public sealed class LuceneIndexFactory : IIndexerFactory - { - private readonly IGrainFactory grainFactory = A.Fake(); - private readonly IIndexStorage storage; - private LuceneTextIndexGrain grain; - - public LuceneIndexFactory(IIndexStorage storage) - { - this.storage = storage; - - A.CallTo(() => grainFactory.GetGrain(A._, null)) - .ReturnsLazily(() => grain); - } - - public async Task CreateAsync(Guid schemaId) - { - var indexManager = new IndexManager(storage, A.Fake()); - - grain = new LuceneTextIndexGrain(indexManager); - - await grain.ActivateAsync(schemaId); - - return new LuceneTextIndex(grainFactory, indexManager); - } - - public async Task CleanupAsync() - { - if (grain != null) - { - await grain.OnDeactivateAsync(); - } - } - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TestStorages.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TestStorages.cs deleted file mode 100644 index bc3ea6870..000000000 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TestStorages.cs +++ /dev/null @@ -1,49 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System; -using MongoDB.Driver; -using MongoDB.Driver.GridFS; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; -using Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage; -using Squidex.Domain.Apps.Entities.MongoDb.FullText; -using Squidex.Infrastructure.Assets; - -namespace Squidex.Domain.Apps.Entities.Contents.Text -{ - public static class TestStorages - { - public static IIndexStorage Assets() - { - var storage = new AssetIndexStorage(new MemoryAssetStore()); - - return storage; - } - - public static IIndexStorage TempFolder() - { - var storage = new FileIndexStorage(); - - return storage; - } - - public static IIndexStorage MongoDB() - { - var mongoClient = new MongoClient("mongodb://localhost"); - var mongoDatabase = mongoClient.GetDatabase("FullText"); - - var mongoBucket = new GridFSBucket(mongoDatabase, new GridFSBucketOptions - { - BucketName = $"bucket_{DateTime.UtcNow.Ticks}" - }); - - var storage = new MongoIndexStorage(mongoBucket); - - return storage; - } - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs index e67d9d331..63204b93c 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs @@ -37,49 +37,108 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public abstract IIndexerFactory Factory { get; } + public virtual bool SupportsCleanup { get; set; } = false; + + public virtual bool SupportsSearchSyntax { get; set; } = true; + + public virtual bool SupportsMultiLanguage { get; set; } = true; + public virtual InMemoryTextIndexerState State { get; } = new InMemoryTextIndexerState(); protected TextIndexerTestsBase() { app = - Mocks.App(NamedId.Of(Guid.NewGuid(), "my-app"), + Mocks.App(appId, Language.DE, Language.EN); } - [Fact] + [SkippableFact] public async Task Should_throw_exception_for_invalid_query() { + Skip.IfNot(SupportsSearchSyntax); + await Assert.ThrowsAsync(async () => { await TestCombinations(Search(expected: null, text: "~hello")); }); } - [Fact] - public async Task Should_index_invariant_content_and_retrieve() + [SkippableFact] + public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() { + Skip.IfNot(SupportsSearchSyntax); + await TestCombinations( Create(ids1[0], "iv", "Hello"), Create(ids2[0], "iv", "World"), - Search(expected: ids1, text: "Hello"), - Search(expected: ids2, text: "World"), + Search(expected: ids1, text: "helo~"), + Search(expected: ids2, text: "wold~", SearchScope.All) + ); + } - Search(expected: null, text: "Hello", SearchScope.Published), - Search(expected: null, text: "World", SearchScope.Published) + [SkippableFact] + public async Task Should_search_by_field() + { + Skip.IfNot(SupportsSearchSyntax); + + await TestCombinations( + Create(ids1[0], "en", "City"), + Create(ids2[0], "de", "Stadt"), + + Search(expected: ids1, text: "en:city"), + Search(expected: ids2, text: "de:Stadt") ); } [Fact] - public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() + public async Task Should_index_localized_content_and_retrieve() + { + if (SupportsMultiLanguage) + { + await TestCombinations( + Create(ids1[0], "de", "Stadt und Land and Fluss"), + + Create(ids2[0], "en", "City and Country und River"), + + Search(expected: ids1, text: "Stadt"), + Search(expected: ids2, text: "City"), + + Search(expected: ids1, text: "and"), + Search(expected: ids2, text: "und") + ); + } + else + { + var both = ids2.Union(ids1).ToList(); + + await TestCombinations( + Create(ids1[0], "de", "Stadt und Land and Fluss"), + + Create(ids2[0], "en", "City and Country und River"), + + Search(expected: ids1, text: "Stadt"), + Search(expected: ids2, text: "City"), + + Search(expected: null, text: "and"), + Search(expected: both, text: "und") + ); + } + } + + [Fact] + public async Task Should_index_invariant_content_and_retrieve() { await TestCombinations( Create(ids1[0], "iv", "Hello"), Create(ids2[0], "iv", "World"), - Search(expected: ids1, text: "helo~"), - Search(expected: ids2, text: "wold~", SearchScope.All) + Search(expected: ids1, text: "Hello"), + Search(expected: ids2, text: "World"), + + Search(expected: null, text: "Hello", SearchScope.Published), + Search(expected: null, text: "World", SearchScope.Published) ); } @@ -282,36 +341,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text ); } - [Fact] - public async Task Should_search_by_field() - { - await TestCombinations( - Create(ids1[0], "en", "City"), - Create(ids2[0], "de", "Stadt"), - - Search(expected: ids1, text: "en:city"), - Search(expected: ids2, text: "de:Stadt") - ); - } - - [Fact] - public async Task Should_index_localized_content_and_retrieve() - { - await TestCombinations( - Create(ids1[0], "de", "Stadt und Land and Fluss"), - - Create(ids2[0], "en", "City and Country und River"), - - Search(expected: ids1, text: "Stadt"), - Search(expected: ids1, text: "and"), - Search(expected: ids2, text: "und"), - - Search(expected: ids2, text: "City"), - Search(expected: ids2, text: "und"), - Search(expected: ids1, text: "and") - ); - } - private IndexOperation Create(Guid id, string language, string text) { var data = @@ -376,7 +405,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text contentEvent.AppId = appId; contentEvent.SchemaId = schemaId; - return p => p.On(Envelope.Create(contentEvent)); + return p => p.On(Enumerable.Repeat(Envelope.Create(contentEvent), 1)); } private IndexOperation Search(List? expected, string text, SearchScope target = SearchScope.All) @@ -385,7 +414,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { var searchFilter = SearchFilter.ShouldHaveSchemas(schemaId.Id); - var result = await p.TextIndexer.SearchAsync(text, app, searchFilter, target); + var result = await p.TextIndex.SearchAsync(text, app, searchFilter, target); if (expected != null) { @@ -400,9 +429,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private async Task TestCombinations(params IndexOperation[] actions) { - for (var i = 0; i < actions.Length; i++) + if (SupportsCleanup) + { + for (var i = 0; i < actions.Length; i++) + { + await TestCombinations(i, actions); + } + } + else { - await TestCombinations(i, actions); + await TestCombinations(0, actions); } } diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs new file mode 100644 index 000000000..eb26d94cb --- /dev/null +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs @@ -0,0 +1,41 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using System.Threading.Tasks; +using Squidex.Domain.Apps.Entities.Contents.Text.Elastic; +using Xunit; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + [Trait("Category", "Dependencies")] + public class TextIndexerTests_Elastic : TextIndexerTestsBase + { + private sealed class TheFactory : IIndexerFactory + { + public Task CleanupAsync() + { + return Task.CompletedTask; + } + + public Task CreateAsync(Guid schemaId) + { + var index = new ElasticSearchTextIndex("http://localhost:9200", "squidex", true); + + return Task.FromResult(index); + } + } + + public override IIndexerFactory Factory { get; } = new TheFactory(); + + public TextIndexerTests_Elastic() + { + SupportsSearchSyntax = false; + SupportsMultiLanguage = false; + } + } +} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_FS.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_FS.cs deleted file mode 100644 index 1f05ae043..000000000 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_FS.cs +++ /dev/null @@ -1,14 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschraenkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -namespace Squidex.Domain.Apps.Entities.Contents.Text -{ - public class TextIndexerTests_FS : TextIndexerTestsBase - { - public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.TempFolder()); - } -} diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs index c9beb15f1..e00b09919 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs @@ -5,6 +5,10 @@ // All rights reserved. Licensed under the MIT license. // ========================================================================== +using System; +using System.Threading.Tasks; +using MongoDB.Driver; +using Squidex.Domain.Apps.Entities.MongoDb.FullText; using Xunit; namespace Squidex.Domain.Apps.Entities.Contents.Text @@ -12,6 +16,33 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text [Trait("Category", "Dependencies")] public class TextIndexerTests_Mongo : TextIndexerTestsBase { - public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.MongoDB()); + private sealed class TheFactory : IIndexerFactory + { + private readonly MongoClient mongoClient = new MongoClient("mongodb://localhost"); + + public Task CleanupAsync() + { + return Task.CompletedTask; + } + + public async Task CreateAsync(Guid schemaId) + { + var database = mongoClient.GetDatabase("FullText"); + + var index = new MongoTextIndex(database, false); + + await index.InitializeAsync(); + + return index; + } + } + + public override IIndexerFactory Factory { get; } = new TheFactory(); + + public TextIndexerTests_Mongo() + { + SupportsSearchSyntax = false; + SupportsMultiLanguage = false; + } } -} +} \ No newline at end of file diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs index a90ffc9ea..2d980114f 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs @@ -51,21 +51,27 @@ namespace Squidex.Domain.Apps.Entities.Rules } [Fact] - public void Should_return_contents_filter_for_events_filter() + public void Should_return_wildcard_filter_for_events_filter() { - Assert.Equal(".*", sut.EventsFilter); + IEventConsumer consumer = sut; + + Assert.Equal(".*", consumer.EventsFilter); } [Fact] - public void Should_return_type_name_for_name() + public async Task Should_do_nothing_on_clear() { - Assert.Equal(typeof(RuleEnqueuer).Name, sut.Name); + IEventConsumer consumer = sut; + + await consumer.ClearAsync(); } [Fact] - public async Task Should_do_nothing_on_clear() + public void Should_return_type_name_for_name() { - await sut.ClearAsync(); + IEventConsumer consumer = sut; + + Assert.Equal(typeof(RuleEnqueuer).Name, consumer.Name); } [Fact] diff --git a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj index 1eb997e60..c6c1285cb 100644 --- a/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj +++ b/backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj @@ -32,6 +32,7 @@ all runtime; build; native; contentfiles; analyzers + ..\..\Squidex.ruleset diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/CompoundEventConsumerTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/CompoundEventConsumerTests.cs deleted file mode 100644 index 6099dce5d..000000000 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/CompoundEventConsumerTests.cs +++ /dev/null @@ -1,131 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschränkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.Threading.Tasks; -using FakeItEasy; -using Squidex.Infrastructure.TestHelpers; -using Xunit; - -namespace Squidex.Infrastructure.EventSourcing -{ - public class CompoundEventConsumerTests - { - private readonly IEventConsumer consumer1 = A.Fake(); - private readonly IEventConsumer consumer2 = A.Fake(); - - [Fact] - public void Should_return_given_name() - { - var sut = new CompoundEventConsumer("consumer-name", consumer1); - - Assert.Equal("consumer-name", sut.Name); - } - - [Fact] - public void Should_return_first_inner_name() - { - A.CallTo(() => consumer1.Name).Returns("my-inner-consumer"); - - var sut = new CompoundEventConsumer(consumer1, consumer2); - - Assert.Equal("my-inner-consumer", sut.Name); - } - - [Fact] - public void Should_return_compound_filter() - { - A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); - A.CallTo(() => consumer2.EventsFilter).Returns("filter2"); - - var sut = new CompoundEventConsumer("my", consumer1, consumer2); - - Assert.Equal("(filter1)|(filter2)", sut.EventsFilter); - } - - [Fact] - public void Should_return_compound_filter_from_array() - { - A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); - A.CallTo(() => consumer2.EventsFilter).Returns("filter2"); - - var sut = new CompoundEventConsumer(new[] { consumer1, consumer2 }); - - Assert.Equal("(filter1)|(filter2)", sut.EventsFilter); - } - - [Fact] - public void Should_ignore_empty_filters() - { - A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); - A.CallTo(() => consumer2.EventsFilter).Returns(string.Empty); - - var sut = new CompoundEventConsumer("my", consumer1, consumer2); - - Assert.Equal("(filter1)", sut.EventsFilter); - } - - [Fact] - public async Task Should_clear_all_consumers() - { - var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); - - await sut.ClearAsync(); - - A.CallTo(() => consumer1.ClearAsync()).MustHaveHappened(); - A.CallTo(() => consumer2.ClearAsync()).MustHaveHappened(); - } - - [Fact] - public async Task Should_invoke_all_consumers() - { - var @event = Envelope.Create(new MyEvent()); - - var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); - - await sut.On(@event); - - A.CallTo(() => consumer1.On(@event)).MustHaveHappened(); - A.CallTo(() => consumer2.On(@event)).MustHaveHappened(); - } - - [Fact] - public void Should_handle_if_any_consumer_handles() - { - var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload")); - - A.CallTo(() => consumer1.Handles(stored)) - .Returns(false); - - A.CallTo(() => consumer2.Handles(stored)) - .Returns(true); - - var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); - - var result = sut.Handles(stored); - - Assert.True(result); - } - - [Fact] - public void Should_no_handle_if_no_consumer_handles() - { - var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload")); - - A.CallTo(() => consumer1.Handles(stored)) - .Returns(false); - - A.CallTo(() => consumer2.Handles(stored)) - .Returns(false); - - var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); - - var result = sut.Handles(stored); - - Assert.False(result); - } - } -} diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs index bc7d114d4..d4e394f8a 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs @@ -399,10 +399,7 @@ namespace Squidex.Infrastructure.EventSourcing } finally { - if (subscription != null) - { - await subscription.StopAsync(); - } + subscription?.Unsubscribe(); } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs index c9fd499b2..5112129fd 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs @@ -6,10 +6,10 @@ // ========================================================================== using System; +using System.Collections.Generic; using System.Threading.Tasks; using FakeItEasy; using FluentAssertions; -using Orleans.Concurrency; using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; using Squidex.Infrastructure.Reflection; @@ -22,6 +22,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains { public sealed class MyEventConsumerGrain : EventConsumerGrain { + private IEventSubscriber currentSubscriber; + public MyEventConsumerGrain( EventConsumerFactory eventConsumerFactory, IGrainState state, @@ -32,14 +34,26 @@ namespace Squidex.Infrastructure.EventSourcing.Grains { } - protected override IEventConsumerGrain GetSelf() + public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) { - return this; + return currentSubscriber.OnEventAsync(subscription, storedEvent); } - protected override IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string? filter, string? position) + public Task OnErrorAsync(IEventSubscription subscription, Exception exception) { - return store.CreateSubscription(subscriber, filter, position); + return currentSubscriber.OnErrorAsync(subscription, exception); + } + + protected override IEventSubscription CreateRetrySubscription(IEventSubscriber subscriber) + { + return CreateSubscription(subscriber); + } + + protected override IEventSubscription CreateSubscription(IEventSubscriber subscriber) + { + currentSubscriber = subscriber; + + return base.CreateSubscription(subscriber); } } @@ -51,7 +65,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains private readonly IEventDataFormatter formatter = A.Fake(); private readonly EventData eventData = new EventData("Type", new EnvelopeHeaders(), "Payload"); private readonly Envelope envelope = new Envelope(new MyEvent()); - private readonly EventConsumerGrain sut; + private readonly MyEventConsumerGrain sut; private readonly string consumerName; private readonly string initialPosition = Guid.NewGuid().ToString(); @@ -70,6 +84,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => eventConsumer.Handles(A._)) .Returns(true); + A.CallTo(() => eventConsumer.On(A>>._)) + .Invokes((IEnumerable> events) => + { + foreach (var @event in events) + { + eventConsumer.On(@event).Wait(); + } + }); + + A.CallTo(() => eventSubscription.Sender) + .Returns(eventSubscription); + A.CallTo(() => formatter.Parse(eventData, null)) .Returns(envelope); @@ -89,6 +115,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(consumerName); await sut.ActivateAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = null }); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) @@ -101,10 +129,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(consumerName); await sut.ActivateAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null }); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } [Fact] @@ -115,10 +145,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(consumerName); await sut.ActivateAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null }); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } [Fact] @@ -127,10 +159,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(consumerName); await sut.ActivateAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null }); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } [Fact] @@ -141,13 +175,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.StopAsync(); await sut.StopAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = null }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -158,22 +194,24 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.StopAsync(); await sut.ResetAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = null, Error = null }); A.CallTo(() => grainState.WriteAsync()) .MustHaveHappened(2, Times.Exactly); A.CallTo(() => eventConsumer.ClearAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventStore.CreateSubscription(A._, A._, grainState.Value.Position)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventStore.CreateSubscription(A._, A._, null)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } [Fact] @@ -186,13 +224,71 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventConsumer.On(envelope)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); + } + + [Fact] + public async Task Should_invoke_and_update_position_when_event_received_one_by_one() + { + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); + + A.CallTo(() => eventConsumer.BatchSize) + .Returns(1); + + await sut.ActivateAsync(consumerName); + await sut.ActivateAsync(); + + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + + await sut.CompleteAsync(); + + AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 5 }); + + A.CallTo(() => grainState.WriteAsync()) + .MustHaveHappened(5, Times.Exactly); + + A.CallTo(() => eventConsumer.On(A>>._)) + .MustHaveHappened(5, Times.Exactly); + } + + [Fact] + public async Task Should_invoke_and_update_position_when_event_received_batched() + { + var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData); + + A.CallTo(() => eventConsumer.BatchSize) + .Returns(100); + + await sut.ActivateAsync(consumerName); + await sut.ActivateAsync(); + + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + await OnEventAsync(eventSubscription, @event); + + await sut.CompleteAsync(); + + AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 5 }); + + A.CallTo(() => grainState.WriteAsync()) + .MustHaveHappenedOnceExactly(); + + A.CallTo(() => eventConsumer.On(A>>._)) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -208,10 +304,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); - AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 }); + await sut.CompleteAsync(); + + AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 0 }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventConsumer.On(envelope)) .MustNotHaveHappened(); @@ -230,10 +328,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); - AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 }); + await sut.CompleteAsync(); + + AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 0 }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventConsumer.On(envelope)) .MustNotHaveHappened(); @@ -249,6 +349,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(A.Fake(), @event); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null }); A.CallTo(() => eventConsumer.On(envelope)) @@ -265,13 +367,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnErrorAsync(eventSubscription, ex); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -284,6 +388,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnErrorAsync(A.Fake(), ex); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null }); A.CallTo(() => grainState.WriteAsync()) @@ -297,6 +403,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(); await sut.ActivateAsync(); + await sut.CompleteAsync(); + A.CallTo(() => eventSubscription.WakeUp()) .MustHaveHappened(); } @@ -313,13 +421,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await sut.ActivateAsync(); await sut.ResetAsync(); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() }); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -337,16 +447,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() }); A.CallTo(() => eventConsumer.On(envelope)) .MustHaveHappened(); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -364,16 +476,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); + await sut.CompleteAsync(); + AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() }); A.CallTo(() => eventConsumer.On(envelope)) .MustNotHaveHappened(); A.CallTo(() => grainState.WriteAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); } [Fact] @@ -391,6 +505,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains await OnEventAsync(eventSubscription, @event); + await sut.CompleteAsync(); + await sut.StopAsync(); await sut.StartAsync(); await sut.StartAsync(); @@ -403,21 +519,21 @@ namespace Squidex.Infrastructure.EventSourcing.Grains A.CallTo(() => grainState.WriteAsync()) .MustHaveHappened(2, Times.Exactly); - A.CallTo(() => eventSubscription.StopAsync()) - .MustHaveHappened(1, Times.Exactly); + A.CallTo(() => eventSubscription.Unsubscribe()) + .MustHaveHappenedOnceExactly(); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) .MustHaveHappened(2, Times.Exactly); } - private Task OnErrorAsync(IEventSubscription subscriber, Exception ex) + private Task OnErrorAsync(IEventSubscription subscription, Exception exception) { - return sut.OnErrorAsync(subscriber.AsImmutable(), ex.AsImmutable()); + return sut.OnErrorAsync(subscription, exception); } - private Task OnEventAsync(IEventSubscription subscriber, StoredEvent ev) + private Task OnEventAsync(IEventSubscription subscription, StoredEvent ev) { - return sut.OnEventAsync(subscriber.AsImmutable(), ev.AsImmutable()); + return sut.OnEventAsync(subscription, ev); } private void AssetGrainState(EventConsumerState state) diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs index f527bcb4d..295fddf5c 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs @@ -164,21 +164,6 @@ namespace Squidex.Infrastructure.EventSourcing : base(eventConsumerFactory, state, eventStore, eventDataFormatter, log) { } - - protected override IEventConsumerGrain GetSelf() - { - return this; - } - - protected override TaskScheduler GetScheduler() - { - return scheduler; - } - - protected override IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string? filter, string? position) - { - return store.CreateSubscription(subscriber, filter, position); - } } public class MyEvent : IEvent @@ -206,16 +191,6 @@ namespace Squidex.Infrastructure.EventSourcing this.expectedCount = expectedCount; } - public Task ClearAsync() - { - return Task.CompletedTask; - } - - public bool Handles(StoredEvent @event) - { - return true; - } - public async Task On(Envelope @event) { Received++; diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs index edf405e56..f8f8eebf4 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs @@ -27,7 +27,7 @@ namespace Squidex.Infrastructure.EventSourcing await WaitAndStopAsync(sut); A.CallTo(() => eventStore.QueryAsync(A>._, "^my-stream", position, A._)) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } [Fact] @@ -95,7 +95,7 @@ namespace Squidex.Infrastructure.EventSourcing { await Task.Delay(200); - await sut.StopAsync(); + sut.Unsubscribe(); } } } diff --git a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs index 4b268ae14..b92419eb2 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs @@ -19,23 +19,34 @@ namespace Squidex.Infrastructure.EventSourcing private readonly IEventSubscription eventSubscription = A.Fake(); private readonly IEventSubscriber sutSubscriber; private readonly RetrySubscription sut; - private readonly string streamFilter = Guid.NewGuid().ToString(); public RetrySubscriptionTests() { - A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)).Returns(eventSubscription); + A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) + .Returns(eventSubscription); + + A.CallTo(() => eventSubscription.Sender) + .Returns(eventSubscription); - sut = new RetrySubscription(eventStore, eventSubscriber, streamFilter, null) { ReconnectWaitMs = 50 }; + sut = new RetrySubscription(eventSubscriber, s => eventStore.CreateSubscription(s)) { ReconnectWaitMs = 50 }; sutSubscriber = sut; } [Fact] - public async Task Should_subscribe_after_constructor() + public void Should_return_original_subscription_as_sender() + { + var sender = sut.Sender; + + Assert.Same(eventSubscription, sender); + } + + [Fact] + public void Should_subscribe_after_constructor() { - await sut.StopAsync(); + sut.Unsubscribe(); - A.CallTo(() => eventStore.CreateSubscription(sut, streamFilter, null)) + A.CallTo(() => eventStore.CreateSubscription(sut, A._, A._)) .MustHaveHappened(); } @@ -46,15 +57,15 @@ namespace Squidex.Infrastructure.EventSourcing await Task.Delay(1000); - await sut.StopAsync(); + sut.Unsubscribe(); - A.CallTo(() => eventSubscription.StopAsync()) + A.CallTo(() => eventSubscription.Unsubscribe()) .MustHaveHappened(2, Times.Exactly); A.CallTo(() => eventStore.CreateSubscription(A._, A._, A._)) .MustHaveHappened(2, Times.Exactly); - A.CallTo(() => eventSubscriber.OnErrorAsync(A._, A._)) + A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, A._)) .MustNotHaveHappened(); } @@ -64,26 +75,28 @@ namespace Squidex.Infrastructure.EventSourcing var ex = new InvalidOperationException(); await OnErrorAsync(eventSubscription, ex); - await OnErrorAsync(null!, ex); - await OnErrorAsync(null!, ex); - await OnErrorAsync(null!, ex); - await OnErrorAsync(null!, ex); - await OnErrorAsync(null!, ex); - await sut.StopAsync(); - - A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex)) + await OnErrorAsync(eventSubscription, ex); + await OnErrorAsync(eventSubscription, ex); + await OnErrorAsync(eventSubscription, ex); + await OnErrorAsync(eventSubscription, ex); + await OnErrorAsync(eventSubscription, ex); + + sut.Unsubscribe(); + + A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, ex)) .MustHaveHappened(); } [Fact] - public async Task Should_not_forward_error_when_exception_is_from_another_subscription() + public async Task Should_not_forward_error_when_exception_is_raised_after_unsubscribe() { var ex = new InvalidOperationException(); - await OnErrorAsync(A.Fake(), ex); - await sut.StopAsync(); + await OnErrorAsync(eventSubscription, ex); + + sut.Unsubscribe(); - A.CallTo(() => eventSubscriber.OnErrorAsync(A._, A._)) + A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, A._)) .MustNotHaveHappened(); } @@ -93,22 +106,24 @@ namespace Squidex.Infrastructure.EventSourcing var ev = new StoredEvent("Stream", "1", 2, new EventData("Type", new EnvelopeHeaders(), "Payload")); await OnEventAsync(eventSubscription, ev); - await sut.StopAsync(); - A.CallTo(() => eventSubscriber.OnEventAsync(sut, ev)) + sut.Unsubscribe(); + + A.CallTo(() => eventSubscriber.OnEventAsync(eventSubscription, ev)) .MustHaveHappened(); } [Fact] - public async Task Should_not_forward_event_when_message_is_from_another_subscription() + public async Task Should_forward_event_when_message_is_from_another_subscription() { var ev = new StoredEvent("Stream", "1", 2, new EventData("Type", new EnvelopeHeaders(), "Payload")); await OnEventAsync(A.Fake(), ev); - await sut.StopAsync(); + + sut.Unsubscribe(); A.CallTo(() => eventSubscriber.OnEventAsync(A._, A._)) - .MustNotHaveHappened(); + .MustHaveHappened(); } private Task OnErrorAsync(IEventSubscription subscriber, Exception ex) diff --git a/backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs index 80ac9034d..2888c3473 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs @@ -149,9 +149,9 @@ namespace Squidex.Infrastructure.Migrations await Task.WhenAll(Enumerable.Repeat(0, 10).Select(x => Task.Run(() => sut.MigrateAsync()))); A.CallTo(() => migrator_0_1.UpdateAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); A.CallTo(() => migrator_1_2.UpdateAsync()) - .MustHaveHappened(1, Times.Exactly); + .MustHaveHappenedOnceExactly(); } private IMigration BuildMigration(int fromVersion, int toVersion) diff --git a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoExtensionsTests.cs b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoExtensionsTests.cs index ba28b028a..c2742c05b 100644 --- a/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoExtensionsTests.cs +++ b/backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoExtensionsTests.cs @@ -78,7 +78,7 @@ namespace Squidex.Infrastructure.MongoDb var cursor = new Cursor().Add(0, 1, 2, 3, 4, 5); - await cursor.ForEachPipelineAsync(x => + await cursor.ForEachPipedAsync(x => { result.Add(x); return Task.CompletedTask; @@ -98,7 +98,7 @@ namespace Squidex.Infrastructure.MongoDb { await Assert.ThrowsAsync(() => { - return cursor.ForEachPipelineAsync(x => + return cursor.ForEachPipedAsync(x => { result.Add(x); return Task.CompletedTask; @@ -120,7 +120,7 @@ namespace Squidex.Infrastructure.MongoDb { await Assert.ThrowsAsync(() => { - return cursor.ForEachPipelineAsync(x => + return cursor.ForEachPipedAsync(x => { if (x == 2) { @@ -147,7 +147,7 @@ namespace Squidex.Infrastructure.MongoDb { await Assert.ThrowsAnyAsync(() => { - return cursor.ForEachPipelineAsync(x => + return cursor.ForEachPipedAsync(x => { if (x == 2) { diff --git a/backend/tests/Squidex.Infrastructure.Tests/Tasks/SingleThreadedDispatcherTests.cs b/backend/tests/Squidex.Infrastructure.Tests/Tasks/SingleThreadedDispatcherTests.cs deleted file mode 100644 index 6b57dc5d3..000000000 --- a/backend/tests/Squidex.Infrastructure.Tests/Tasks/SingleThreadedDispatcherTests.cs +++ /dev/null @@ -1,56 +0,0 @@ -// ========================================================================== -// Squidex Headless CMS -// ========================================================================== -// Copyright (c) Squidex UG (haftungsbeschränkt) -// All rights reserved. Licensed under the MIT license. -// ========================================================================== - -using System.Collections.Generic; -using System.Linq; -using System.Threading.Tasks; -using Xunit; - -namespace Squidex.Infrastructure.Tasks -{ - public class SingleThreadedDispatcherTests - { - private readonly SingleThreadedDispatcher sut = new SingleThreadedDispatcher(); - - [Fact] - public async Task Should_handle_with_task_messages_sequentially() - { - var source = Enumerable.Range(1, 100); - var target = new List(); - - foreach (var item in source) - { - sut.DispatchAsync(() => - { - target.Add(item); - - return Task.CompletedTask; - }).Forget(); - } - - await sut.StopAndWaitAsync(); - - Assert.Equal(source, target); - } - - [Fact] - public async Task Should_handle_messages_sequentially() - { - var source = Enumerable.Range(1, 100); - var target = new List(); - - foreach (var item in source) - { - sut.DispatchAsync(() => target.Add(item)).Forget(); - } - - await sut.StopAndWaitAsync(); - - Assert.Equal(source, target); - } - } -}