Feature/batch processing (#578)

* Mongo full text index and batch processing for event consumers.

backend/src/Migrations/Migrations/ConvertEventStore.cs

@@ -32,20 +32,20 @@ namespace Migrations.Migrations
 
                 var filter = Builders<BsonDocument>.Filter;
 
-                var writesBatches = new List<WriteModel<BsonDocument>>();
+                var writes = new List<WriteModel<BsonDocument>>();
 
                 async Task WriteAsync(WriteModel<BsonDocument>? model, bool force)
                 {
                     if (model != null)
                     {
-                        writesBatches.Add(model);
+                        writes.Add(model);
                     }
 
-                    if (writesBatches.Count == 1000 || (force && writesBatches.Count > 0))
+                    if (writes.Count == 1000 || (force && writes.Count > 0))
                     {
-                        await collection.BulkWriteAsync(writesBatches);
+                        await collection.BulkWriteAsync(writes);
 
-                        writesBatches.Clear();
+                        writes.Clear();
                     }
                 }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetFolderRepository_SnapshotStore.cs

@@ -55,7 +55,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
         {
             using (Profiler.TraceMethod<MongoAssetFolderRepository>())
             {
-                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(Map(x), x.Version), ct);
+                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(Map(x), x.Version), ct);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/Assets/MongoAssetRepository_SnapshotStore.cs

@@ -55,7 +55,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Assets
         {
             using (Profiler.TraceMethod<MongoAssetRepository>())
             {
-                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(Map(x), x.Version), ct);
+                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(Map(x), x.Version), ct);
             }
         }
 

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs

@@ -1,124 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Threading.Tasks;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-using MongoDB.Driver.GridFS;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
-using Squidex.Infrastructure;
-using LuceneDirectory = Lucene.Net.Store.Directory;
-
-namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
-{
-    public sealed class MongoIndexStorage : IIndexStorage
-    {
-        private readonly IGridFSBucket<string> bucket;
-
-        public MongoIndexStorage(IGridFSBucket<string> bucket)
-        {
-            Guard.NotNull(bucket, nameof(bucket));
-
-            this.bucket = bucket;
-        }
-
-        public async Task<LuceneDirectory> CreateDirectoryAsync(Guid ownerId)
-        {
-            var fileId = $"index_{ownerId}";
-
-            var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), fileId));
-
-            if (directoryInfo.Exists)
-            {
-                directoryInfo.Delete(true);
-            }
-
-            directoryInfo.Create();
-
-            try
-            {
-                using (var stream = await bucket.OpenDownloadStreamAsync(fileId))
-                {
-                    using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, true))
-                    {
-                        foreach (var entry in zipArchive.Entries)
-                        {
-                            var file = new FileInfo(Path.Combine(directoryInfo.FullName, entry.Name));
-
-                            using (var entryStream = entry.Open())
-                            {
-                                using (var fileStream = file.OpenWrite())
-                                {
-                                    await entryStream.CopyToAsync(fileStream);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            catch (GridFSFileNotFoundException)
-            {
-            }
-
-            var directory = FSDirectory.Open(directoryInfo);
-
-            return directory;
-        }
-
-        public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
-        {
-            var directoryInfo = ((FSDirectory)directory).Directory;
-
-            var commit = snapshotter.Snapshot();
-            try
-            {
-                var fileId = directoryInfo.Name;
-
-                try
-                {
-                    await bucket.DeleteAsync(fileId);
-                }
-                catch (GridFSFileNotFoundException)
-                {
-                }
-
-                using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId))
-                {
-                    using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true))
-                    {
-                        foreach (var fileName in commit.FileNames)
-                        {
-                            var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
-
-                            using (var fileStream = file.OpenRead())
-                            {
-                                var entry = zipArchive.CreateEntry(fileStream.Name);
-
-                                using (var entryStream = entry.Open())
-                                {
-                                    await fileStream.CopyToAsync(entryStream);
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            finally
-            {
-                snapshotter.Release(commit);
-            }
-        }
-
-        public Task ClearAsync()
-        {
-            return bucket.DropAsync();
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndex.cs

@@ -0,0 +1,168 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using Squidex.Domain.Apps.Entities.Apps;
+using Squidex.Domain.Apps.Entities.Contents;
+using Squidex.Domain.Apps.Entities.Contents.Text;
+using Squidex.Infrastructure.MongoDb;
+using Squidex.Infrastructure.Tasks;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
+{
+    public sealed class MongoTextIndex : MongoRepositoryBase<MongoTextIndexEntity>, ITextIndex
+    {
+        private static readonly List<Guid> EmptyResults = new List<Guid>();
+
+        public MongoTextIndex(IMongoDatabase database, bool setup = false)
+            : base(database, setup)
+        {
+        }
+
+        protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity> collection, CancellationToken ct = default)
+        {
+            return collection.Indexes.CreateManyAsync(new[]
+            {
+                new CreateIndexModel<MongoTextIndexEntity>(
+                    Index
+                        .Text("t.t")
+                        .Ascending(x => x.AppId)
+                        .Ascending(x => x.ServeAll)
+                        .Ascending(x => x.ServePublished)
+                        .Ascending(x => x.SchemaId))
+            }, ct);
+        }
+
+        protected override string CollectionName()
+        {
+            return "TextIndex";
+        }
+
+        public Task ExecuteAsync(params IndexCommand[] commands)
+        {
+            var writes = new List<WriteModel<MongoTextIndexEntity>>(commands.Length);
+
+            foreach (var command in commands)
+            {
+                switch (command)
+                {
+                    case DeleteIndexEntry delete:
+                        writes.Add(
+                            new DeleteOneModel<MongoTextIndexEntity>(
+                                Filter.Eq(x => x.DocId, command.DocId)));
+                        break;
+                    case UpdateIndexEntry update:
+                        writes.Add(
+                            new UpdateOneModel<MongoTextIndexEntity>(
+                                Filter.Eq(x => x.DocId, command.DocId),
+                                Update
+                                    .Set(x => x.ServeAll, update.ServeAll)
+                                    .Set(x => x.ServePublished, update.ServePublished)));
+                        break;
+                    case UpsertIndexEntry upsert when upsert.Texts.Count > 0:
+                        writes.Add(
+                            new ReplaceOneModel<MongoTextIndexEntity>(
+                                Filter.Eq(x => x.DocId, command.DocId),
+                                new MongoTextIndexEntity
+                                {
+                                    DocId = upsert.DocId,
+                                    ContentId = upsert.ContentId,
+                                    SchemaId = upsert.SchemaId.Id,
+                                    ServeAll = upsert.ServeAll,
+                                    ServePublished = upsert.ServePublished,
+                                    Texts = upsert.Texts.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList(),
+                                    AppId = upsert.AppId.Id
+                                })
+                            {
+                                IsUpsert = true
+                            });
+                        break;
+                }
+            }
+
+            if (writes.Count == 0)
+            {
+                return Task.CompletedTask;
+            }
+
+            return Collection.BulkWriteAsync(writes);
+        }
+
+        public async Task<List<Guid>?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope)
+        {
+            if (string.IsNullOrWhiteSpace(queryText))
+            {
+                return EmptyResults;
+            }
+
+            if (filter == null)
+            {
+                return await SearchByAppAsync(queryText, app, scope, 2000);
+            }
+            else if (filter.Must)
+            {
+                return await SearchBySchemaAsync(queryText, app, filter, scope, 2000);
+            }
+            else
+            {
+                var (bySchema, byApp) =
+                    await AsyncHelper.WhenAll(
+                        SearchBySchemaAsync(queryText, app, filter, scope, 1000),
+                        SearchByAppAsync(queryText, app, scope, 1000));
+
+                return bySchema.Union(byApp).Distinct().ToList();
+            }
+        }
+
+        private async Task<List<Guid>> SearchBySchemaAsync(string queryText, IAppEntity app, SearchFilter filter, SearchScope scope, int limit)
+        {
+            var bySchema =
+                await Collection.Find(
+                    Filter.And(
+                        Filter.Eq(x => x.AppId, app.Id),
+                        Filter.In(x => x.SchemaId, filter.SchemaIds),
+                        Filter_ByScope(scope),
+                        Filter.Text(queryText)))
+                    .Only(x => x.ContentId).Limit(limit)
+                    .ToListAsync();
+
+            return bySchema.Select(x => Guid.Parse(x["_ci"].AsString)).Distinct().ToList();
+        }
+
+        private async Task<List<Guid>> SearchByAppAsync(string queryText, IAppEntity app, SearchScope scope, int limit)
+        {
+            var bySchema =
+                await Collection.Find(
+                    Filter.And(
+                        Filter.Eq(x => x.AppId, app.Id),
+                        Filter.Exists(x => x.SchemaId),
+                        Filter_ByScope(scope),
+                        Filter.Text(queryText)))
+                    .Only(x => x.ContentId).Limit(limit)
+                    .ToListAsync();
+
+            return bySchema.Select(x => Guid.Parse(x["_ci"].AsString)).Distinct().ToList();
+        }
+
+        private static FilterDefinition<MongoTextIndexEntity> Filter_ByScope(SearchScope scope)
+        {
+            if (scope == SearchScope.All)
+            {
+                return Filter.Eq(x => x.ServeAll, true);
+            }
+            else
+            {
+                return Filter.Eq(x => x.ServePublished, true);
+            }
+        }
+    }
+}

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntity.cs

@@ -0,0 +1,49 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using MongoDB.Bson;
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
+{
+    public sealed class MongoTextIndexEntity
+    {
+        [BsonId]
+        [BsonElement]
+        [BsonRepresentation(BsonType.String)]
+        public string DocId { get; set; }
+
+        [BsonRequired]
+        [BsonElement("_ci")]
+        [BsonRepresentation(BsonType.String)]
+        public Guid ContentId { get; set; }
+
+        [BsonRequired]
+        [BsonElement("_ai")]
+        [BsonRepresentation(BsonType.String)]
+        public Guid AppId { get; set; }
+
+        [BsonRequired]
+        [BsonElement("_si")]
+        [BsonRepresentation(BsonType.String)]
+        public Guid SchemaId { get; set; }
+
+        [BsonRequired]
+        [BsonElement("fa")]
+        public bool ServeAll { get; set; }
+
+        [BsonRequired]
+        [BsonElement("fp")]
+        public bool ServePublished { get; set; }
+
+        [BsonIgnoreIfNull]
+        [BsonElement("t")]
+        public List<MongoTextIndexEntityText> Texts { get; set; }
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Assets.cs → backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexEntityText.cs

@@ -5,10 +5,18 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-namespace Squidex.Domain.Apps.Entities.Contents.Text
+using MongoDB.Bson.Serialization.Attributes;
+
+namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
 {
-    public class TextIndexerTests_Assets : TextIndexerTestsBase
+    public sealed class MongoTextIndexEntityText
     {
-        public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.Assets());
+        [BsonRequired]
+        [BsonElement("t")]
+        public string Text { get; set; }
+
+        [BsonIgnoreIfNull]
+        [BsonElement("language")]
+        public string Language { get; set; }
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs

@@ -6,6 +6,8 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization;
 using MongoDB.Driver;
@@ -57,5 +59,38 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.FullText
         {
             return Collection.ReplaceOneAsync(x => x.ContentId == state.ContentId, state, UpsertReplace);
         }
+
+        public async Task<Dictionary<Guid, TextContentState>> GetAsync(HashSet<Guid> ids)
+        {
+            var entities = await Collection.Find(Filter.In(x => x.ContentId, ids)).ToListAsync();
+
+            return entities.ToDictionary(x => x.ContentId);
+        }
+
+        public Task SetAsync(List<TextContentState> updates)
+        {
+            var writes = new List<WriteModel<TextContentState>>();
+
+            foreach (var update in updates)
+            {
+                if (update.IsDeleted)
+                {
+                    writes.Add(
+                        new DeleteOneModel<TextContentState>(
+                            Filter.Eq(x => x.ContentId, update.ContentId)));
+                }
+                else
+                {
+                    writes.Add(
+                        new ReplaceOneModel<TextContentState>(
+                            Filter.Eq(x => x.ContentId, update.ContentId), update)
+                        {
+                            IsUpsert = true
+                        });
+                }
+            }
+
+            return Collection.BulkWriteAsync(writes);
+        }
     }
 }

backend/src/Squidex.Domain.Apps.Entities.MongoDb/History/MongoHistoryEventRepository.cs

@@ -7,6 +7,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using MongoDB.Bson.Serialization;
@@ -64,9 +65,20 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.History
             }
         }
 
-        public Task InsertAsync(HistoryEvent item)
+        public async Task InsertManyAsync(IEnumerable<HistoryEvent> historyEvents)
         {
-            return Collection.ReplaceOneAsync(x => x.Id == item.Id, item, UpsertReplace);
+            var writes = historyEvents
+                .Select(x =>
+                    new ReplaceOneModel<HistoryEvent>(Filter.Eq(y => y.Id, x.Id), x)
+                    {
+                        IsUpsert = true
+                    })
+                .ToList();
+
+            if (writes.Count > 0)
+            {
+                await Collection.BulkWriteAsync(writes);
+            }
         }
 
         public Task RemoveAsync(Guid appId)

backend/src/Squidex.Domain.Apps.Entities/Apps/Invitation/InvitationEventConsumer.cs

@@ -45,16 +45,6 @@ namespace Squidex.Domain.Apps.Entities.Apps.Invitation
             this.log = log;
         }
 
-        public bool Handles(StoredEvent @event)
-        {
-            return true;
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-
         public async Task On(Envelope<IEvent> @event)
         {
             if (!emailSender.IsActive)

backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker.cs

@@ -9,14 +9,13 @@ using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Squidex.Infrastructure;
-using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.UsageTracking;
 
 #pragma warning disable CS0649
 
 namespace Squidex.Domain.Apps.Entities.Assets
 {
-    public partial class AssetUsageTracker : IAssetUsageTracker, IEventConsumer
+    public partial class AssetUsageTracker : IAssetUsageTracker
     {
         private const string CounterTotalCount = "TotalAssets";
         private const string CounterTotalSize = "TotalSize";

backend/src/Squidex.Domain.Apps.Entities/Assets/AssetUsageTracker_EventHandling.cs

@@ -13,26 +13,26 @@ using Squidex.Infrastructure.UsageTracking;
 
 namespace Squidex.Domain.Apps.Entities.Assets
 {
-    public partial class AssetUsageTracker
+    public partial class AssetUsageTracker : IEventConsumer
     {
-        public string Name
+        public int BatchSize
         {
-            get { return GetType().Name; }
+            get { return 1000; }
         }
 
-        public string EventsFilter
+        public int BatchDelay
         {
-            get { return "^asset-"; }
+            get { return 1000; }
         }
 
-        public bool Handles(StoredEvent @event)
+        public string Name
         {
-            return true;
+            get { return GetType().Name; }
         }
 
-        public Task ClearAsync()
+        public string EventsFilter
         {
-            return Task.CompletedTask;
+            get { return "^asset-"; }
         }
 
         public Task On(Envelope<IEvent> @event)

backend/src/Squidex.Domain.Apps.Entities/Assets/RecursiveDeleter.cs

@@ -57,11 +57,6 @@ namespace Squidex.Domain.Apps.Entities.Assets
             folderDeletedType = typeNameRegistry.GetName<AssetFolderDeleted>();
         }
 
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-
         public bool Handles(StoredEvent @event)
         {
             return @event.Data.Type == folderDeletedType;

backend/src/Squidex.Domain.Apps.Entities/Backup/RestoreGrain.cs

@@ -223,7 +223,7 @@ namespace Squidex.Domain.Apps.Entities.Backup
 
                     log.LogError(ex, logContext, (ctx, w) =>
                     {
-                        w.WriteProperty("action", "retore");
+                        w.WriteProperty("action", "restore");
                         w.WriteProperty("status", "failed");
                         w.WriteProperty("operationId", ctx.jobId);
                         w.WriteProperty("url", ctx.jobUrl);

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs

@@ -18,14 +18,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
     [ExcludeFromCodeCoverage]
     public sealed class ElasticSearchTextIndex : ITextIndex
     {
-        private const string IndexName = "contents";
         private readonly ElasticLowLevelClient client;
+        private readonly string indexName;
+        private readonly bool waitForTesting;
 
-        public ElasticSearchTextIndex()
+        public ElasticSearchTextIndex(string configurationString, string indexName, bool waitForTesting = false)
         {
-            var config = new ConnectionConfiguration(new Uri("http://localhost:9200"));
+            Guard.NotNull(configurationString, nameof(configurationString));
+            Guard.NotNull(indexName, nameof(indexName));
+
+            var config = new ConnectionConfiguration(new Uri(configurationString));
 
             client = new ElasticLowLevelClient(config);
+
+            this.indexName = indexName;
+
+            this.waitForTesting = waitForTesting;
         }
 
         public Task ClearAsync()
@@ -33,14 +41,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
             return Task.CompletedTask;
         }
 
-        public async Task ExecuteAsync(NamedId<Guid> appId, NamedId<Guid> schemaId, params IndexCommand[] commands)
+        public async Task ExecuteAsync(params IndexCommand[] commands)
         {
             foreach (var command in commands)
             {
                 switch (command)
                 {
                     case UpsertIndexEntry upsert:
-                        await UpsertAsync(appId, schemaId, upsert);
+                        await UpsertAsync(upsert);
                         break;
                     case UpdateIndexEntry update:
                         await UpdateAsync(update);
@@ -50,23 +58,28 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
                         break;
                 }
             }
+
+            if (waitForTesting)
+            {
+                await Task.Delay(1000);
+            }
         }
 
-        private async Task UpsertAsync(NamedId<Guid> appId, NamedId<Guid> schemaId, UpsertIndexEntry upsert)
+        private async Task UpsertAsync(UpsertIndexEntry upsert)
         {
             var data = new
             {
-                appId = appId.Id,
+                appId = upsert.AppId.Id,
-                appName = appId.Name,
+                appName = upsert.AppId.Name,
                 contentId = upsert.ContentId,
-                schemaId = schemaId.Id,
+                schemaId = upsert.SchemaId.Id,
-                schemaName = schemaId.Name,
+                schemaName = upsert.SchemaId.Name,
                 serveAll = upsert.ServeAll,
                 servePublished = upsert.ServePublished,
                 texts = upsert.Texts,
             };
 
-            var result = await client.IndexAsync<StringResponse>(IndexName, upsert.DocId, CreatePost(data));
+            var result = await client.IndexAsync<StringResponse>(indexName, upsert.DocId, CreatePost(data));
 
             if (!result.Success)
             {
@@ -80,12 +93,12 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
             {
                 doc = new
                 {
-                    update.ServeAll,
+                    serveAll = update.ServeAll,
-                    update.ServePublished
+                    servePublished = update.ServePublished
                 }
             };
 
-            var result = await client.UpdateAsync<StringResponse>(IndexName, update.DocId, CreatePost(data));
+            var result = await client.UpdateAsync<StringResponse>(indexName, update.DocId, CreatePost(data));
 
             if (!result.Success)
             {
@@ -95,7 +108,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
 
         private Task DeleteAsync(DeleteIndexEntry delete)
         {
-            return client.DeleteAsync<StringResponse>(IndexName, delete.DocId);
+            return client.DeleteAsync<StringResponse>(indexName, delete.DocId);
         }
 
         private static PostData CreatePost<T>(T data)
@@ -155,7 +168,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
             {
                 var bySchema = new
                 {
-                    term = new Dictionary<string, object>
+                    terms = new Dictionary<string, object>
                     {
                         ["schemaId.keyword"] = filter.SchemaIds
                     }
@@ -171,7 +184,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.Elastic
                 }
             }
 
-            var result = await client.SearchAsync<DynamicResponse>(IndexName, CreatePost(query));
+            var result = await client.SearchAsync<DynamicResponse>(indexName, CreatePost(query));
 
             if (!result.Success)
             {

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs

@@ -9,7 +9,6 @@ using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using Squidex.Domain.Apps.Entities.Apps;
-using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
@@ -19,6 +18,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         Task ClearAsync();
 
-        Task ExecuteAsync(NamedId<Guid> appId, NamedId<Guid> schemaId, params IndexCommand[] commands);
+        Task ExecuteAsync(params IndexCommand[] commands);
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexCommand.cs

@@ -5,10 +5,17 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System;
+using Squidex.Infrastructure;
+
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
     public abstract class IndexCommand
     {
+        public NamedId<Guid> AppId { get; set; }
+
+        public NamedId<Guid> SchemaId { get; set; }
+
         public string DocId { get; set; }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IIndex.cs

@@ -1,28 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using Lucene.Net.Analysis;
-using Lucene.Net.Index;
-using Lucene.Net.Search;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public interface IIndex
-    {
-        Analyzer? Analyzer { get; }
-
-        IndexReader? Reader { get; }
-
-        IndexSearcher? Searcher { get; }
-
-        IndexWriter Writer { get; }
-
-        void EnsureReader();
-
-        void MarkStale();
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/ILuceneTextIndexGrain.cs

@@ -1,23 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Orleans;
-using Orleans.Concurrency;
-using Squidex.Infrastructure;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public interface ILuceneTextIndexGrain : IGrainWithGuidKey
-    {
-        Task IndexAsync(NamedId<Guid> schemaId, Immutable<IndexCommand[]> updates);
-
-        Task<List<Guid>> SearchAsync(string queryText, SearchFilter? filter, SearchContext context);
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager.cs

@@ -1,151 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Log;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public sealed partial class IndexManager : DisposableObjectBase
-    {
-        private readonly Dictionary<Guid, IndexHolder> indices = new Dictionary<Guid, IndexHolder>();
-        private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1);
-        private readonly IIndexStorage indexStorage;
-        private readonly ISemanticLog log;
-
-        public IndexManager(IIndexStorage indexStorage, ISemanticLog log)
-        {
-            Guard.NotNull(indexStorage, nameof(indexStorage));
-            Guard.NotNull(log, nameof(log));
-
-            this.indexStorage = indexStorage;
-
-            this.log = log;
-        }
-
-        protected override void DisposeObject(bool disposing)
-        {
-            if (disposing)
-            {
-                ReleaseAllAsync().Wait();
-            }
-        }
-
-        public Task ClearAsync()
-        {
-            return indexStorage.ClearAsync();
-        }
-
-        public async Task<IIndex> AcquireAsync(Guid ownerId)
-        {
-            IndexHolder? indexHolder;
-
-            try
-            {
-                await lockObject.WaitAsync();
-
-                if (indices.TryGetValue(ownerId, out indexHolder))
-                {
-                    log.LogWarning(w => w
-                        .WriteProperty("message", "Unreleased index found.")
-                        .WriteProperty("ownerId", ownerId.ToString()));
-
-                    await CommitInternalAsync(indexHolder, true);
-                }
-
-                var directory = await indexStorage.CreateDirectoryAsync(ownerId);
-
-                indexHolder = new IndexHolder(ownerId, directory);
-                indices[ownerId] = indexHolder;
-            }
-            finally
-            {
-                lockObject.Release();
-            }
-
-            return indexHolder;
-        }
-
-        public async Task ReleaseAsync(IIndex index)
-        {
-            Guard.NotNull(index, nameof(index));
-
-            var indexHolder = (IndexHolder)index;
-
-            try
-            {
-                lockObject.Wait();
-
-                indexHolder.Dispose();
-                indices.Remove(indexHolder.Id);
-            }
-            finally
-            {
-                lockObject.Release();
-            }
-
-            await CommitInternalAsync(indexHolder, true);
-        }
-
-        public Task CommitAsync(IIndex index)
-        {
-            Guard.NotNull(index, nameof(index));
-
-            return CommitInternalAsync(index, false);
-        }
-
-        private async Task CommitInternalAsync(IIndex index, bool dispose)
-        {
-            if (index is IndexHolder holder)
-            {
-                if (dispose)
-                {
-                    holder.Dispose();
-                }
-                else
-                {
-                    holder.Commit();
-                }
-
-                await indexStorage.WriteAsync(holder.Directory, holder.Snapshotter);
-            }
-        }
-
-        private async Task ReleaseAllAsync()
-        {
-            var current = indices.Values.ToList();
-
-            try
-            {
-                lockObject.Wait();
-
-                indices.Clear();
-            }
-            finally
-            {
-                lockObject.Release();
-            }
-
-            if (current.Count > 0)
-            {
-                log.LogWarning(w => w
-                    .WriteProperty("message", "Unreleased indices found.")
-                    .WriteProperty("count", indices.Count));
-
-                foreach (var index in current)
-                {
-                    await CommitInternalAsync(index, true);
-                }
-            }
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager_Impl.cs

@@ -1,179 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using Lucene.Net.Analysis;
-using Lucene.Net.Index;
-using Lucene.Net.Search;
-using Lucene.Net.Store;
-using Lucene.Net.Util;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public sealed partial class IndexManager
-    {
-        private sealed class IndexHolder : IDisposable, IIndex
-        {
-            private const LuceneVersion Version = LuceneVersion.LUCENE_48;
-            private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler();
-            private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version);
-            private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
-            private readonly Directory directory;
-            private IndexWriter indexWriter;
-            private IndexSearcher? indexSearcher;
-            private DirectoryReader? indexReader;
-            private bool isDisposed;
-
-            public Analyzer Analyzer
-            {
-                get { return SharedAnalyzer; }
-            }
-
-            public SnapshotDeletionPolicy Snapshotter
-            {
-                get { return snapshotter; }
-            }
-
-            public Directory Directory
-            {
-                get { return directory; }
-            }
-
-            public IndexWriter Writer
-            {
-                get
-                {
-                    ThrowIfReleased();
-
-                    if (indexWriter == null)
-                    {
-                        throw new InvalidOperationException("Index writer has not been created yet. Call Open()");
-                    }
-
-                    return indexWriter;
-                }
-            }
-
-            public IndexReader? Reader
-            {
-                get
-                {
-                    ThrowIfReleased();
-
-                    return indexReader;
-                }
-            }
-
-            public IndexSearcher? Searcher
-            {
-                get
-                {
-                    ThrowIfReleased();
-
-                    return indexSearcher;
-                }
-            }
-
-            public Guid Id { get; }
-
-            public IndexHolder(Guid id, Directory directory)
-            {
-                Id = id;
-
-                this.directory = directory;
-
-                RecreateIndexWriter();
-
-                if (indexWriter.NumDocs > 0)
-                {
-                    EnsureReader();
-                }
-            }
-
-            public void Dispose()
-            {
-                if (!isDisposed)
-                {
-                    indexReader?.Dispose();
-                    indexReader = null;
-
-                    indexWriter.Dispose();
-
-                    isDisposed = true;
-                }
-            }
-
-            private IndexWriter RecreateIndexWriter()
-            {
-                var config = new IndexWriterConfig(Version, Analyzer)
-                {
-                    IndexDeletionPolicy = snapshotter,
-                    MergePolicy = new TieredMergePolicy(),
-                    MergeScheduler = MergeScheduler
-                };
-
-                indexWriter = new IndexWriter(directory, config);
-
-                MarkStale();
-
-                return indexWriter;
-            }
-
-            public void EnsureReader()
-            {
-                ThrowIfReleased();
-
-                if (indexReader == null && indexWriter != null)
-                {
-                    indexReader = indexWriter.GetReader(true);
-                    indexSearcher = new IndexSearcher(indexReader);
-                }
-            }
-
-            public void MarkStale()
-            {
-                ThrowIfReleased();
-
-                MarkStaleInternal();
-            }
-
-            private void MarkStaleInternal()
-            {
-                if (indexReader != null)
-                {
-                    indexReader.Dispose();
-                    indexReader = null;
-                    indexSearcher = null;
-                }
-            }
-
-            internal void Commit()
-            {
-                try
-                {
-                    MarkStaleInternal();
-
-                    indexWriter.Commit();
-                }
-                catch (OutOfMemoryException)
-                {
-                    RecreateIndexWriter();
-
-                    throw;
-                }
-            }
-
-            private void ThrowIfReleased()
-            {
-                if (indexWriter == null)
-                {
-                    throw new InvalidOperationException("Index is already released or not open yet.");
-                }
-            }
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneExtensions.cs

@@ -1,53 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using Lucene.Net.Index;
-using Lucene.Net.Util;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public static class LuceneExtensions
-    {
-        public static BytesRef GetBinaryValue(this IndexReader? reader, string field, int docId, BytesRef? result = null)
-        {
-            if (result != null)
-            {
-                Array.Clear(result.Bytes, 0, result.Bytes.Length);
-            }
-            else
-            {
-                result = new BytesRef();
-            }
-
-            if (reader == null || docId < 0)
-            {
-                return result;
-            }
-
-            var leaves = reader.Leaves;
-
-            if (leaves.Count == 1)
-            {
-                var docValues = leaves[0].AtomicReader.GetBinaryDocValues(field);
-
-                docValues.Get(docId, result);
-            }
-            else if (leaves.Count > 1)
-            {
-                var subIndex = ReaderUtil.SubIndex(docId, leaves);
-
-                var subLeave = leaves[subIndex];
-                var subValues = subLeave.AtomicReader.GetBinaryDocValues(field);
-
-                subValues.Get(docId - subLeave.DocBase, result);
-            }
-
-            return result;
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndex.cs

@@ -1,70 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Collections.Generic;
-using System.Threading.Tasks;
-using Orleans;
-using Orleans.Concurrency;
-using Squidex.Domain.Apps.Entities.Apps;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Log;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public sealed class LuceneTextIndex : ITextIndex
-    {
-        private readonly IGrainFactory grainFactory;
-        private readonly IndexManager indexManager;
-
-        public LuceneTextIndex(IGrainFactory grainFactory, IndexManager indexManager)
-        {
-            Guard.NotNull(grainFactory, nameof(grainFactory));
-            Guard.NotNull(indexManager, nameof(indexManager));
-
-            this.grainFactory = grainFactory;
-
-            this.indexManager = indexManager;
-        }
-
-        public Task ClearAsync()
-        {
-            return indexManager.ClearAsync();
-        }
-
-        public async Task<List<Guid>?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope)
-        {
-            if (string.IsNullOrWhiteSpace(queryText))
-            {
-                return null;
-            }
-
-            var index = grainFactory.GetGrain<ILuceneTextIndexGrain>(app.Id);
-
-            using (Profiler.TraceMethod<LuceneTextIndex>())
-            {
-                var context = CreateContext(app, scope);
-
-                return await index.SearchAsync(queryText, filter, context);
-            }
-        }
-
-        private static SearchContext CreateContext(IAppEntity app, SearchScope scope)
-        {
-            var languages = new HashSet<string>(app.LanguagesConfig.AllKeys);
-
-            return new SearchContext { Languages = languages, Scope = scope };
-        }
-
-        public Task ExecuteAsync(NamedId<Guid> appId, NamedId<Guid> schemaId, params IndexCommand[] commands)
-        {
-            var index = grainFactory.GetGrain<ILuceneTextIndexGrain>(appId.Id);
-
-            return index.IndexAsync(schemaId, commands.AsImmutable());
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs

@@ -1,262 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Lucene.Net.Documents;
-using Lucene.Net.Index;
-using Lucene.Net.QueryParsers.Classic;
-using Lucene.Net.Search;
-using Lucene.Net.Util;
-using Orleans.Concurrency;
-using Squidex.Domain.Apps.Core;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Orleans;
-using Squidex.Infrastructure.Validation;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public sealed class LuceneTextIndexGrain : GrainOfGuid, ILuceneTextIndexGrain
-    {
-        private const LuceneVersion Version = LuceneVersion.LUCENE_48;
-        private const int MaxResults = 2000;
-        private const int MaxUpdates = 400;
-        private const string MetaId = "_id";
-        private const string MetaFor = "_fd";
-        private const string MetaContentId = "_cid";
-        private const string MetaSchemaId = "_si";
-        private const string MetaSchemaName = "_sn";
-        private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(10);
-        private static readonly string[] Invariant = { InvariantPartitioning.Key };
-        private readonly IndexManager indexManager;
-        private IDisposable? timer;
-        private IIndex index;
-        private QueryParser? queryParser;
-        private HashSet<string>? currentLanguages;
-        private int updates;
-
-        public LuceneTextIndexGrain(IndexManager indexManager)
-        {
-            Guard.NotNull(indexManager, nameof(indexManager));
-
-            this.indexManager = indexManager;
-        }
-
-        public override async Task OnDeactivateAsync()
-        {
-            if (index != null)
-            {
-                await CommitAsync();
-
-                await indexManager.ReleaseAsync(index);
-            }
-        }
-
-        protected override async Task OnActivateAsync(Guid key)
-        {
-            index = await indexManager.AcquireAsync(key);
-        }
-
-        public Task<List<Guid>> SearchAsync(string queryText, SearchFilter? filter, SearchContext context)
-        {
-            var result = new List<Guid>();
-
-            if (!string.IsNullOrWhiteSpace(queryText))
-            {
-                index.EnsureReader();
-
-                if (index.Searcher != null)
-                {
-                    var query = BuildQuery(queryText, filter, context);
-
-                    var hits = index.Searcher.Search(query, MaxResults).ScoreDocs;
-
-                    if (hits.Length > 0)
-                    {
-                        var buffer = new BytesRef(2);
-
-                        var found = new HashSet<Guid>();
-
-                        foreach (var hit in hits)
-                        {
-                            var forValue = index.Reader.GetBinaryValue(MetaFor, hit.Doc, buffer);
-
-                            if (context.Scope == SearchScope.All && forValue.Bytes[0] != 1)
-                            {
-                                continue;
-                            }
-
-                            if (context.Scope == SearchScope.Published && forValue.Bytes[1] != 1)
-                            {
-                                continue;
-                            }
-
-                            var document = index.Searcher.Doc(hit.Doc);
-
-                            if (document != null)
-                            {
-                                var idString = document.Get(MetaContentId);
-
-                                if (Guid.TryParse(idString, out var id))
-                                {
-                                    if (found.Add(id))
-                                    {
-                                        result.Add(id);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-            return Task.FromResult(result);
-        }
-
-        private Query BuildQuery(string query, SearchFilter? filter, SearchContext context)
-        {
-            if (queryParser == null || currentLanguages == null || !currentLanguages.SetEquals(context.Languages))
-            {
-                var fields = context.Languages.Union(Invariant).ToArray();
-
-                queryParser = new MultiFieldQueryParser(Version, fields, index.Analyzer);
-
-                currentLanguages = context.Languages;
-            }
-
-            try
-            {
-                var byQuery = queryParser.Parse(query);
-
-                if (filter?.SchemaIds.Count > 0)
-                {
-                    var bySchemas = new BooleanQuery
-                    {
-                        Boost = 2f
-                    };
-
-                    foreach (var schemaId in filter.SchemaIds)
-                    {
-                        var term = new Term(MetaSchemaId, schemaId.ToString());
-
-                        bySchemas.Add(new TermQuery(term), Occur.SHOULD);
-                    }
-
-                    var occur = filter.Must ? Occur.MUST : Occur.SHOULD;
-
-                    return new BooleanQuery
-                    {
-                        { byQuery, Occur.MUST },
-                        { bySchemas, occur }
-                    };
-                }
-
-                return byQuery;
-            }
-            catch (ParseException ex)
-            {
-                throw new ValidationException(ex.Message);
-            }
-        }
-
-        private async Task<bool> TryCommitAsync()
-        {
-            timer?.Dispose();
-
-            updates++;
-
-            if (updates >= MaxUpdates)
-            {
-                await CommitAsync();
-
-                return true;
-            }
-            else
-            {
-                index.MarkStale();
-
-                try
-                {
-                    timer = RegisterTimer(_ => CommitAsync(), null, CommitDelay, CommitDelay);
-                }
-                catch (InvalidOperationException)
-                {
-                    return false;
-                }
-            }
-
-            return false;
-        }
-
-        public async Task CommitAsync()
-        {
-            if (updates > 0)
-            {
-                await indexManager.CommitAsync(index);
-
-                updates = 0;
-            }
-        }
-
-        public Task IndexAsync(NamedId<Guid> schemaId, Immutable<IndexCommand[]> updates)
-        {
-            foreach (var command in updates.Value)
-            {
-                switch (command)
-                {
-                    case DeleteIndexEntry delete:
-                        index.Writer.DeleteDocuments(new Term(MetaId, delete.DocId));
-                        break;
-                    case UpdateIndexEntry update:
-                        try
-                        {
-                            var values = GetValue(update.ServeAll, update.ServePublished);
-
-                            index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, values);
-                        }
-                        catch (ArgumentException)
-                        {
-                        }
-
-                        break;
-                    case UpsertIndexEntry upsert:
-                        {
-                            var document = new Document();
-
-                            document.AddStringField(MetaId, upsert.DocId, Field.Store.YES);
-                            document.AddStringField(MetaContentId, upsert.ContentId.ToString(), Field.Store.YES);
-                            document.AddStringField(MetaSchemaId, schemaId.Id.ToString(), Field.Store.YES);
-                            document.AddStringField(MetaSchemaName, schemaId.Name, Field.Store.YES);
-                            document.AddBinaryDocValuesField(MetaFor, GetValue(upsert.ServeAll, upsert.ServePublished));
-
-                            foreach (var (key, value) in upsert.Texts)
-                            {
-                                document.AddTextField(key, value, Field.Store.NO);
-                            }
-
-                            index.Writer.UpdateDocument(new Term(MetaId, upsert.DocId), document);
-
-                            break;
-                        }
-                }
-            }
-
-            return TryCommitAsync();
-        }
-
-        private static BytesRef GetValue(bool forDraft, bool forPublished)
-        {
-            return new BytesRef(new[]
-            {
-                (byte)(forDraft ? 1 : 0),
-                (byte)(forPublished ? 1 : 0)
-            });
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/MultiLanguageAnalyzer.cs

@@ -1,65 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using Lucene.Net.Analysis;
-using Lucene.Net.Analysis.Standard;
-using Lucene.Net.Util;
-using Squidex.Infrastructure;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public sealed class MultiLanguageAnalyzer : AnalyzerWrapper
-    {
-        private readonly StandardAnalyzer fallbackAnalyzer;
-        private readonly Dictionary<string, Analyzer> analyzers = new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase);
-
-        public MultiLanguageAnalyzer(LuceneVersion version)
-            : base(PER_FIELD_REUSE_STRATEGY)
-        {
-            fallbackAnalyzer = new StandardAnalyzer(version);
-
-            foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes())
-            {
-                if (typeof(Analyzer).IsAssignableFrom(type))
-                {
-                    var language = type.Namespace!.Split('.').Last();
-
-                    if (language.Length == 2)
-                    {
-                        try
-                        {
-                            var analyzer = Activator.CreateInstance(type, version)!;
-
-                            analyzers[language] = (Analyzer)analyzer;
-                        }
-                        catch (MissingMethodException)
-                        {
-                            continue;
-                        }
-                    }
-                }
-            }
-        }
-
-        protected override Analyzer GetWrappedAnalyzer(string fieldName)
-        {
-            if (fieldName.Length >= 2)
-            {
-                var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer;
-
-                return analyzer;
-            }
-            else
-            {
-                return fallbackAnalyzer;
-            }
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs

@@ -1,114 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.IO;
-using System.IO.Compression;
-using System.Threading.Tasks;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-using Squidex.Infrastructure;
-using Squidex.Infrastructure.Assets;
-using LuceneDirectory = Lucene.Net.Store.Directory;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
-{
-    public sealed class AssetIndexStorage : IIndexStorage
-    {
-        private const string ArchiveFile = "Archive.zip";
-        private readonly IAssetStore assetStore;
-
-        public AssetIndexStorage(IAssetStore assetStore)
-        {
-            Guard.NotNull(assetStore, nameof(assetStore));
-
-            this.assetStore = assetStore;
-        }
-
-        public async Task<LuceneDirectory> CreateDirectoryAsync(Guid ownerId)
-        {
-            var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "LocalIndices", ownerId.ToString()));
-
-            if (directoryInfo.Exists)
-            {
-                directoryInfo.Delete(true);
-            }
-
-            directoryInfo.Create();
-
-            using (var fileStream = GetArchiveStream(directoryInfo))
-            {
-                try
-                {
-                    await assetStore.DownloadAsync(directoryInfo.Name, fileStream);
-
-                    fileStream.Position = 0;
-
-                    using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true))
-                    {
-                        zipArchive.ExtractToDirectory(directoryInfo.FullName);
-                    }
-                }
-                catch (AssetNotFoundException)
-                {
-                }
-            }
-
-            var directory = FSDirectory.Open(directoryInfo);
-
-            return directory;
-        }
-
-        public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
-        {
-            var directoryInfo = ((FSDirectory)directory).Directory;
-
-            var commit = snapshotter.Snapshot();
-            try
-            {
-                using (var fileStream = GetArchiveStream(directoryInfo))
-                {
-                    using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
-                    {
-                        foreach (var fileName in commit.FileNames)
-                        {
-                            var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName));
-
-                            zipArchive.CreateEntryFromFile(file.FullName, file.Name);
-                        }
-                    }
-
-                    fileStream.Position = 0;
-
-                    await assetStore.UploadAsync(directoryInfo.Name, fileStream, true);
-                }
-            }
-            finally
-            {
-                snapshotter.Release(commit);
-            }
-        }
-
-        private static FileStream GetArchiveStream(DirectoryInfo directoryInfo)
-        {
-            var path = Path.Combine(directoryInfo.FullName, ArchiveFile);
-
-            return new FileStream(
-                path,
-                FileMode.Create,
-                FileAccess.ReadWrite,
-                FileShare.None,
-                4096,
-                FileOptions.DeleteOnClose);
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs

@@ -1,37 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.IO;
-using System.Threading.Tasks;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-using LuceneDirectory = Lucene.Net.Store.Directory;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage
-{
-    public sealed class FileIndexStorage : IIndexStorage
-    {
-        public Task<LuceneDirectory> CreateDirectoryAsync(Guid ownerId)
-        {
-            var folderName = $"Indexes/{ownerId}";
-            var folderPath = Path.Combine(Path.GetTempPath(), folderName);
-
-            return Task.FromResult<LuceneDirectory>(FSDirectory.Open(folderPath));
-        }
-
-        public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter)
-        {
-            return Task.CompletedTask;
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs

@@ -1,23 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Threading.Tasks;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene
-{
-    public interface IIndexStorage
-    {
-        Task<Directory> CreateDirectoryAsync(Guid ownerId);
-
-        Task WriteAsync(Directory directory, SnapshotDeletionPolicy snapshotter);
-
-        Task ClearAsync();
-    }
-}

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs

@@ -6,6 +6,7 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Caching;
@@ -15,7 +16,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State
     public sealed class CachingTextIndexerState : ITextIndexerState
     {
         private readonly ITextIndexerState inner;
-        private LRUCache<Guid, Tuple<TextContentState?>> cache = new LRUCache<Guid, Tuple<TextContentState?>>(1000);
+        private readonly LRUCache<Guid, Tuple<TextContentState?>> cache = new LRUCache<Guid, Tuple<TextContentState?>>(10000);
 
         public CachingTextIndexerState(ITextIndexerState inner)
         {
@@ -28,37 +29,69 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State
         {
             await inner.ClearAsync();
 
-            cache = new LRUCache<Guid, Tuple<TextContentState?>>(1000);
+            cache.Clear();
         }
 
-        public async Task<TextContentState?> GetAsync(Guid contentId)
+        public async Task<Dictionary<Guid, TextContentState>> GetAsync(HashSet<Guid> ids)
         {
-            if (cache.TryGetValue(contentId, out var value))
+            Guard.NotNull(ids, nameof(ids));
+
+            var missingIds = new HashSet<Guid>();
+
+            var result = new Dictionary<Guid, TextContentState>();
+
+            foreach (var id in ids)
             {
-                return value.Item1;
+                if (cache.TryGetValue(id, out var state))
+                {
+                    if (state.Item1 != null)
+                    {
+                        result[id] = state.Item1;
+                    }
+                }
+                else
+                {
+                    missingIds.Add(id);
+                }
             }
 
-            var result = await inner.GetAsync(contentId);
+            if (missingIds.Count > 0)
+            {
+                var fromInner = await inner.GetAsync(missingIds);
 
-            cache.Set(contentId, Tuple.Create(result));
+                foreach (var (id, state) in fromInner)
+                {
+                    result[id] = state;
+                }
 
-            return result;
+                foreach (var id in missingIds)
-        }
+                {
+                    var state = fromInner.GetOrDefault(id);
 
-        public Task SetAsync(TextContentState state)
+                    cache.Set(id, Tuple.Create<TextContentState?>(state));
-        {
+                }
-            Guard.NotNull(state, nameof(state));
+            }
-
-            cache.Set(state.ContentId, Tuple.Create<TextContentState?>(state));
 
-            return inner.SetAsync(state);
+            return result;
         }
 
-        public Task RemoveAsync(Guid contentId)
+        public Task SetAsync(List<TextContentState> updates)
         {
-            cache.Set(contentId, Tuple.Create<TextContentState?>(null));
+            Guard.NotNull(updates, nameof(updates));
+
+            foreach (var update in updates)
+            {
+                if (update.IsDeleted)
+                {
+                    cache.Set(update.ContentId, Tuple.Create<TextContentState?>(null));
+                }
+                else
+                {
+                    cache.Set(update.ContentId, Tuple.Create<TextContentState?>(update));
+                }
+            }
 
-            return inner.RemoveAsync(contentId);
+            return inner.SetAsync(updates);
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs

@@ -6,17 +6,16 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text.State
 {
     public interface ITextIndexerState
     {
-        Task<TextContentState?> GetAsync(Guid contentId);
+        Task<Dictionary<Guid, TextContentState>> GetAsync(HashSet<Guid> ids);
 
-        Task SetAsync(TextContentState state);
+        Task SetAsync(List<TextContentState> updates);
-
-        Task RemoveAsync(Guid contentId);
 
         Task ClearAsync();
     }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs

@@ -8,6 +8,7 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
+using Squidex.Infrastructure;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text.State
 {
@@ -22,26 +23,38 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State
             return Task.CompletedTask;
         }
 
-        public Task<TextContentState?> GetAsync(Guid contentId)
+        public Task<Dictionary<Guid, TextContentState>> GetAsync(HashSet<Guid> ids)
         {
-            if (states.TryGetValue(contentId, out var result))
+            Guard.NotNull(ids, nameof(ids));
+
+            var result = new Dictionary<Guid, TextContentState>();
+
+            foreach (var id in ids)
             {
-                return Task.FromResult<TextContentState?>(result);
+                if (states.TryGetValue(id, out var state))
+                {
+                    result.Add(id, state);
+                }
             }
 
-            return Task.FromResult<TextContentState?>(null);
+            return Task.FromResult(result);
         }
 
-        public Task RemoveAsync(Guid contentId)
+        public Task SetAsync(List<TextContentState> updates)
         {
-            states.Remove(contentId);
+            Guard.NotNull(updates, nameof(updates));
-
-            return Task.CompletedTask;
-        }
 
-        public Task SetAsync(TextContentState state)
+            foreach (var update in updates)
-        {
+            {
-            states[state.ContentId] = state;
+                if (update.IsDeleted)
+                {
+                    states.Remove(update.ContentId);
+                }
+                else
+                {
+                    states[update.ContentId] = update;
+                }
+            }
 
             return Task.CompletedTask;
         }

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/TextContentState.cs

@@ -19,6 +19,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text.State
 
         public string? DocIdForPublished { get; set; }
 
+        public bool IsDeleted { get; set; }
+
         public void GenerateDocIdNew()
         {
             if (DocIdCurrent?.EndsWith("_2") != false)

backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs

@@ -5,6 +5,9 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
@@ -17,266 +20,332 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     public sealed class TextIndexingProcess : IEventConsumer
     {
         private const string NotFound = "<404>";
-        private readonly ITextIndex textIndexer;
+        private readonly ITextIndex textIndex;
         private readonly ITextIndexerState textIndexerState;
 
-        public string Name
+        public int BatchSize
         {
-            get { return "TextIndexer4"; }
+            get { return 1000; }
         }
 
-        public string EventsFilter
+        public int BatchDelay
         {
-            get { return "^content-"; }
+            get { return 1000; }
         }
 
-        public ITextIndex TextIndexer
+        public string Name
         {
-            get { return textIndexer; }
+            get { return "TextIndexer5"; }
         }
 
-        public TextIndexingProcess(ITextIndex textIndexer, ITextIndexerState textIndexerState)
+        public string EventsFilter
         {
-            Guard.NotNull(textIndexer, nameof(textIndexer));
+            get { return "^content-"; }
-            Guard.NotNull(textIndexerState, nameof(textIndexerState));
-
-            this.textIndexer = textIndexer;
-            this.textIndexerState = textIndexerState;
         }
 
-        public bool Handles(StoredEvent @event)
+        public ITextIndex TextIndex
         {
-            return true;
+            get { return textIndex; }
         }
 
-        public async Task ClearAsync()
+        private sealed class Updates
         {
-            await textIndexer.ClearAsync();
+            private readonly Dictionary<Guid, TextContentState> states;
-            await textIndexerState.ClearAsync();
+            private readonly Dictionary<Guid, TextContentState> updates = new Dictionary<Guid, TextContentState>();
-        }
+            private readonly Dictionary<string, IndexCommand> commands = new Dictionary<string, IndexCommand>();
 
-        public async Task On(Envelope<IEvent> @event)
+            public Updates(Dictionary<Guid, TextContentState> states)
-        {
-            switch (@event.Payload)
             {
-                case ContentCreated created:
+                this.states = states;
-                    await CreateAsync(created, created.Data);
+            }
-                    break;
-                case ContentUpdated updated:
-                    await UpdateAsync(updated, updated.Data);
-                    break;
-                case ContentStatusChanged statusChanged when statusChanged.Status == Status.Published:
-                    await PublishAsync(statusChanged);
-                    break;
-                case ContentStatusChanged statusChanged:
-                    await UnpublishAsync(statusChanged);
-                    break;
-                case ContentDraftDeleted draftDelted:
-                    await DeleteDraftAsync(draftDelted);
-                    break;
-                case ContentDeleted deleted:
-                    await DeleteAsync(deleted);
-                    break;
-                case ContentDraftCreated draftCreated:
-                    {
-                        await CreateDraftAsync(draftCreated);
 
-                        if (draftCreated.MigratedData != null)
+            public async Task WriteAsync(ITextIndex textIndex, ITextIndexerState textIndexerState)
+            {
+                if (commands.Count > 0)
+                {
+                    await textIndex.ExecuteAsync(commands.Values.ToArray());
+                }
+
+                if (updates.Count > 0)
+                {
+                    await textIndexerState.SetAsync(updates.Values.ToList());
+                }
+            }
+
+            public void On(Envelope<IEvent> @event)
+            {
+                switch (@event.Payload)
+                {
+                    case ContentCreated created:
+                        Create(created, created.Data);
+                        break;
+                    case ContentUpdated updated:
+                        Update(updated, updated.Data);
+                        break;
+                    case ContentStatusChanged statusChanged when statusChanged.Status == Status.Published:
+                        Publish(statusChanged);
+                        break;
+                    case ContentStatusChanged statusChanged:
+                        Unpublish(statusChanged);
+                        break;
+                    case ContentDraftDeleted draftDelted:
+                        DeleteDraft(draftDelted);
+                        break;
+                    case ContentDeleted deleted:
+                        Delete(deleted);
+                        break;
+                    case ContentDraftCreated draftCreated:
                         {
-                            await UpdateAsync(draftCreated, draftCreated.MigratedData);
+                            CreateDraft(draftCreated);
+
+                            if (draftCreated.MigratedData != null)
+                            {
+                                Update(draftCreated, draftCreated.MigratedData);
+                            }
                         }
-                    }
 
-                    break;
+                        break;
+                }
             }
-        }
 
-        private async Task CreateAsync(ContentEvent @event, NamedContentData data)
+            private void Create(ContentEvent @event, NamedContentData data)
-        {
-            var state = new TextContentState
             {
-                ContentId = @event.ContentId
+                var state = new TextContentState
-            };
+                {
+                    ContentId = @event.ContentId
+                };
 
-            state.GenerateDocIdCurrent();
+                state.GenerateDocIdCurrent();
 
-            await IndexAsync(@event,
+                Index(@event,
-                new UpsertIndexEntry
+                    new UpsertIndexEntry
-                {
+                    {
-                    ContentId = @event.ContentId,
+                        ContentId = @event.ContentId,
-                    DocId = state.DocIdCurrent,
+                        DocId = state.DocIdCurrent,
-                    ServeAll = true,
+                        ServeAll = true,
-                    ServePublished = false,
+                        ServePublished = false,
-                    Texts = data.ToTexts()
+                        Texts = data.ToTexts()
-                });
+                    });
-
-            await textIndexerState.SetAsync(state);
-        }
 
-        private async Task CreateDraftAsync(ContentEvent @event)
+                states[state.ContentId] = state;
-        {
+
-            var state = await textIndexerState.GetAsync(@event.ContentId);
+                updates[state.ContentId] = state;
+            }
 
-            if (state != null)
+            private void CreateDraft(ContentEvent @event)
             {
-                state.GenerateDocIdNew();
+                if (states.TryGetValue(@event.ContentId, out var state))
+                {
+                    state.GenerateDocIdNew();
 
-                await textIndexerState.SetAsync(state);
+                    updates[state.ContentId] = state;
+                }
             }
-        }
-
-        private async Task UpdateAsync(ContentEvent @event, NamedContentData data)
-        {
-            var state = await textIndexerState.GetAsync(@event.ContentId);
 
-            if (state != null)
+            private void Unpublish(ContentEvent @event)
             {
-                if (state.DocIdNew != null)
+                if (states.TryGetValue(@event.ContentId, out var state) && state.DocIdForPublished != null)
                 {
-                    await IndexAsync(@event,
+                    Index(@event,
-                        new UpsertIndexEntry
-                        {
-                            ContentId = @event.ContentId,
-                            DocId = state.DocIdNew,
-                            ServeAll = true,
-                            ServePublished = false,
-                            Texts = data.ToTexts()
-                        },
                         new UpdateIndexEntry
                         {
-                            DocId = state.DocIdCurrent,
+                            DocId = state.DocIdForPublished,
-                            ServeAll = false,
-                            ServePublished = true
-                        });
-                }
-                else
-                {
-                    var isPublished = state.DocIdCurrent == state.DocIdForPublished;
-
-                    await IndexAsync(@event,
-                        new UpsertIndexEntry
-                        {
-                            ContentId = @event.ContentId,
-                            DocId = state.DocIdCurrent,
                             ServeAll = true,
-                            ServePublished = isPublished,
+                            ServePublished = false
-                            Texts = data.ToTexts()
                         });
-                }
 
-                await textIndexerState.SetAsync(state);
+                    state.DocIdForPublished = null;
+
+                    updates[state.ContentId] = state;
+                }
             }
-        }
 
-        private async Task UnpublishAsync(ContentEvent @event)
+            private void Update(ContentEvent @event, NamedContentData data)
-        {
+            {
-            var state = await textIndexerState.GetAsync(@event.ContentId);
+                if (states.TryGetValue(@event.ContentId, out var state))
+                {
+                    if (state.DocIdNew != null)
+                    {
+                        Index(@event,
+                            new UpsertIndexEntry
+                            {
+                                ContentId = @event.ContentId,
+                                DocId = state.DocIdNew,
+                                ServeAll = true,
+                                ServePublished = false,
+                                Texts = data.ToTexts()
+                            });
+
+                        Index(@event,
+                            new UpdateIndexEntry
+                            {
+                                DocId = state.DocIdCurrent,
+                                ServeAll = false,
+                                ServePublished = true
+                            });
+                    }
+                    else
+                    {
+                        var isPublished = state.DocIdCurrent == state.DocIdForPublished;
+
+                        Index(@event,
+                            new UpsertIndexEntry
+                            {
+                                ContentId = @event.ContentId,
+                                DocId = state.DocIdCurrent,
+                                ServeAll = true,
+                                ServePublished = isPublished,
+                                Texts = data.ToTexts()
+                            });
+                    }
+                }
+            }
 
-            if (state != null && state.DocIdForPublished != null)
+            private void Publish(ContentEvent @event)
             {
-                await IndexAsync(@event,
+                if (states.TryGetValue(@event.ContentId, out var state))
-                    new UpdateIndexEntry
+                {
+                    if (state.DocIdNew != null)
                     {
-                        DocId = state.DocIdForPublished,
+                        Index(@event,
-                        ServeAll = true,
+                            new UpdateIndexEntry
-                        ServePublished = false
+                            {
-                    });
+                                DocId = state.DocIdNew,
+                                ServeAll = true,
+                                ServePublished = true
+                            });
+
+                        Index(@event,
+                            new DeleteIndexEntry
+                            {
+                                DocId = state.DocIdCurrent
+                            });
+
+                        state.DocIdForPublished = state.DocIdNew;
+                        state.DocIdCurrent = state.DocIdNew;
+                    }
+                    else
+                    {
+                        Index(@event,
+                            new UpdateIndexEntry
+                            {
+                                DocId = state.DocIdCurrent,
+                                ServeAll = true,
+                                ServePublished = true
+                            });
+
+                        state.DocIdForPublished = state.DocIdCurrent;
+                    }
 
-                state.DocIdForPublished = null;
+                    state.DocIdNew = null;
 
-                await textIndexerState.SetAsync(state);
+                    updates[state.ContentId] = state;
+                }
             }
-        }
 
-        private async Task PublishAsync(ContentEvent @event)
+            private void DeleteDraft(ContentEvent @event)
-        {
-            var state = await textIndexerState.GetAsync(@event.ContentId);
-
-            if (state != null)
             {
-                if (state.DocIdNew != null)
+                if (states.TryGetValue(@event.ContentId, out var state) && state.DocIdNew != null)
                 {
-                    await IndexAsync(@event,
+                    Index(@event,
                         new UpdateIndexEntry
                         {
-                            DocId = state.DocIdNew,
+                            DocId = state.DocIdCurrent,
                             ServeAll = true,
                             ServePublished = true
-                        },
+                        });
+
+                    Index(@event,
                         new DeleteIndexEntry
                         {
-                            DocId = state.DocIdCurrent
+                            DocId = state.DocIdNew
                         });
 
-                    state.DocIdForPublished = state.DocIdNew;
+                    state.DocIdNew = null;
-                    state.DocIdCurrent = state.DocIdNew;
+
+                    updates[state.ContentId] = state;
                 }
-                else
+            }
+
+            private void Delete(ContentEvent @event)
+            {
+                if (states.TryGetValue(@event.ContentId, out var state))
                 {
-                    await IndexAsync(@event,
+                    Index(@event,
-                        new UpdateIndexEntry
+                        new DeleteIndexEntry
                         {
-                            DocId = state.DocIdCurrent,
+                            DocId = state.DocIdCurrent
-                            ServeAll = true,
-                            ServePublished = true
                         });
 
-                    state.DocIdForPublished = state.DocIdCurrent;
+                    Index(@event,
+                        new DeleteIndexEntry
+                        {
+                            DocId = state.DocIdNew ?? NotFound
+                        });
+
+                    state.IsDeleted = true;
+
+                    updates[state.ContentId] = state;
                 }
+            }
 
-                state.DocIdNew = null;
+            private void Index(ContentEvent @event, IndexCommand command)
+            {
+                command.AppId = @event.AppId;
+                command.SchemaId = @event.SchemaId;
 
-                await textIndexerState.SetAsync(state);
+                if (command is UpdateIndexEntry update &&
+                    commands.TryGetValue(command.DocId, out var existing) &&
+                    existing is UpsertIndexEntry upsert)
+                {
+                    upsert.ServeAll = update.ServeAll;
+                    upsert.ServePublished = update.ServePublished;
+                }
+                else
+                {
+                    commands[command.DocId] = command;
+                }
             }
         }
 
-        private async Task DeleteDraftAsync(ContentEvent @event)
+        public TextIndexingProcess(ITextIndex textIndexer, ITextIndexerState textIndexerState)
         {
-            var state = await textIndexerState.GetAsync(@event.ContentId);
+            Guard.NotNull(textIndexer, nameof(textIndexer));
-
+            Guard.NotNull(textIndexerState, nameof(textIndexerState));
-            if (state != null && state.DocIdNew != null)
-            {
-                await IndexAsync(@event,
-                    new UpdateIndexEntry
-                    {
-                        DocId = state.DocIdCurrent,
-                        ServeAll = true,
-                        ServePublished = true
-                    },
-                    new DeleteIndexEntry
-                    {
-                        DocId = state.DocIdNew
-                    });
 
-                state.DocIdNew = null;
+            this.textIndex = textIndexer;
+            this.textIndexerState = textIndexerState;
+        }
 
-                await textIndexerState.SetAsync(state);
+        public async Task ClearAsync()
-            }
+        {
+            await textIndex.ClearAsync();
+            await textIndexerState.ClearAsync();
         }
 
-        private async Task DeleteAsync(ContentEvent @event)
+        public async Task On(IEnumerable<Envelope<IEvent>> @events)
         {
-            var state = await textIndexerState.GetAsync(@event.ContentId);
+            var states = await QueryStatesAsync(events);
 
-            if (state != null)
+            var updates = new Updates(states);
-            {
-                await IndexAsync(@event,
-                    new DeleteIndexEntry
-                    {
-                        DocId = state.DocIdCurrent
-                    },
-                    new DeleteIndexEntry
-                    {
-                        DocId = state.DocIdNew ?? NotFound
-                    });
 
-                await textIndexerState.RemoveAsync(state.ContentId);
+            foreach (var @event in events)
+            {
+                updates.On(@event);
             }
+
+            await updates.WriteAsync(textIndex, textIndexerState);
         }
 
-        private Task IndexAsync(ContentEvent @event, params IndexCommand[] commands)
+        private Task<Dictionary<Guid, TextContentState>> QueryStatesAsync(IEnumerable<Envelope<IEvent>> events)
         {
-            return textIndexer.ExecuteAsync(@event.AppId, @event.SchemaId, commands);
+            var ids =
+                events
+                    .Select(x => x.Payload).OfType<ContentEvent>()
+                    .Select(x => x.ContentId)
+                    .ToHashSet();
+
+            return textIndexerState.GetAsync(ids);
         }
     }
 }

backend/src/Squidex.Domain.Apps.Entities/History/HistoryService.cs

@@ -23,14 +23,19 @@ namespace Squidex.Domain.Apps.Entities.History
         private readonly IHistoryEventRepository repository;
         private readonly NotifoService notifo;
 
-        public string Name
+        public int BatchSize
         {
-            get { return GetType().Name; }
+            get { return 1000; }
         }
 
-        public string EventsFilter
+        public int BatchDelay
         {
-            get { return ".*"; }
+            get { return 1000; }
+        }
+
+        public string Name
+        {
+            get { return GetType().Name; }
         }
 
         public HistoryService(IHistoryEventRepository repository, IEnumerable<IHistoryEventsCreator> creators, NotifoService notifo)
@@ -54,38 +59,48 @@ namespace Squidex.Domain.Apps.Entities.History
             this.notifo = notifo;
         }
 
-        public bool Handles(StoredEvent @event)
-        {
-            return true;
-        }
-
         public Task ClearAsync()
         {
             return repository.ClearAsync();
         }
 
-        public async Task On(Envelope<IEvent> @event)
+        public async Task On(IEnumerable<Envelope<IEvent>> @events)
         {
-            await notifo.HandleEventAsync(@event);
+            var targets = new List<(Envelope<AppEvent> Event, HistoryEvent? HistoryEvent)>();
 
-            foreach (var creator in creators)
+            foreach (var @event in events)
             {
-                var historyEvent = await creator.CreateEventAsync(@event);
+                if (@event.Payload is AppEvent)
-
-                if (historyEvent != null)
                 {
                     var appEvent = @event.To<AppEvent>();
 
-                    await notifo.HandleHistoryEventAsync(appEvent, historyEvent);
+                    HistoryEvent? historyEvent = null;
+
+                    foreach (var creator in creators)
+                    {
+                        historyEvent = await creator.CreateEventAsync(@event);
+
+                        if (historyEvent != null)
+                        {
+                            historyEvent.Actor = appEvent.Payload.Actor;
+                            historyEvent.AppId = appEvent.Payload.AppId.Id;
+                            historyEvent.Created = @event.Headers.Timestamp();
+                            historyEvent.Version = @event.Headers.EventStreamNumber();
 
-                    historyEvent.Actor = appEvent.Payload.Actor;
+                            break;
-                    historyEvent.AppId = appEvent.Payload.AppId.Id;
+                        }
-                    historyEvent.Created = @event.Headers.Timestamp();
+                    }
-                    historyEvent.Version = @event.Headers.EventStreamNumber();
 
-                    await repository.InsertAsync(historyEvent);
+                    targets.Add((appEvent, historyEvent));
                 }
             }
+
+            if (targets.Count > 0)
+            {
+                await notifo.HandleEventsAsync(targets);
+
+                await repository.InsertManyAsync(targets.NotNull(x => x.HistoryEvent));
+            }
         }
 
         public async Task<IReadOnlyList<ParsedHistoryEvent>> QueryByChannelAsync(Guid appId, string channelPrefix, int count)

backend/src/Squidex.Domain.Apps.Entities/History/NotifoService.cs

@@ -5,6 +5,8 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Collections.Generic;
+using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Grpc.Core;
@@ -117,32 +119,33 @@ namespace Squidex.Domain.Apps.Entities.History
             await userResolver.SetClaimAsync(user.Id, SquidexClaimTypes.NotifoKey, response.User.ApiKey);
         }
 
-        public async Task HandleEventAsync(Envelope<IEvent> @event)
+        public async Task HandleEventsAsync(IEnumerable<(Envelope<AppEvent> AppEvent, HistoryEvent? HistoryEvent)> events)
         {
-            Guard.NotNull(@event, nameof(@event));
+            Guard.NotNull(events, nameof(events));
 
             if (client == null)
             {
                 return;
             }
 
-            switch (@event.Payload)
+            var now = clock.GetCurrentInstant();
+
+            var publishedEvents = events
+                .Where(x => IsTooOld(x.AppEvent.Headers, now) == false)
+                .Where(x => IsComment(x.AppEvent.Payload) || x.HistoryEvent != null)
+                .ToList();
+
+            if (publishedEvents.Any())
             {
-                case CommentCreated comment:
+                using (var stream = client.PublishMany())
+                {
+                    foreach (var @event in publishedEvents)
                     {
-                        if (IsTooOld(@event.Headers))
+                        var payload = @event.AppEvent.Payload;
-                        {
-                            return;
-                        }
 
-                        if (comment.Mentions == null || comment.Mentions.Length == 0)
+                        if (payload is CommentCreated comment && IsComment(payload))
                         {
-                            break;
+                            foreach (var userId in comment.Mentions!)
-                        }
-
-                        using (var stream = client.PublishMany())
-                        {
-                            foreach (var userId in comment.Mentions)
                             {
                                 var publishRequest = new PublishRequest
                                 {
@@ -164,52 +167,87 @@ namespace Squidex.Domain.Apps.Entities.History
 
                                 await stream.RequestStream.WriteAsync(publishRequest);
                             }
-
-                            await stream.RequestStream.CompleteAsync();
-                            await stream.ResponseAsync;
                         }
+                        else if (@event.HistoryEvent != null)
+                        {
+                            var historyEvent = @event.HistoryEvent;
 
-                        break;
+                            var publishRequest = new PublishRequest
-                    }
+                            {
+                                AppId = options.AppId
+                            };
 
-                case AppContributorAssigned contributorAssigned:
+                            foreach (var (key, value) in historyEvent.Parameters)
-                    {
+                            {
-                        var user = await userResolver.FindByIdAsync(contributorAssigned.ContributorId);
+                                publishRequest.Properties.Add(key, value);
+                            }
 
-                        if (user != null)
+                            publishRequest.Properties["SquidexApp"] = payload.AppId.Name;
-                        {
-                            await UpsertUserAsync(user);
-                        }
 
-                        var request = BuildAllowedTopicRequest(contributorAssigned, contributorAssigned.ContributorId);
+                            if (payload is ContentEvent c && !(payload is ContentDeleted))
+                            {
+                                var url = urlGenerator.ContentUI(c.AppId, c.SchemaId, c.ContentId);
 
-                        try
+                                publishRequest.Properties["SquidexUrl"] = url;
-                        {
+                            }
-                            await client.AddAllowedTopicAsync(request);
+
-                        }
+                            publishRequest.TemplateCode = @event.HistoryEvent.EventType;
-                        catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound)
-                        {
-                            break;
-                        }
 
-                        break;
+                            SetUser(payload, publishRequest);
+                            SetTopic(payload, publishRequest, historyEvent);
+
+                            await stream.RequestStream.WriteAsync(publishRequest);
+                        }
                     }
 
-                case AppContributorRemoved contributorRemoved:
+                    await stream.RequestStream.CompleteAsync();
-                    {
+                    await stream.ResponseAsync;
-                        var request = BuildAllowedTopicRequest(contributorRemoved, contributorRemoved.ContributorId);
+                }
+            }
 
-                        try
+            foreach (var @event in events)
+            {
+                switch (@event.AppEvent.Payload)
+                {
+                    case AppContributorAssigned contributorAssigned:
                         {
-                            await client.RemoveAllowedTopicAsync(request);
+                            var user = await userResolver.FindByIdAsync(contributorAssigned.ContributorId);
+
+                            if (user != null)
+                            {
+                                await UpsertUserAsync(user);
+                            }
+
+                            var request = BuildAllowedTopicRequest(contributorAssigned, contributorAssigned.ContributorId);
+
+                            try
+                            {
+                                await client.AddAllowedTopicAsync(request);
+                            }
+                            catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound)
+                            {
+                                break;
+                            }
+
+                            break;
                         }
-                        catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound)
+
+                    case AppContributorRemoved contributorRemoved:
                         {
+                            var request = BuildAllowedTopicRequest(contributorRemoved, contributorRemoved.ContributorId);
+
+                            try
+                            {
+                                await client.RemoveAllowedTopicAsync(request);
+                            }
+                            catch (RpcException ex) when (ex.StatusCode == StatusCode.NotFound)
+                            {
+                                break;
+                            }
+
                             break;
                         }
-
+                }
-                        break;
-                    }
             }
         }
 
@@ -226,52 +264,14 @@ namespace Squidex.Domain.Apps.Entities.History
             return topicRequest;
         }
 
-        public async Task HandleHistoryEventAsync(Envelope<AppEvent> @event, HistoryEvent historyEvent)
+        private static bool IsTooOld(EnvelopeHeaders headers, Instant now)
         {
-            if (client == null)
+            return now - headers.Timestamp() > MaxAge;
-            {
-                return;
-            }
-
-            if (IsTooOld(@event.Headers))
-            {
-                return;
-            }
-
-            var appEvent = @event.Payload;
-
-            var publishRequest = new PublishRequest
-            {
-                AppId = options.AppId
-            };
-
-            foreach (var (key, value) in historyEvent.Parameters)
-            {
-                publishRequest.Properties.Add(key, value);
-            }
-
-            publishRequest.Properties["SquidexApp"] = appEvent.AppId.Name;
-
-            if (appEvent is ContentEvent c && !(appEvent is ContentDeleted))
-            {
-                var url = urlGenerator.ContentUI(c.AppId, c.SchemaId, c.ContentId);
-
-                publishRequest.Properties["SquidexUrl"] = url;
-            }
-
-            publishRequest.TemplateCode = historyEvent.EventType;
-
-            SetUser(appEvent, publishRequest);
-            SetTopic(appEvent, publishRequest, historyEvent);
-
-            await client.PublishAsync(publishRequest);
         }
 
-        private bool IsTooOld(EnvelopeHeaders headers)
+        private static bool IsComment(AppEvent appEvent)
         {
-            var now = clock.GetCurrentInstant();
+            return appEvent is CommentCreated comment && comment.Mentions?.Length > 0;
-
-            return now - headers.Timestamp() > MaxAge;
         }
 
         private static void SetUser(AppEvent appEvent, PublishRequest publishRequest)

backend/src/Squidex.Domain.Apps.Entities/History/Repositories/IHistoryEventRepository.cs

@@ -15,7 +15,7 @@ namespace Squidex.Domain.Apps.Entities.History.Repositories
     {
         Task<IReadOnlyList<HistoryEvent>> QueryByChannelAsync(Guid appId, string channelPrefix, int count);
 
-        Task InsertAsync(HistoryEvent item);
+        Task InsertManyAsync(IEnumerable<HistoryEvent> historyEvents);
 
         Task ClearAsync();
     }

backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs

@@ -33,11 +33,6 @@ namespace Squidex.Domain.Apps.Entities.Rules
             get { return GetType().Name; }
         }
 
-        public string EventsFilter
-        {
-            get { return ".*"; }
-        }
-
         public RuleEnqueuer(IAppProvider appProvider, IMemoryCache cache, ILocalCache localCache, IRuleEventRepository ruleEventRepository,
             RuleService ruleService)
         {
@@ -55,16 +50,6 @@ namespace Squidex.Domain.Apps.Entities.Rules
             this.localCache = localCache;
         }
 
-        public bool Handles(StoredEvent @event)
-        {
-            return true;
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-
         public async Task Enqueue(Rule rule, Guid ruleId, Envelope<IEvent> @event)
         {
             Guard.NotNull(rule, nameof(rule));

backend/src/Squidex.Domain.Apps.Entities/Squidex.Domain.Apps.Entities.csproj

@@ -25,10 +25,6 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="GraphQL" Version="3.0.0" />
-    <PackageReference Include="Lucene.Net" Version="4.8.0-beta00005" />
-    <PackageReference Include="Lucene.Net.Analysis.Common" Version="4.8.0-beta00005" />
-    <PackageReference Include="Lucene.Net.Queries" Version="4.8.0-beta00005" />
-    <PackageReference Include="Lucene.Net.QueryParser" Version="4.8.0-beta00005" />
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="3.1.7" />
     <PackageReference Include="Microsoft.Orleans.CodeGenerator.MSBuild" Version="3.2.2">
       <PrivateAssets>all</PrivateAssets>

backend/src/Squidex.Infrastructure.Azure/EventSourcing/CosmosDbSubscription.cs

@@ -24,10 +24,9 @@ namespace Squidex.Infrastructure.EventSourcing
     internal sealed class CosmosDbSubscription : IEventSubscription, IChangeFeedObserverFactory, IChangeFeedObserver
     {
         private readonly TaskCompletionSource<bool> processorStopRequested = new TaskCompletionSource<bool>();
-        private readonly Task processorTask;
         private readonly CosmosDbEventStore store;
         private readonly Regex regex;
-        private readonly string? hostName;
+        private readonly string hostName;
         private readonly IEventSubscriber subscriber;
 
         public CosmosDbSubscription(CosmosDbEventStore store, IEventSubscriber subscriber, string? streamFilter, string? position = null)
@@ -42,7 +41,7 @@ namespace Squidex.Infrastructure.EventSourcing
             }
             else
             {
-                hostName = position;
+                hostName = position ?? "none";
             }
 
             if (!StreamFilter.IsAll(streamFilter))
@@ -52,7 +51,7 @@ namespace Squidex.Infrastructure.EventSourcing
 
             this.subscriber = subscriber;
 
-            processorTask = Task.Run(async () =>
+            Task.Run(async () =>
             {
                 try
                 {
@@ -128,7 +127,7 @@ namespace Squidex.Infrastructure.EventSourcing
 
                                 var eventData = @event.ToEventData();
 
-                                await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName ?? "None", eventStreamOffset, eventData));
+                                await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName, eventStreamOffset, eventData));
                             }
                         }
                     }
@@ -136,15 +135,9 @@ namespace Squidex.Infrastructure.EventSourcing
             }
         }
 
-        public void WakeUp()
+        public void Unsubscribe()
-        {
-        }
-
-        public Task StopAsync()
         {
             processorStopRequested.SetResult(true);
-
-            return processorTask;
         }
     }
 }

backend/src/Squidex.Infrastructure.GetEventStore/EventSourcing/GetEventStoreSubscription.cs

@@ -5,7 +5,6 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using System.Threading.Tasks;
 using EventStore.ClientAPI;
 using EventStore.ClientAPI.Exceptions;
 using Squidex.Infrastructure.Json;
@@ -44,15 +43,9 @@ namespace Squidex.Infrastructure.EventSourcing
             subscription = SubscribeToStream(streamName);
         }
 
-        public Task StopAsync()
+        public void Unsubscribe()
         {
             subscription.Stop();
-
-            return Task.CompletedTask;
-        }
-
-        public void WakeUp()
-        {
         }
 
         private EventStoreCatchUpSubscription SubscribeToStream(string streamName)

backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStoreSubscription.cs

@@ -11,6 +11,7 @@ using System.Threading.Tasks;
 using MongoDB.Bson;
 using MongoDB.Driver;
 using NodaTime;
+using Squidex.Infrastructure.Tasks;
 
 namespace Squidex.Infrastructure.EventSourcing
 {
@@ -19,14 +20,13 @@ namespace Squidex.Infrastructure.EventSourcing
         private readonly MongoEventStore eventStore;
         private readonly IEventSubscriber eventSubscriber;
         private readonly CancellationTokenSource stopToken = new CancellationTokenSource();
-        private readonly Task task;
 
         public MongoEventStoreSubscription(MongoEventStore eventStore, IEventSubscriber eventSubscriber, string? streamFilter, string? position)
         {
             this.eventStore = eventStore;
             this.eventSubscriber = eventSubscriber;
 
-            task = QueryAsync(streamFilter, position);
+            QueryAsync(streamFilter, position).Forget();
         }
 
         private async Task QueryAsync(string? streamFilter, string? position)
@@ -155,15 +155,9 @@ namespace Squidex.Infrastructure.EventSourcing
             return result;
         }
 
-        public Task StopAsync()
+        public void Unsubscribe()
         {
             stopToken.Cancel();
-
-            return task;
-        }
-
-        public void WakeUp()
-        {
         }
     }
 }

backend/src/Squidex.Infrastructure.MongoDb/EventSourcing/MongoEventStore_Reader.cs

@@ -138,7 +138,7 @@ namespace Squidex.Infrastructure.EventSourcing
         {
             using (Profiler.TraceMethod<MongoEventStore>())
             {
-                await Collection.Find(filter, options: Batching.Options).Sort(Sort.Ascending(TimestampField)).ForEachPipelineAsync(async commit =>
+                await Collection.Find(filter, options: Batching.Options).Sort(Sort.Ascending(TimestampField)).ForEachPipedAsync(async commit =>
                 {
                     foreach (var @event in commit.Filtered(position, predicate))
                     {

backend/src/Squidex.Infrastructure.MongoDb/MongoDb/MongoExtensions.cs

@@ -163,15 +163,15 @@ namespace Squidex.Infrastructure.MongoDb
             return BsonClassMap.LookupClassMap(typeof(TEntity)).GetMemberMap(nameof(IVersionedEntity<TKey>.Version)).ElementName;
         }
 
-        public static async Task ForEachPipelineAsync<TDocument>(this IAsyncCursorSource<TDocument> source, Func<TDocument, Task> processor, CancellationToken cancellationToken = default)
+        public static async Task ForEachPipedAsync<TDocument>(this IAsyncCursorSource<TDocument> source, Func<TDocument, Task> processor, CancellationToken cancellationToken = default)
         {
             using (var cursor = await source.ToCursorAsync(cancellationToken))
             {
-                await cursor.ForEachPipelineAsync(processor, cancellationToken);
+                await cursor.ForEachPipedAsync(processor, cancellationToken);
             }
         }
 
-        public static async Task ForEachPipelineAsync<TDocument>(this IAsyncCursor<TDocument> source, Func<TDocument, Task> processor, CancellationToken cancellationToken = default)
+        public static async Task ForEachPipedAsync<TDocument>(this IAsyncCursor<TDocument> source, Func<TDocument, Task> processor, CancellationToken cancellationToken = default)
         {
             using (var selfToken = new CancellationTokenSource())
             {

backend/src/Squidex.Infrastructure.MongoDb/States/MongoSnapshotStore.cs

@@ -71,7 +71,7 @@ namespace Squidex.Infrastructure.States
         {
             using (Profiler.TraceMethod<MongoSnapshotStore<T, TKey>>())
             {
-                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipelineAsync(x => callback(x.Doc, x.Version), ct);
+                await Collection.Find(new BsonDocument(), options: Batching.Options).ForEachPipedAsync(x => callback(x.Doc, x.Version), ct);
             }
         }
 

backend/src/Squidex.Infrastructure.RabbitMq/CQRS/Events/RabbitMqEventConsumer.cs

@@ -80,16 +80,6 @@ namespace Squidex.Infrastructure.CQRS.Events
             }
         }
 
-        public bool Handles(StoredEvent @event)
-        {
-            return true;
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.CompletedTask;
-        }
-
         public Task On(Envelope<IEvent> @event)
         {
             var jsonString = jsonSerializer.Serialize(@event);

backend/src/Squidex.Infrastructure/CollectionExtensions.cs

@@ -85,6 +85,11 @@ namespace Squidex.Infrastructure
             return source.Where(x => x != null)!;
         }
 
+        public static IEnumerable<TOut> NotNull<TIn, TOut>(this IEnumerable<TIn> source, Func<TIn, TOut?> selector) where TOut : class
+        {
+            return source.Select(selector).Where(x => x != null)!;
+        }
+
         public static IEnumerable<T> Concat<T>(this IEnumerable<T> source, T value)
         {
             return source.Concat(Enumerable.Repeat(value, 1));

backend/src/Squidex.Infrastructure/EventSourcing/CompoundEventConsumer.cs

@@ -1,77 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschränkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System.Linq;
-using System.Threading.Tasks;
-
-namespace Squidex.Infrastructure.EventSourcing
-{
-    public sealed class CompoundEventConsumer : IEventConsumer
-    {
-        private readonly IEventConsumer[] inners;
-
-        public string Name { get; }
-
-        public string EventsFilter { get; }
-
-        public CompoundEventConsumer(IEventConsumer first, params IEventConsumer[] inners)
-            : this(first?.Name!, first!, inners)
-        {
-        }
-
-        public CompoundEventConsumer(IEventConsumer[] inners)
-        {
-            Guard.NotNull(inners, nameof(inners));
-            Guard.NotEmpty(inners, nameof(inners));
-
-            this.inners = inners;
-
-            Name = inners.First().Name;
-
-            var innerFilters =
-                this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter))
-                    .Select(x => $"({x.EventsFilter})");
-
-            EventsFilter = string.Join("|", innerFilters);
-        }
-
-        public CompoundEventConsumer(string name, IEventConsumer first, params IEventConsumer[] inners)
-        {
-            Guard.NotNull(first, nameof(first));
-            Guard.NotNull(inners, nameof(inners));
-            Guard.NotNullOrEmpty(name, nameof(name));
-
-            this.inners = new[] { first }.Union(inners).ToArray();
-
-            Name = name;
-
-            var innerFilters =
-                this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter))
-                    .Select(x => $"({x.EventsFilter})");
-
-            EventsFilter = string.Join("|", innerFilters);
-        }
-
-        public bool Handles(StoredEvent @event)
-        {
-            return inners.Any(x => x.Handles(@event));
-        }
-
-        public Task ClearAsync()
-        {
-            return Task.WhenAll(inners.Select(i => i.ClearAsync()));
-        }
-
-        public async Task On(Envelope<IEvent> @event)
-        {
-            foreach (var inner in inners)
-            {
-                await inner.On(@event);
-            }
-        }
-    }
-}

backend/src/Squidex.Infrastructure/EventSourcing/Grains/BatchSubscriber.cs

@@ -0,0 +1,193 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using Squidex.Infrastructure.Reflection;
+using Squidex.Infrastructure.Tasks;
+
+namespace Squidex.Infrastructure.EventSourcing.Grains
+{
+    internal sealed class BatchSubscriber : IEventSubscriber
+    {
+        private readonly ITargetBlock<Job> pipelineStart;
+        private readonly IEventDataFormatter eventDataFormatter;
+        private readonly IEventSubscription eventSubscription;
+        private readonly IDataflowBlock pipelineEnd;
+
+        private sealed class Job
+        {
+            public StoredEvent? StoredEvent { get; set; }
+
+            public Exception? Exception { get; set; }
+
+            public Envelope<IEvent>? Event { get; set; }
+
+            public bool ShouldHandle { get; set; }
+
+            public object Sender { get; set; }
+        }
+
+        public BatchSubscriber(
+            EventConsumerGrain grain,
+            IEventDataFormatter eventDataFormatter,
+            IEventConsumer eventConsumer,
+            Func<IEventSubscriber, IEventSubscription> factory,
+            TaskScheduler scheduler)
+        {
+            this.eventDataFormatter = eventDataFormatter;
+
+            var batchSize = Math.Max(1, eventConsumer!.BatchSize);
+            var batchDelay = Math.Max(100, eventConsumer.BatchDelay);
+
+            var parse = new TransformBlock<Job, Job>(job =>
+            {
+                if (job.StoredEvent != null)
+                {
+                    job.ShouldHandle = eventConsumer.Handles(job.StoredEvent);
+                }
+
+                if (job.ShouldHandle)
+                {
+                    try
+                    {
+                        job.Event = ParseKnownEvent(job.StoredEvent!);
+                    }
+                    catch (Exception ex)
+                    {
+                        job.Exception = ex;
+                    }
+                }
+
+                return job;
+            }, new ExecutionDataflowBlockOptions
+            {
+                BoundedCapacity = batchSize,
+                MaxDegreeOfParallelism = 1,
+                MaxMessagesPerTask = 1
+            });
+
+            var buffer = AsyncHelper.CreateBatchBlock<Job>(batchSize, batchDelay, new GroupingDataflowBlockOptions
+            {
+                BoundedCapacity = batchSize * 2
+            });
+
+            var handle = new ActionBlock<IList<Job>>(async jobs =>
+            {
+                foreach (var jobsBySender in jobs.GroupBy<Job, object>(x => x.Sender))
+                {
+                    var sender = jobsBySender.Key;
+
+                    if (ReferenceEquals(sender, eventSubscription.Sender))
+                    {
+                        var exception = jobs.FirstOrDefault(x => x.Exception != null)?.Exception;
+
+                        if (exception != null)
+                        {
+                            await grain.OnErrorAsync(exception);
+                        }
+                        else
+                        {
+                            await grain.OnEventsAsync(GetEvents(jobsBySender), GetPosition(jobsBySender));
+                        }
+                    }
+                }
+            },
+            new ExecutionDataflowBlockOptions
+            {
+                BoundedCapacity = 2,
+                MaxDegreeOfParallelism = 1,
+                MaxMessagesPerTask = 1,
+                TaskScheduler = scheduler
+            });
+
+            parse.LinkTo(buffer, new DataflowLinkOptions
+            {
+                PropagateCompletion = true
+            });
+
+            buffer.LinkTo(handle, new DataflowLinkOptions
+            {
+                PropagateCompletion = true
+            });
+
+            pipelineStart = parse;
+            pipelineEnd = handle;
+
+            eventSubscription = factory(this);
+        }
+
+        private static List<Envelope<IEvent>> GetEvents(IEnumerable<Job> jobsBySender)
+        {
+            return jobsBySender.NotNull(x => x.Event).ToList();
+        }
+
+        private static string GetPosition(IEnumerable<Job> jobsBySender)
+        {
+            return jobsBySender.Last().StoredEvent!.EventPosition;
+        }
+
+        public Task CompleteAsync()
+        {
+            pipelineStart.Complete();
+
+            return pipelineEnd.Completion;
+        }
+
+        public void WakeUp()
+        {
+            eventSubscription.WakeUp();
+        }
+
+        public void Unsubscribe()
+        {
+            eventSubscription.Unsubscribe();
+        }
+
+        private Envelope<IEvent>? ParseKnownEvent(StoredEvent storedEvent)
+        {
+            try
+            {
+                var @event = eventDataFormatter.Parse(storedEvent.Data);
+
+                @event.SetEventPosition(storedEvent.EventPosition);
+                @event.SetEventStreamNumber(storedEvent.EventStreamNumber);
+
+                return @event;
+            }
+            catch (TypeNameNotFoundException)
+            {
+                return null;
+            }
+        }
+
+        public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
+        {
+            var job = new Job
+            {
+                Sender = subscription,
+                StoredEvent = storedEvent
+            };
+
+            return pipelineStart.SendAsync(job);
+        }
+
+        public Task OnErrorAsync(IEventSubscription subscription, Exception exception)
+        {
+            var job = new Job
+            {
+                Sender = subscription,
+                Exception = exception
+            };
+
+            return pipelineStart.SendAsync(job);
+        }
+    }
+}

backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerGrain.cs

@@ -6,15 +6,13 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
-using Orleans;
 using Orleans.Concurrency;
 using Squidex.Infrastructure.Log;
 using Squidex.Infrastructure.Orleans;
-using Squidex.Infrastructure.Reflection;
-using Squidex.Infrastructure.Tasks;
 
 namespace Squidex.Infrastructure.EventSourcing.Grains
 {
@@ -26,7 +24,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
         private readonly IEventStore eventStore;
         private readonly ISemanticLog log;
         private TaskScheduler? scheduler;
-        private IEventSubscription? currentSubscription;
+        private BatchSubscriber? currentSubscriber;
         private IEventConsumer? eventConsumer;
 
         private EventConsumerState State
@@ -65,6 +63,14 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             return Task.CompletedTask;
         }
 
+        public async Task CompleteAsync()
+        {
+            if (currentSubscriber != null)
+            {
+                await currentSubscriber.CompleteAsync();
+            }
+        }
+
         public Task<Immutable<EventConsumerInfo>> GetStateAsync()
         {
             return Task.FromResult(CreateInfo());
@@ -75,41 +81,23 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             return State.ToInfo(eventConsumer!.Name).AsImmutable();
         }
 
-        public Task OnEventAsync(Immutable<IEventSubscription> subscription, Immutable<StoredEvent> storedEvent)
+        public Task OnEventsAsync(IReadOnlyList<Envelope<IEvent>> events, string position)
         {
-            if (subscription.Value != currentSubscription)
-            {
-                return Task.CompletedTask;
-            }
-
             return DoAndUpdateStateAsync(async () =>
             {
-                if (eventConsumer!.Handles(storedEvent.Value))
+                await DispatchAsync(events);
-                {
-                    var @event = ParseKnownEvent(storedEvent.Value);
-
-                    if (@event != null)
-                    {
-                        await DispatchConsumerAsync(@event);
-                    }
-                }
 
-                State = State.Handled(storedEvent.Value.EventPosition);
+                State = State.Handled(position, events.Count);
             });
         }
 
-        public Task OnErrorAsync(Immutable<IEventSubscription> subscription, Immutable<Exception> exception)
+        public Task OnErrorAsync(Exception exception)
         {
-            if (subscription.Value != currentSubscription)
-            {
-                return Task.CompletedTask;
-            }
-
             return DoAndUpdateStateAsync(() =>
             {
                 Unsubscribe();
 
-                State = State.Stopped(exception.Value);
+                State = State.Stopped(exception);
             });
         }
 
@@ -119,14 +107,14 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             {
                 await DoAndUpdateStateAsync(() =>
                 {
-                    Subscribe(State.Position);
+                    Subscribe();
 
                     State = State.Started();
                 });
             }
             else if (!State.IsStopped)
             {
-                Subscribe(State.Position);
+                Subscribe();
             }
         }
 
@@ -139,7 +127,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await DoAndUpdateStateAsync(() =>
             {
-                Subscribe(State.Position);
+                Subscribe();
 
                 State = State.Started();
             });
@@ -172,21 +160,36 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
                 await ClearAsync();
 
-                Subscribe(null);
-
                 State = State.Reset();
+
+                Subscribe();
             });
 
             return CreateInfo();
         }
 
+        private async Task DispatchAsync(IReadOnlyList<Envelope<IEvent>> events)
+        {
+            if (events.Count > 0)
+            {
+                await eventConsumer!.On(events);
+            }
+        }
+
         private Task DoAndUpdateStateAsync(Action action, [CallerMemberName] string? caller = null)
         {
-            return DoAndUpdateStateAsync(() => { action(); return Task.CompletedTask; }, caller);
+            return DoAndUpdateStateAsync(() =>
+            {
+                action();
+
+                return Task.CompletedTask;
+            }, caller);
         }
 
         private async Task DoAndUpdateStateAsync(Func<Task> action, [CallerMemberName] string? caller = null)
         {
+            var previousState = State;
+
             try
             {
                 await action();
@@ -207,10 +210,13 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
                     .WriteProperty("status", "Failed")
                     .WriteProperty("eventConsumer", eventConsumer!.Name));
 
-                State = State.Stopped(ex);
+                State = previousState.Stopped(ex);
             }
 
-            await state.WriteAsync();
+            if (State != previousState)
+            {
+                await state.WriteAsync();
+            }
         }
 
         private async Task ClearAsync()
@@ -233,92 +239,43 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             }
         }
 
-        private async Task DispatchConsumerAsync(Envelope<IEvent> @event)
-        {
-            var eventId = @event.Headers.EventId().ToString();
-            var eventType = @event.Payload.GetType().Name;
-
-            var logContext = (eventId, eventType, consumer: eventConsumer!.Name);
-
-            log.LogDebug(logContext, (ctx, w) => w
-                .WriteProperty("action", "HandleEvent")
-                .WriteProperty("actionId", ctx.eventId)
-                .WriteProperty("status", "Started")
-                .WriteProperty("eventId", ctx.eventId)
-                .WriteProperty("eventType", ctx.eventType)
-                .WriteProperty("eventConsumer", ctx.consumer));
-
-            using (log.MeasureInformation(logContext, (ctx, w) => w
-                .WriteProperty("action", "HandleEvent")
-                .WriteProperty("actionId", ctx.eventId)
-                .WriteProperty("status", "Completed")
-                .WriteProperty("eventId", ctx.eventId)
-                .WriteProperty("eventType", ctx.eventType)
-                .WriteProperty("eventConsumer", ctx.consumer)))
-            {
-                await eventConsumer.On(@event);
-            }
-        }
-
         private void Unsubscribe()
         {
-            var subscription = Interlocked.Exchange(ref currentSubscription, null);
+            var subscription = Interlocked.Exchange(ref currentSubscriber, null);
 
-            if (subscription != null)
+            subscription?.Unsubscribe();
-            {
-                subscription.StopAsync().Forget();
-            }
         }
 
-        private void Subscribe(string? position)
+        private void Subscribe()
         {
-            if (currentSubscription == null)
+            if (currentSubscriber == null)
             {
-                currentSubscription = CreateSubscription(eventConsumer!.EventsFilter, position);
+                currentSubscriber = CreateSubscription();
             }
             else
             {
-                currentSubscription.WakeUp();
+                currentSubscriber.WakeUp();
             }
         }
 
-        private Envelope<IEvent>? ParseKnownEvent(StoredEvent storedEvent)
+        protected virtual TaskScheduler GetScheduler()
-        {
-            try
-            {
-                var @event = eventDataFormatter.Parse(storedEvent.Data);
-
-                @event.SetEventPosition(storedEvent.EventPosition);
-                @event.SetEventStreamNumber(storedEvent.EventStreamNumber);
-
-                return @event;
-            }
-            catch (TypeNameNotFoundException)
-            {
-                log.LogDebug(w => w.WriteProperty("oldEventFound", storedEvent.Data.Type));
-
-                return null;
-            }
-        }
-
-        protected virtual IEventConsumerGrain GetSelf()
         {
-            return this.AsReference<IEventConsumerGrain>();
+            return scheduler!;
         }
 
-        protected virtual IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string filter, string? position)
+        private BatchSubscriber CreateSubscription()
         {
-            return new RetrySubscription(store, subscriber, filter, position);
+            return new BatchSubscriber(this, eventDataFormatter, eventConsumer!, CreateRetrySubscription, GetScheduler());
         }
 
-        protected virtual TaskScheduler GetScheduler()
+        protected virtual IEventSubscription CreateRetrySubscription(IEventSubscriber subscriber)
         {
-            return scheduler!;
+            return new RetrySubscription(subscriber, CreateSubscription);
         }
 
-        private IEventSubscription CreateSubscription(string streamFilter, string? position)
+        protected virtual IEventSubscription CreateSubscription(IEventSubscriber subscriber)
         {
-            return CreateSubscription(eventStore, new WrapperSubscription(GetSelf(), GetScheduler()), streamFilter, position);
+            return eventStore.CreateSubscription(subscriber, eventConsumer!.EventsFilter, State.Position);
         }
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/Grains/EventConsumerState.cs

@@ -46,9 +46,9 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             return new EventConsumerState();
         }
 
-        public EventConsumerState Handled(string position)
+        public EventConsumerState Handled(string position, int offset = 1)
         {
-            return new EventConsumerState(position, Count + 1);
+            return new EventConsumerState(position, Count + offset);
         }
 
         public EventConsumerState Stopped(Exception? ex = null)

backend/src/Squidex.Infrastructure/EventSourcing/Grains/IEventConsumerGrain.cs

@@ -5,7 +5,6 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using System;
 using System.Threading.Tasks;
 using Orleans.Concurrency;
 using Squidex.Infrastructure.Orleans;
@@ -21,9 +20,5 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
         Task<Immutable<EventConsumerInfo>> StartAsync();
 
         Task<Immutable<EventConsumerInfo>> ResetAsync();
-
-        Task OnEventAsync(Immutable<IEventSubscription> subscription, Immutable<StoredEvent> storedEvent);
-
-        Task OnErrorAsync(Immutable<IEventSubscription> subscription, Immutable<Exception> exception);
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/Grains/WrapperSubscription.cs

@@ -1,42 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschränkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Threading;
-using System.Threading.Tasks;
-using Orleans.Concurrency;
-
-namespace Squidex.Infrastructure.EventSourcing.Grains
-{
-    internal sealed class WrapperSubscription : IEventSubscriber
-    {
-        private readonly IEventConsumerGrain grain;
-        private readonly TaskScheduler scheduler;
-
-        public WrapperSubscription(IEventConsumerGrain grain, TaskScheduler scheduler)
-        {
-            this.grain = grain;
-
-            this.scheduler = scheduler ?? TaskScheduler.Default;
-        }
-
-        public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
-        {
-            return Dispatch(() => grain.OnEventAsync(subscription.AsImmutable(), storedEvent.AsImmutable()));
-        }
-
-        public Task OnErrorAsync(IEventSubscription subscription, Exception exception)
-        {
-            return Dispatch(() => grain.OnErrorAsync(subscription.AsImmutable(), exception.AsImmutable()));
-        }
-
-        private Task Dispatch(Func<Task> task)
-        {
-            return Task<Task>.Factory.StartNew(task, CancellationToken.None, TaskCreationOptions.None, scheduler).Unwrap();
-        }
-    }
-}

backend/src/Squidex.Infrastructure/EventSourcing/IEventConsumer.cs

@@ -5,6 +5,7 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System.Collections.Generic;
 using System.Threading.Tasks;
 
 namespace Squidex.Infrastructure.EventSourcing
@@ -13,14 +14,35 @@ namespace Squidex.Infrastructure.EventSourcing
 
     public interface IEventConsumer
     {
+        int BatchDelay => 500;
+
+        int BatchSize => 1;
+
         string Name { get; }
 
-        string EventsFilter { get; }
+        string EventsFilter => ".*";
+
+        bool Handles(StoredEvent @event)
+        {
+            return true;
+        }
 
-        bool Handles(StoredEvent @event);
+        Task ClearAsync()
+        {
+            return Task.CompletedTask;
+        }
 
-        Task ClearAsync();
+        Task On(Envelope<IEvent> @event)
+        {
+            return Task.CompletedTask;
+        }
 
-        Task On(Envelope<IEvent> @event);
+        async Task On(IEnumerable<Envelope<IEvent>> @events)
+        {
+            foreach (var @event in events)
+            {
+                await On(@event);
+            }
+        }
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/IEventSubscription.cs

@@ -5,14 +5,18 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
-using System.Threading.Tasks;
-
 namespace Squidex.Infrastructure.EventSourcing
 {
     public interface IEventSubscription
     {
-        void WakeUp();
+        object? Sender => this;
+
+        void Unsubscribe()
+        {
+        }
 
-        Task StopAsync();
+        void WakeUp()
+        {
+        }
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/PollingSubscription.cs

@@ -6,7 +6,7 @@
 // ==========================================================================
 
 using System;
-using System.Threading.Tasks;
+using Squidex.Infrastructure.Tasks;
 using Squidex.Infrastructure.Timers;
 
 namespace Squidex.Infrastructure.EventSourcing
@@ -47,9 +47,9 @@ namespace Squidex.Infrastructure.EventSourcing
             timer.SkipCurrentDelay();
         }
 
-        public Task StopAsync()
+        public void Unsubscribe()
         {
-            return timer.StopAsync();
+            timer.StopAsync().Forget();
         }
     }
 }

backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs

@@ -8,7 +8,6 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
-using Squidex.Infrastructure.Tasks;
 
 #pragma warning disable RECS0002 // Convert anonymous method to method group
 
@@ -16,109 +15,78 @@ namespace Squidex.Infrastructure.EventSourcing
 {
     public sealed class RetrySubscription : IEventSubscription, IEventSubscriber
     {
-        private readonly SingleThreadedDispatcher dispatcher = new SingleThreadedDispatcher(10);
-        private readonly CancellationTokenSource timerCancellation = new CancellationTokenSource();
         private readonly RetryWindow retryWindow = new RetryWindow(TimeSpan.FromMinutes(5), 5);
-        private readonly IEventStore eventStore;
         private readonly IEventSubscriber eventSubscriber;
-        private readonly string? streamFilter;
+        private readonly Func<IEventSubscriber, IEventSubscription> eventSubscriptionFactory;
+        private CancellationTokenSource timerCancellation = new CancellationTokenSource();
         private IEventSubscription? currentSubscription;
-        private string? position;
 
         public int ReconnectWaitMs { get; set; } = 5000;
 
-        public RetrySubscription(IEventStore eventStore, IEventSubscriber eventSubscriber, string? streamFilter, string? position)
+        public object? Sender => currentSubscription?.Sender;
+
+        public RetrySubscription(IEventSubscriber eventSubscriber, Func<IEventSubscriber, IEventSubscription> eventSubscriptionFactory)
         {
-            Guard.NotNull(eventStore, nameof(eventStore));
             Guard.NotNull(eventSubscriber, nameof(eventSubscriber));
-            Guard.NotNull(streamFilter, nameof(streamFilter));
+            Guard.NotNull(eventSubscriptionFactory, nameof(eventSubscriptionFactory));
-
-            this.position = position;
 
-            this.eventStore = eventStore;
             this.eventSubscriber = eventSubscriber;
-
+            this.eventSubscriptionFactory = eventSubscriptionFactory;
-            this.streamFilter = streamFilter;
 
             Subscribe();
         }
 
         private void Subscribe()
         {
-            if (currentSubscription == null)
+            lock (this)
-            {
-                currentSubscription = eventStore.CreateSubscription(this, streamFilter, position);
-            }
-        }
-
-        private void Unsubscribe()
-        {
-            var subscription = Interlocked.Exchange(ref currentSubscription, null);
-
-            if (subscription != null)
             {
-                subscription.StopAsync().Forget();
+                if (currentSubscription == null)
+                {
+                    currentSubscription = eventSubscriptionFactory(this);
+                }
             }
         }
 
-        public void WakeUp()
+        public void Unsubscribe()
-        {
-            currentSubscription?.WakeUp();
-        }
-
-        private async Task HandleEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
         {
-            if (subscription == currentSubscription)
+            lock (this)
             {
-                await eventSubscriber.OnEventAsync(this, storedEvent);
+                if (currentSubscription != null)
-
+                {
-                position = storedEvent.EventPosition;
+                    timerCancellation.Cancel();
-            }
+                    timerCancellation.Dispose();
-        }
 
-        private async Task HandleErrorAsync(IEventSubscription subscription, Exception exception)
+                    currentSubscription.Unsubscribe();
-        {
+                    currentSubscription = null;
-            if (subscription == currentSubscription)
-            {
-                Unsubscribe();
 
-                if (retryWindow.CanRetryAfterFailure())
+                    timerCancellation = new CancellationTokenSource();
-                {
-                    RetryAsync().Forget();
-                }
-                else
-                {
-                    await eventSubscriber.OnErrorAsync(this, exception);
                 }
             }
         }
 
-        private async Task RetryAsync()
+        public void WakeUp()
         {
-            await Task.Delay(ReconnectWaitMs, timerCancellation.Token);
+            currentSubscription?.WakeUp();
-
-            await dispatcher.DispatchAsync(Subscribe);
         }
 
-        Task IEventSubscriber.OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
+        public async Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
         {
-            return dispatcher.DispatchAsync(() => HandleEventAsync(subscription, storedEvent));
+            await eventSubscriber.OnEventAsync(subscription, storedEvent);
         }
 
-        Task IEventSubscriber.OnErrorAsync(IEventSubscription subscription, Exception exception)
+        public async Task OnErrorAsync(IEventSubscription subscription, Exception exception)
         {
-            return dispatcher.DispatchAsync(() => HandleErrorAsync(subscription, exception));
+            Unsubscribe();
-        }
 
-        public async Task StopAsync()
+            if (retryWindow.CanRetryAfterFailure())
-        {
+            {
-            await dispatcher.DispatchAsync(Unsubscribe);
+                await Task.Delay(ReconnectWaitMs, timerCancellation.Token);
-            await dispatcher.StopAndWaitAsync();
 
-            if (!timerCancellation.IsCancellationRequested)
+                Subscribe();
+            }
+            else
             {
-                timerCancellation.Cancel();
+                await eventSubscriber.OnErrorAsync(subscription, exception);
-                timerCancellation.Dispose();
             }
         }
     }

backend/src/Squidex.Infrastructure/Tasks/AsyncHelper.cs

@@ -8,6 +8,7 @@
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
 
 namespace Squidex.Infrastructure.Tasks
 {
@@ -50,5 +51,37 @@ namespace Squidex.Infrastructure.Tasks
                 .GetAwaiter()
                 .GetResult();
         }
+
+        public static IPropagatorBlock<T, T[]> CreateBatchBlock<T>(int batchSize, int timeout, GroupingDataflowBlockOptions? dataflowBlockOptions = null)
+        {
+            dataflowBlockOptions ??= new GroupingDataflowBlockOptions();
+
+            var batchBlock = new BatchBlock<T>(batchSize, dataflowBlockOptions);
+
+            var timer = new Timer(_ => batchBlock.TriggerBatch());
+
+            var timerBlock = new TransformBlock<T, T>((T value) =>
+            {
+                timer.Change(timeout, Timeout.Infinite);
+
+                return value;
+            }, new ExecutionDataflowBlockOptions()
+            {
+                BoundedCapacity = 1,
+                CancellationToken = dataflowBlockOptions.CancellationToken,
+                EnsureOrdered = dataflowBlockOptions.EnsureOrdered,
+                MaxDegreeOfParallelism = 1,
+                MaxMessagesPerTask = dataflowBlockOptions.MaxMessagesPerTask,
+                NameFormat = dataflowBlockOptions.NameFormat,
+                TaskScheduler = dataflowBlockOptions.TaskScheduler
+            });
+
+            timerBlock.LinkTo(batchBlock, new DataflowLinkOptions()
+            {
+                PropagateCompletion = true
+            });
+
+            return DataflowBlock.Encapsulate(timerBlock, batchBlock);
+        }
     }
 }

backend/src/Squidex.Infrastructure/Tasks/SingleThreadedDispatcher.cs

@@ -1,67 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschränkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Threading.Tasks;
-using System.Threading.Tasks.Dataflow;
-
-namespace Squidex.Infrastructure.Tasks
-{
-    public sealed class SingleThreadedDispatcher
-    {
-        private readonly ActionBlock<Func<Task>> block;
-        private bool isStopped;
-
-        public SingleThreadedDispatcher(int capacity = 1)
-        {
-            var options = new ExecutionDataflowBlockOptions
-            {
-                BoundedCapacity = capacity,
-                MaxMessagesPerTask = 1,
-                MaxDegreeOfParallelism = 1
-            };
-
-            block = new ActionBlock<Func<Task>>(Handle, options);
-        }
-
-        public Task DispatchAsync(Func<Task> action)
-        {
-            Guard.NotNull(action, nameof(action));
-
-            return block.SendAsync(action);
-        }
-
-        public Task DispatchAsync(Action action)
-        {
-            Guard.NotNull(action, nameof(action));
-
-            return block.SendAsync(() => { action(); return Task.CompletedTask; });
-        }
-
-        public async Task StopAndWaitAsync()
-        {
-            await DispatchAsync(() =>
-            {
-                isStopped = true;
-
-                block.Complete();
-            });
-
-            await block.Completion;
-        }
-
-        private Task Handle(Func<Task> action)
-        {
-            if (isStopped)
-            {
-                return Task.CompletedTask;
-            }
-
-            return action();
-        }
-    }
-}

backend/src/Squidex.Infrastructure/Translations/ResourcesLocalizer.cs

@@ -113,11 +113,16 @@ namespace Squidex.Infrastructure.Translations
                     {
                         try
                         {
-                            variableValue = Convert.ToString(property.GetValue(args), culture);
+                            var value = property.GetValue(args);
+
+                            if (value != null)
+                            {
+                                variableValue = Convert.ToString(value, culture) ?? variableName;
+                            }
                         }
                         catch
                         {
-                            variableValue = null;
+                            variableValue = variableName;
                         }
                     }
 

backend/src/Squidex/Config/Domain/ContentsServices.cs

@@ -13,7 +13,7 @@ using Squidex.Domain.Apps.Entities.Contents;
 using Squidex.Domain.Apps.Entities.Contents.Queries;
 using Squidex.Domain.Apps.Entities.Contents.Queries.Steps;
 using Squidex.Domain.Apps.Entities.Contents.Text;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
+using Squidex.Domain.Apps.Entities.Contents.Text.Elastic;
 using Squidex.Domain.Apps.Entities.Contents.Validation;
 using Squidex.Domain.Apps.Entities.History;
 using Squidex.Domain.Apps.Entities.Search;
@@ -86,20 +86,27 @@ namespace Squidex.Config.Domain
             services.AddSingletonAs<DefaultWorkflowsValidator>()
                 .AsOptional<IWorkflowsValidator>();
 
-            services.AddSingletonAs<LuceneTextIndex>()
-                .As<ITextIndex>();
-
             services.AddSingletonAs<TextIndexingProcess>()
                 .As<IEventConsumer>();
 
             services.AddSingletonAs<ContentsSearchSource>()
                 .As<ISearchSource>();
 
-            services.AddSingletonAs<IndexManager>()
-                .AsSelf();
-
             services.AddSingletonAs<GrainBootstrap<IContentSchedulerGrain>>()
                 .AsSelf();
+
+            config.ConfigureByOption("fullText:type", new Alternatives
+            {
+                ["Elastic"] = () =>
+                {
+                    var elasticConfiguration = config.GetRequiredValue("fullText:elastic:configuration");
+                    var elasticIndexName = config.GetRequiredValue("fullText:elastic:indexName");
+
+                    services.AddSingletonAs(c => new ElasticSearchTextIndex(elasticConfiguration, elasticIndexName))
+                        .As<ITextIndex>();
+                },
+                ["Default"] = () => { }
+            });
         }
     }
 }

backend/src/Squidex/Config/Domain/StoreServices.cs

@@ -14,12 +14,11 @@ using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Migrations.Migrations.MongoDb;
 using MongoDB.Driver;
-using MongoDB.Driver.GridFS;
 using Squidex.Domain.Apps.Entities.Assets.Repositories;
 using Squidex.Domain.Apps.Entities.Assets.State;
 using Squidex.Domain.Apps.Entities.Contents.Repositories;
 using Squidex.Domain.Apps.Entities.Contents.State;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
+using Squidex.Domain.Apps.Entities.Contents.Text;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
 using Squidex.Domain.Apps.Entities.History.Repositories;
 using Squidex.Domain.Apps.Entities.MongoDb.Assets;
@@ -118,6 +117,9 @@ namespace Squidex.Config.Domain
                     services.AddSingletonAs(c => ActivatorUtilities.CreateInstance<MongoContentRepository>(c, GetDatabase(c, mongoContentDatabaseName)))
                         .As<IContentRepository>().As<ISnapshotStore<ContentState, Guid>>();
 
+                    services.AddSingletonAs<MongoTextIndex>()
+                        .AsOptional<ITextIndex>();
+
                     services.AddSingletonAs<MongoTextIndexerState>()
                         .As<ITextIndexerState>();
 
@@ -128,18 +130,6 @@ namespace Squidex.Config.Domain
                         services.AddSingletonAs<MongoPersistedGrantStore>()
                             .As<IPersistedGrantStore>();
                     }
-
-                    services.AddSingletonAs(c =>
-                    {
-                        var database = c.GetRequiredService<IMongoDatabase>();
-
-                        var mongoBucket = new GridFSBucket<string>(database, new GridFSBucketOptions
-                        {
-                            BucketName = "fullText"
-                        });
-
-                        return new MongoIndexStorage(mongoBucket);
-                    }).As<IIndexStorage>();
                 }
             });
 

backend/src/Squidex/appsettings.json

@@ -23,6 +23,27 @@
     "enableXForwardedHost": false
   },
 
+  "fullText": {
+    /*
+     * Define the type of the full text store.
+     * 
+     * Supported: elastic (ElasticSearch). Default: MongoDB
+     */
+    "type": "default",
+    "elastic": {
+      /*
+       * The configuration to your elastic search cluster.
+       * 
+       * Read More: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-configuration.html
+       */
+      "configuration": "http://localhost:9200",
+      /*
+       * The name of the index.
+       */
+      "indexName": "squidex"
+    }
+  },
+
   /*
    * Define optional paths to plugins.
    */

backend/tests/Squidex.Domain.Apps.Entities.Tests/Assets/RecursiveDeleterTests.cs

@@ -38,10 +38,28 @@ namespace Squidex.Domain.Apps.Entities.Assets
             sut = new RecursiveDeleter(commandBus, assetRepository, assetFolderRepository, typeNameRegistry, log);
         }
 
+        [Fact]
+        public void Should_return_assets_filter_for_events_filter()
+        {
+            IEventConsumer consumer = sut;
+
+            Assert.Equal("^assetFolder\\-", consumer.EventsFilter);
+        }
+
         [Fact]
         public async Task Should_do_nothing_on_clear()
         {
-            await sut.ClearAsync();
+            IEventConsumer consumer = sut;
+
+            await consumer.ClearAsync();
+        }
+
+        [Fact]
+        public void Should_return_type_name_for_name()
+        {
+            IEventConsumer consumer = sut;
+
+            Assert.Equal(typeof(RecursiveDeleter).Name, consumer.Name);
         }
 
         [Fact]

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs

@@ -6,9 +6,12 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using FakeItEasy;
 using Squidex.Domain.Apps.Entities.Contents.Text.State;
+using Squidex.Domain.Apps.Entities.TestHelpers;
+using Squidex.Infrastructure;
 using Xunit;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -27,19 +30,46 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         public async Task Should_retrieve_from_inner_when_not_cached()
         {
             var contentId = Guid.NewGuid();
+            var contentIds = HashSet.Of(contentId);
 
             var state = new TextContentState { ContentId = contentId };
 
-            A.CallTo(() => inner.GetAsync(contentId))
+            var states = new Dictionary<Guid, TextContentState>
-                .Returns(state);
+            {
+                [contentId] = state
+            };
 
-            var found1 = await sut.GetAsync(contentId);
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>.That.Is(contentIds)))
-            var found2 = await sut.GetAsync(contentId);
+                .Returns(states);
 
-            Assert.Same(state, found1);
+            var found1 = await sut.GetAsync(HashSet.Of(contentId));
-            Assert.Same(state, found2);
+            var found2 = await sut.GetAsync(HashSet.Of(contentId));
 
-            A.CallTo(() => inner.GetAsync(contentId))
+            Assert.Same(state, found1[contentId]);
+            Assert.Same(state, found2[contentId]);
+
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>.That.Is(contentIds)))
+                .MustHaveHappenedOnceExactly();
+        }
+
+        [Fact]
+        public async Task Should_retrieve_from_inner_when_not_cached_and_not_found()
+        {
+            var contentId = Guid.NewGuid();
+            var contentIds = HashSet.Of(contentId);
+
+            var state = new TextContentState { ContentId = contentId };
+
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>.That.Is(contentIds)))
+                .Returns(new Dictionary<Guid, TextContentState>());
+
+            var found1 = await sut.GetAsync(HashSet.Of(contentId));
+            var found2 = await sut.GetAsync(HashSet.Of(contentId));
+
+            Assert.Empty(found1);
+            Assert.Empty(found2);
+
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>.That.Is(contentIds)))
                 .MustHaveHappenedOnceExactly();
         }
 
@@ -47,21 +77,25 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         public async Task Should_not_retrieve_from_inner_when_cached()
         {
             var contentId = Guid.NewGuid();
+            var contentIds = HashSet.Of(contentId);
 
             var state = new TextContentState { ContentId = contentId };
 
-            await sut.SetAsync(state);
+            await sut.SetAsync(new List<TextContentState>
+            {
+                state
+            });
 
-            var found1 = await sut.GetAsync(contentId);
+            var found1 = await sut.GetAsync(contentIds);
-            var found2 = await sut.GetAsync(contentId);
+            var found2 = await sut.GetAsync(contentIds);
 
-            Assert.Same(state, found1);
+            Assert.Same(state, found1[contentId]);
-            Assert.Same(state, found2);
+            Assert.Same(state, found2[contentId]);
 
-            A.CallTo(() => inner.SetAsync(state))
+            A.CallTo(() => inner.SetAsync(A<List<TextContentState>>.That.IsSameSequenceAs(state)))
                 .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => inner.GetAsync(contentId))
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>._))
                 .MustNotHaveHappened();
         }
 
@@ -69,22 +103,30 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         public async Task Should_not_retrieve_from_inner_when_removed()
         {
             var contentId = Guid.NewGuid();
+            var contentIds = HashSet.Of(contentId);
 
             var state = new TextContentState { ContentId = contentId };
 
-            await sut.SetAsync(state);
+            await sut.SetAsync(new List<TextContentState>
-            await sut.RemoveAsync(contentId);
+            {
+                state
+            });
+
+            await sut.SetAsync(new List<TextContentState>
+            {
+                new TextContentState { ContentId = contentId, IsDeleted = true }
+            });
 
-            var found1 = await sut.GetAsync(contentId);
+            var found1 = await sut.GetAsync(contentIds);
-            var found2 = await sut.GetAsync(contentId);
+            var found2 = await sut.GetAsync(contentIds);
 
-            Assert.Null(found1);
+            Assert.Empty(found1);
-            Assert.Null(found2);
+            Assert.Empty(found2);
 
-            A.CallTo(() => inner.RemoveAsync(contentId))
+            A.CallTo(() => inner.SetAsync(A<List<TextContentState>>.That.Matches(x => x.Count == 1 && x[0].IsDeleted)))
                 .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => inner.GetAsync(contentId))
+            A.CallTo(() => inner.GetAsync(A<HashSet<Guid>>._))
                 .MustNotHaveHappened();
         }
     }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/DocValuesTests.cs

@@ -1,56 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using Lucene.Net.Analysis.Standard;
-using Lucene.Net.Documents;
-using Lucene.Net.Index;
-using Lucene.Net.Store;
-using Lucene.Net.Util;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
-using Xunit;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text
-{
-    public class DocValuesTests
-    {
-        [Fact]
-        public void Should_read_and_write_doc_values()
-        {
-            var version = LuceneVersion.LUCENE_48;
-
-            var indexWriter =
-                new IndexWriter(new RAMDirectory(),
-                    new IndexWriterConfig(version, new StandardAnalyzer(version)));
-
-            using (indexWriter)
-            {
-                for (byte i = 0; i < 255; i++)
-                {
-                    var document = new Document();
-
-                    document.AddBinaryDocValuesField("field", new BytesRef(new[] { i }));
-
-                    indexWriter.AddDocument(document);
-                }
-
-                indexWriter.Commit();
-
-                using (var reader = indexWriter.GetReader(true))
-                {
-                    var bytesRef = new BytesRef(1);
-
-                    for (byte i = 0; i < 255; i++)
-                    {
-                        reader.GetBinaryValue("field", i, bytesRef);
-
-                        Assert.Equal(i, bytesRef.Bytes[0]);
-                    }
-                }
-            }
-        }
-    }
-}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/LuceneIndexFactory.cs

@@ -1,50 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using System.Threading.Tasks;
-using FakeItEasy;
-using Orleans;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
-using Squidex.Infrastructure.Log;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text
-{
-    public sealed class LuceneIndexFactory : IIndexerFactory
-    {
-        private readonly IGrainFactory grainFactory = A.Fake<IGrainFactory>();
-        private readonly IIndexStorage storage;
-        private LuceneTextIndexGrain grain;
-
-        public LuceneIndexFactory(IIndexStorage storage)
-        {
-            this.storage = storage;
-
-            A.CallTo(() => grainFactory.GetGrain<ILuceneTextIndexGrain>(A<Guid>._, null))
-                .ReturnsLazily(() => grain);
-        }
-
-        public async Task<ITextIndex> CreateAsync(Guid schemaId)
-        {
-            var indexManager = new IndexManager(storage, A.Fake<ISemanticLog>());
-
-            grain = new LuceneTextIndexGrain(indexManager);
-
-            await grain.ActivateAsync(schemaId);
-
-            return new LuceneTextIndex(grainFactory, indexManager);
-        }
-
-        public async Task CleanupAsync()
-        {
-            if (grain != null)
-            {
-                await grain.OnDeactivateAsync();
-            }
-        }
-    }
-}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TestStorages.cs

@@ -1,49 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System;
-using MongoDB.Driver;
-using MongoDB.Driver.GridFS;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene;
-using Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage;
-using Squidex.Domain.Apps.Entities.MongoDb.FullText;
-using Squidex.Infrastructure.Assets;
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text
-{
-    public static class TestStorages
-    {
-        public static IIndexStorage Assets()
-        {
-            var storage = new AssetIndexStorage(new MemoryAssetStore());
-
-            return storage;
-        }
-
-        public static IIndexStorage TempFolder()
-        {
-            var storage = new FileIndexStorage();
-
-            return storage;
-        }
-
-        public static IIndexStorage MongoDB()
-        {
-            var mongoClient = new MongoClient("mongodb://localhost");
-            var mongoDatabase = mongoClient.GetDatabase("FullText");
-
-            var mongoBucket = new GridFSBucket<string>(mongoDatabase, new GridFSBucketOptions
-            {
-                BucketName = $"bucket_{DateTime.UtcNow.Ticks}"
-            });
-
-            var storage = new MongoIndexStorage(mongoBucket);
-
-            return storage;
-        }
-    }
-}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs

@@ -37,49 +37,108 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public abstract IIndexerFactory Factory { get; }
 
+        public virtual bool SupportsCleanup { get; set; } = false;
+
+        public virtual bool SupportsSearchSyntax { get; set; } = true;
+
+        public virtual bool SupportsMultiLanguage { get; set; } = true;
+
         public virtual InMemoryTextIndexerState State { get; } = new InMemoryTextIndexerState();
 
         protected TextIndexerTestsBase()
         {
             app =
-                Mocks.App(NamedId.Of(Guid.NewGuid(), "my-app"),
+                Mocks.App(appId,
                     Language.DE,
                     Language.EN);
         }
 
-        [Fact]
+        [SkippableFact]
         public async Task Should_throw_exception_for_invalid_query()
         {
+            Skip.IfNot(SupportsSearchSyntax);
+
             await Assert.ThrowsAsync<ValidationException>(async () =>
             {
                 await TestCombinations(Search(expected: null, text: "~hello"));
             });
         }
 
-        [Fact]
+        [SkippableFact]
-        public async Task Should_index_invariant_content_and_retrieve()
+        public async Task Should_index_invariant_content_and_retrieve_with_fuzzy()
         {
+            Skip.IfNot(SupportsSearchSyntax);
+
             await TestCombinations(
                 Create(ids1[0], "iv", "Hello"),
                 Create(ids2[0], "iv", "World"),
 
-                Search(expected: ids1, text: "Hello"),
+                Search(expected: ids1, text: "helo~"),
-                Search(expected: ids2, text: "World"),
+                Search(expected: ids2, text: "wold~", SearchScope.All)
+            );
+        }
 
-                Search(expected: null, text: "Hello", SearchScope.Published),
+        [SkippableFact]
-                Search(expected: null, text: "World", SearchScope.Published)
+        public async Task Should_search_by_field()
+        {
+            Skip.IfNot(SupportsSearchSyntax);
+
+            await TestCombinations(
+                Create(ids1[0], "en", "City"),
+                Create(ids2[0], "de", "Stadt"),
+
+                Search(expected: ids1, text: "en:city"),
+                Search(expected: ids2, text: "de:Stadt")
             );
         }
 
         [Fact]
-        public async Task Should_index_invariant_content_and_retrieve_with_fuzzy()
+        public async Task Should_index_localized_content_and_retrieve()
+        {
+            if (SupportsMultiLanguage)
+            {
+                await TestCombinations(
+                    Create(ids1[0], "de", "Stadt und Land and Fluss"),
+
+                    Create(ids2[0], "en", "City and Country und River"),
+
+                    Search(expected: ids1, text: "Stadt"),
+                    Search(expected: ids2, text: "City"),
+
+                    Search(expected: ids1, text: "and"),
+                    Search(expected: ids2, text: "und")
+                );
+            }
+            else
+            {
+                var both = ids2.Union(ids1).ToList();
+
+                await TestCombinations(
+                    Create(ids1[0], "de", "Stadt und Land and Fluss"),
+
+                    Create(ids2[0], "en", "City and Country und River"),
+
+                    Search(expected: ids1, text: "Stadt"),
+                    Search(expected: ids2, text: "City"),
+
+                    Search(expected: null, text: "and"),
+                    Search(expected: both, text: "und")
+                );
+            }
+        }
+
+        [Fact]
+        public async Task Should_index_invariant_content_and_retrieve()
         {
             await TestCombinations(
                 Create(ids1[0], "iv", "Hello"),
                 Create(ids2[0], "iv", "World"),
 
-                Search(expected: ids1, text: "helo~"),
+                Search(expected: ids1, text: "Hello"),
-                Search(expected: ids2, text: "wold~", SearchScope.All)
+                Search(expected: ids2, text: "World"),
+
+                Search(expected: null, text: "Hello", SearchScope.Published),
+                Search(expected: null, text: "World", SearchScope.Published)
             );
         }
 
@@ -282,36 +341,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             );
         }
 
-        [Fact]
-        public async Task Should_search_by_field()
-        {
-            await TestCombinations(
-                Create(ids1[0], "en", "City"),
-                Create(ids2[0], "de", "Stadt"),
-
-                Search(expected: ids1, text: "en:city"),
-                Search(expected: ids2, text: "de:Stadt")
-            );
-        }
-
-        [Fact]
-        public async Task Should_index_localized_content_and_retrieve()
-        {
-            await TestCombinations(
-                Create(ids1[0], "de", "Stadt und Land and Fluss"),
-
-                Create(ids2[0], "en", "City and Country und River"),
-
-                Search(expected: ids1, text: "Stadt"),
-                Search(expected: ids1, text: "and"),
-                Search(expected: ids2, text: "und"),
-
-                Search(expected: ids2, text: "City"),
-                Search(expected: ids2, text: "und"),
-                Search(expected: ids1, text: "and")
-            );
-        }
-
         private IndexOperation Create(Guid id, string language, string text)
         {
             var data =
@@ -376,7 +405,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             contentEvent.AppId = appId;
             contentEvent.SchemaId = schemaId;
 
-            return p => p.On(Envelope.Create(contentEvent));
+            return p => p.On(Enumerable.Repeat(Envelope.Create<IEvent>(contentEvent), 1));
         }
 
         private IndexOperation Search(List<Guid>? expected, string text, SearchScope target = SearchScope.All)
@@ -385,7 +414,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             {
                 var searchFilter = SearchFilter.ShouldHaveSchemas(schemaId.Id);
 
-                var result = await p.TextIndexer.SearchAsync(text, app, searchFilter, target);
+                var result = await p.TextIndex.SearchAsync(text, app, searchFilter, target);
 
                 if (expected != null)
                 {
@@ -400,9 +429,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         private async Task TestCombinations(params IndexOperation[] actions)
         {
-            for (var i = 0; i < actions.Length; i++)
+            if (SupportsCleanup)
+            {
+                for (var i = 0; i < actions.Length; i++)
+                {
+                    await TestCombinations(i, actions);
+                }
+            }
+            else
             {
-                await TestCombinations(i, actions);
+                await TestCombinations(0, actions);
             }
         }
 

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Elastic.cs

@@ -0,0 +1,41 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using System.Threading.Tasks;
+using Squidex.Domain.Apps.Entities.Contents.Text.Elastic;
+using Xunit;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text
+{
+    [Trait("Category", "Dependencies")]
+    public class TextIndexerTests_Elastic : TextIndexerTestsBase
+    {
+        private sealed class TheFactory : IIndexerFactory
+        {
+            public Task CleanupAsync()
+            {
+                return Task.CompletedTask;
+            }
+
+            public Task<ITextIndex> CreateAsync(Guid schemaId)
+            {
+                var index = new ElasticSearchTextIndex("http://localhost:9200", "squidex", true);
+
+                return Task.FromResult<ITextIndex>(index);
+            }
+        }
+
+        public override IIndexerFactory Factory { get; } = new TheFactory();
+
+        public TextIndexerTests_Elastic()
+        {
+            SupportsSearchSyntax = false;
+            SupportsMultiLanguage = false;
+        }
+    }
+}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_FS.cs

@@ -1,14 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschraenkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-namespace Squidex.Domain.Apps.Entities.Contents.Text
-{
-    public class TextIndexerTests_FS : TextIndexerTestsBase
-    {
-        public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.TempFolder());
-    }
-}

backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTests_Mongo.cs

@@ -5,6 +5,10 @@
 //  All rights reserved. Licensed under the MIT license.
 // ==========================================================================
 
+using System;
+using System.Threading.Tasks;
+using MongoDB.Driver;
+using Squidex.Domain.Apps.Entities.MongoDb.FullText;
 using Xunit;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -12,6 +16,33 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     [Trait("Category", "Dependencies")]
     public class TextIndexerTests_Mongo : TextIndexerTestsBase
     {
-        public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.MongoDB());
+        private sealed class TheFactory : IIndexerFactory
+        {
+            private readonly MongoClient mongoClient = new MongoClient("mongodb://localhost");
+
+            public Task CleanupAsync()
+            {
+                return Task.CompletedTask;
+            }
+
+            public async Task<ITextIndex> CreateAsync(Guid schemaId)
+            {
+                var database = mongoClient.GetDatabase("FullText");
+
+                var index = new MongoTextIndex(database, false);
+
+                await index.InitializeAsync();
+
+                return index;
+            }
+        }
+
+        public override IIndexerFactory Factory { get; } = new TheFactory();
+
+        public TextIndexerTests_Mongo()
+        {
+            SupportsSearchSyntax = false;
+            SupportsMultiLanguage = false;
+        }
     }
 }

backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs

@@ -51,21 +51,27 @@ namespace Squidex.Domain.Apps.Entities.Rules
         }
 
         [Fact]
-        public void Should_return_contents_filter_for_events_filter()
+        public void Should_return_wildcard_filter_for_events_filter()
         {
-            Assert.Equal(".*", sut.EventsFilter);
+            IEventConsumer consumer = sut;
+
+            Assert.Equal(".*", consumer.EventsFilter);
         }
 
         [Fact]
-        public void Should_return_type_name_for_name()
+        public async Task Should_do_nothing_on_clear()
         {
-            Assert.Equal(typeof(RuleEnqueuer).Name, sut.Name);
+            IEventConsumer consumer = sut;
+
+            await consumer.ClearAsync();
         }
 
         [Fact]
-        public async Task Should_do_nothing_on_clear()
+        public void Should_return_type_name_for_name()
         {
-            await sut.ClearAsync();
+            IEventConsumer consumer = sut;
+
+            Assert.Equal(typeof(RuleEnqueuer).Name, consumer.Name);
         }
 
         [Fact]

backend/tests/Squidex.Domain.Apps.Entities.Tests/Squidex.Domain.Apps.Entities.Tests.csproj

@@ -32,6 +32,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="Xunit.SkippableFact" Version="1.4.13" />
   </ItemGroup>
   <PropertyGroup>
     <CodeAnalysisRuleSet>..\..\Squidex.ruleset</CodeAnalysisRuleSet>

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/CompoundEventConsumerTests.cs

@@ -1,131 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschränkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System.Threading.Tasks;
-using FakeItEasy;
-using Squidex.Infrastructure.TestHelpers;
-using Xunit;
-
-namespace Squidex.Infrastructure.EventSourcing
-{
-    public class CompoundEventConsumerTests
-    {
-        private readonly IEventConsumer consumer1 = A.Fake<IEventConsumer>();
-        private readonly IEventConsumer consumer2 = A.Fake<IEventConsumer>();
-
-        [Fact]
-        public void Should_return_given_name()
-        {
-            var sut = new CompoundEventConsumer("consumer-name", consumer1);
-
-            Assert.Equal("consumer-name", sut.Name);
-        }
-
-        [Fact]
-        public void Should_return_first_inner_name()
-        {
-            A.CallTo(() => consumer1.Name).Returns("my-inner-consumer");
-
-            var sut = new CompoundEventConsumer(consumer1, consumer2);
-
-            Assert.Equal("my-inner-consumer", sut.Name);
-        }
-
-        [Fact]
-        public void Should_return_compound_filter()
-        {
-            A.CallTo(() => consumer1.EventsFilter).Returns("filter1");
-            A.CallTo(() => consumer2.EventsFilter).Returns("filter2");
-
-            var sut = new CompoundEventConsumer("my", consumer1, consumer2);
-
-            Assert.Equal("(filter1)|(filter2)", sut.EventsFilter);
-        }
-
-        [Fact]
-        public void Should_return_compound_filter_from_array()
-        {
-            A.CallTo(() => consumer1.EventsFilter).Returns("filter1");
-            A.CallTo(() => consumer2.EventsFilter).Returns("filter2");
-
-            var sut = new CompoundEventConsumer(new[] { consumer1, consumer2 });
-
-            Assert.Equal("(filter1)|(filter2)", sut.EventsFilter);
-        }
-
-        [Fact]
-        public void Should_ignore_empty_filters()
-        {
-            A.CallTo(() => consumer1.EventsFilter).Returns("filter1");
-            A.CallTo(() => consumer2.EventsFilter).Returns(string.Empty);
-
-            var sut = new CompoundEventConsumer("my", consumer1, consumer2);
-
-            Assert.Equal("(filter1)", sut.EventsFilter);
-        }
-
-        [Fact]
-        public async Task Should_clear_all_consumers()
-        {
-            var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2);
-
-            await sut.ClearAsync();
-
-            A.CallTo(() => consumer1.ClearAsync()).MustHaveHappened();
-            A.CallTo(() => consumer2.ClearAsync()).MustHaveHappened();
-        }
-
-        [Fact]
-        public async Task Should_invoke_all_consumers()
-        {
-            var @event = Envelope.Create<IEvent>(new MyEvent());
-
-            var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2);
-
-            await sut.On(@event);
-
-            A.CallTo(() => consumer1.On(@event)).MustHaveHappened();
-            A.CallTo(() => consumer2.On(@event)).MustHaveHappened();
-        }
-
-        [Fact]
-        public void Should_handle_if_any_consumer_handles()
-        {
-            var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload"));
-
-            A.CallTo(() => consumer1.Handles(stored))
-                .Returns(false);
-
-            A.CallTo(() => consumer2.Handles(stored))
-                .Returns(true);
-
-            var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2);
-
-            var result = sut.Handles(stored);
-
-            Assert.True(result);
-        }
-
-        [Fact]
-        public void Should_no_handle_if_no_consumer_handles()
-        {
-            var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload"));
-
-            A.CallTo(() => consumer1.Handles(stored))
-                .Returns(false);
-
-            A.CallTo(() => consumer2.Handles(stored))
-                .Returns(false);
-
-            var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2);
-
-            var result = sut.Handles(stored);
-
-            Assert.False(result);
-        }
-    }
-}

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/EventStoreTests.cs

@@ -399,10 +399,7 @@ namespace Squidex.Infrastructure.EventSourcing
             }
             finally
             {
-                if (subscription != null)
+                subscription?.Unsubscribe();
-                {
-                    await subscription.StopAsync();
-                }
             }
         }
 

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cs

@@ -6,10 +6,10 @@
 // ==========================================================================
 
 using System;
+using System.Collections.Generic;
 using System.Threading.Tasks;
 using FakeItEasy;
 using FluentAssertions;
-using Orleans.Concurrency;
 using Squidex.Infrastructure.Log;
 using Squidex.Infrastructure.Orleans;
 using Squidex.Infrastructure.Reflection;
@@ -22,6 +22,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
     {
         public sealed class MyEventConsumerGrain : EventConsumerGrain
         {
+            private IEventSubscriber currentSubscriber;
+
             public MyEventConsumerGrain(
                 EventConsumerFactory eventConsumerFactory,
                 IGrainState<EventConsumerState> state,
@@ -32,14 +34,26 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             {
             }
 
-            protected override IEventConsumerGrain GetSelf()
+            public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent)
             {
-                return this;
+                return currentSubscriber.OnEventAsync(subscription, storedEvent);
             }
 
-            protected override IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string? filter, string? position)
+            public Task OnErrorAsync(IEventSubscription subscription, Exception exception)
             {
-                return store.CreateSubscription(subscriber, filter, position);
+                return currentSubscriber.OnErrorAsync(subscription, exception);
+            }
+
+            protected override IEventSubscription CreateRetrySubscription(IEventSubscriber subscriber)
+            {
+                return CreateSubscription(subscriber);
+            }
+
+            protected override IEventSubscription CreateSubscription(IEventSubscriber subscriber)
+            {
+                currentSubscriber = subscriber;
+
+                return base.CreateSubscription(subscriber);
             }
         }
 
@@ -51,7 +65,7 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
         private readonly IEventDataFormatter formatter = A.Fake<IEventDataFormatter>();
         private readonly EventData eventData = new EventData("Type", new EnvelopeHeaders(), "Payload");
         private readonly Envelope<IEvent> envelope = new Envelope<IEvent>(new MyEvent());
-        private readonly EventConsumerGrain sut;
+        private readonly MyEventConsumerGrain sut;
         private readonly string consumerName;
         private readonly string initialPosition = Guid.NewGuid().ToString();
 
@@ -70,6 +84,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             A.CallTo(() => eventConsumer.Handles(A<StoredEvent>._))
                 .Returns(true);
 
+            A.CallTo(() => eventConsumer.On(A<IEnumerable<Envelope<IEvent>>>._))
+                .Invokes((IEnumerable<Envelope<IEvent>> events) =>
+                {
+                    foreach (var @event in events)
+                    {
+                        eventConsumer.On(@event).Wait();
+                    }
+                });
+
+            A.CallTo(() => eventSubscription.Sender)
+                .Returns(eventSubscription);
+
             A.CallTo(() => formatter.Parse(eventData, null))
                 .Returns(envelope);
 
@@ -89,6 +115,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync(consumerName);
             await sut.ActivateAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = null });
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
@@ -101,10 +129,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync(consumerName);
             await sut.ActivateAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null });
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -115,10 +145,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync(consumerName);
             await sut.ActivateAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null });
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -127,10 +159,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync(consumerName);
             await sut.ActivateAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null });
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -141,13 +175,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.StopAsync();
             await sut.StopAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = null });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -158,22 +194,24 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.StopAsync();
             await sut.ResetAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = null, Error = null });
 
             A.CallTo(() => grainState.WriteAsync())
                 .MustHaveHappened(2, Times.Exactly);
 
             A.CallTo(() => eventConsumer.ClearAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, grainState.Value.Position))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, null))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -186,13 +224,71 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventConsumer.On(envelope))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
+        }
+
+        [Fact]
+        public async Task Should_invoke_and_update_position_when_event_received_one_by_one()
+        {
+            var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData);
+
+            A.CallTo(() => eventConsumer.BatchSize)
+                .Returns(1);
+
+            await sut.ActivateAsync(consumerName);
+            await sut.ActivateAsync();
+
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+
+            await sut.CompleteAsync();
+
+            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 5 });
+
+            A.CallTo(() => grainState.WriteAsync())
+                .MustHaveHappened(5, Times.Exactly);
+
+            A.CallTo(() => eventConsumer.On(A<IEnumerable<Envelope<IEvent>>>._))
+                .MustHaveHappened(5, Times.Exactly);
+        }
+
+        [Fact]
+        public async Task Should_invoke_and_update_position_when_event_received_batched()
+        {
+            var @event = new StoredEvent("Stream", Guid.NewGuid().ToString(), 123, eventData);
+
+            A.CallTo(() => eventConsumer.BatchSize)
+                .Returns(100);
+
+            await sut.ActivateAsync(consumerName);
+            await sut.ActivateAsync();
+
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+            await OnEventAsync(eventSubscription, @event);
+
+            await sut.CompleteAsync();
+
+            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 5 });
+
+            A.CallTo(() => grainState.WriteAsync())
+                .MustHaveHappenedOnceExactly();
+
+            A.CallTo(() => eventConsumer.On(A<IEnumerable<Envelope<IEvent>>>._))
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -208,10 +304,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
-            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 });
+            await sut.CompleteAsync();
+
+            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 0 });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventConsumer.On(envelope))
                 .MustNotHaveHappened();
@@ -230,10 +328,12 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
-            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 1 });
+            await sut.CompleteAsync();
+
+            AssetGrainState(new EventConsumerState { IsStopped = false, Position = @event.EventPosition, Error = null, Count = 0 });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventConsumer.On(envelope))
                 .MustNotHaveHappened();
@@ -249,6 +349,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(A.Fake<IEventSubscription>(), @event);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null });
 
             A.CallTo(() => eventConsumer.On(envelope))
@@ -265,13 +367,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnErrorAsync(eventSubscription, ex);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -284,6 +388,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnErrorAsync(A.Fake<IEventSubscription>(), ex);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = false, Position = initialPosition, Error = null });
 
             A.CallTo(() => grainState.WriteAsync())
@@ -297,6 +403,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync();
             await sut.ActivateAsync();
 
+            await sut.CompleteAsync();
+
             A.CallTo(() => eventSubscription.WakeUp())
                 .MustHaveHappened();
         }
@@ -313,13 +421,15 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             await sut.ActivateAsync();
             await sut.ResetAsync();
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() });
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -337,16 +447,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() });
 
             A.CallTo(() => eventConsumer.On(envelope))
                 .MustHaveHappened();
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -364,16 +476,18 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
+            await sut.CompleteAsync();
+
             AssetGrainState(new EventConsumerState { IsStopped = true, Position = initialPosition, Error = ex.ToString() });
 
             A.CallTo(() => eventConsumer.On(envelope))
                 .MustNotHaveHappened();
 
             A.CallTo(() => grainState.WriteAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -391,6 +505,8 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
 
             await OnEventAsync(eventSubscription, @event);
 
+            await sut.CompleteAsync();
+
             await sut.StopAsync();
             await sut.StartAsync();
             await sut.StartAsync();
@@ -403,21 +519,21 @@ namespace Squidex.Infrastructure.EventSourcing.Grains
             A.CallTo(() => grainState.WriteAsync())
                 .MustHaveHappened(2, Times.Exactly);
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
                 .MustHaveHappened(2, Times.Exactly);
         }
 
-        private Task OnErrorAsync(IEventSubscription subscriber, Exception ex)
+        private Task OnErrorAsync(IEventSubscription subscription, Exception exception)
         {
-            return sut.OnErrorAsync(subscriber.AsImmutable(), ex.AsImmutable());
+            return sut.OnErrorAsync(subscription, exception);
         }
 
-        private Task OnEventAsync(IEventSubscription subscriber, StoredEvent ev)
+        private Task OnEventAsync(IEventSubscription subscription, StoredEvent ev)
         {
-            return sut.OnEventAsync(subscriber.AsImmutable(), ev.AsImmutable());
+            return sut.OnEventAsync(subscription, ev);
         }
 
         private void AssetGrainState(EventConsumerState state)

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/MongoParallelInsertTests.cs

@@ -164,21 +164,6 @@ namespace Squidex.Infrastructure.EventSourcing
                 : base(eventConsumerFactory, state, eventStore, eventDataFormatter, log)
             {
             }
-
-            protected override IEventConsumerGrain GetSelf()
-            {
-                return this;
-            }
-
-            protected override TaskScheduler GetScheduler()
-            {
-                return scheduler;
-            }
-
-            protected override IEventSubscription CreateSubscription(IEventStore store, IEventSubscriber subscriber, string? filter, string? position)
-            {
-                return store.CreateSubscription(subscriber, filter, position);
-            }
         }
 
         public class MyEvent : IEvent
@@ -206,16 +191,6 @@ namespace Squidex.Infrastructure.EventSourcing
                 this.expectedCount = expectedCount;
             }
 
-            public Task ClearAsync()
-            {
-                return Task.CompletedTask;
-            }
-
-            public bool Handles(StoredEvent @event)
-            {
-                return true;
-            }
-
             public async Task On(Envelope<IEvent> @event)
             {
                 Received++;

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/PollingSubscriptionTests.cs

@@ -27,7 +27,7 @@ namespace Squidex.Infrastructure.EventSourcing
             await WaitAndStopAsync(sut);
 
             A.CallTo(() => eventStore.QueryAsync(A<Func<StoredEvent, Task>>._, "^my-stream", position, A<CancellationToken>._))
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         [Fact]
@@ -95,7 +95,7 @@ namespace Squidex.Infrastructure.EventSourcing
         {
             await Task.Delay(200);
 
-            await sut.StopAsync();
+            sut.Unsubscribe();
         }
     }
 }

backend/tests/Squidex.Infrastructure.Tests/EventSourcing/RetrySubscriptionTests.cs

@@ -19,23 +19,34 @@ namespace Squidex.Infrastructure.EventSourcing
         private readonly IEventSubscription eventSubscription = A.Fake<IEventSubscription>();
         private readonly IEventSubscriber sutSubscriber;
         private readonly RetrySubscription sut;
-        private readonly string streamFilter = Guid.NewGuid().ToString();
 
         public RetrySubscriptionTests()
         {
-            A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._)).Returns(eventSubscription);
+            A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
+                .Returns(eventSubscription);
+
+            A.CallTo(() => eventSubscription.Sender)
+                .Returns(eventSubscription);
 
-            sut = new RetrySubscription(eventStore, eventSubscriber, streamFilter, null) { ReconnectWaitMs = 50 };
+            sut = new RetrySubscription(eventSubscriber, s => eventStore.CreateSubscription(s)) { ReconnectWaitMs = 50 };
 
             sutSubscriber = sut;
         }
 
         [Fact]
-        public async Task Should_subscribe_after_constructor()
+        public void Should_return_original_subscription_as_sender()
+        {
+            var sender = sut.Sender;
+
+            Assert.Same(eventSubscription, sender);
+        }
+
+        [Fact]
+        public void Should_subscribe_after_constructor()
         {
-            await sut.StopAsync();
+            sut.Unsubscribe();
 
-            A.CallTo(() => eventStore.CreateSubscription(sut, streamFilter, null))
+            A.CallTo(() => eventStore.CreateSubscription(sut, A<string>._, A<string>._))
                 .MustHaveHappened();
         }
 
@@ -46,15 +57,15 @@ namespace Squidex.Infrastructure.EventSourcing
 
             await Task.Delay(1000);
 
-            await sut.StopAsync();
+            sut.Unsubscribe();
 
-            A.CallTo(() => eventSubscription.StopAsync())
+            A.CallTo(() => eventSubscription.Unsubscribe())
                 .MustHaveHappened(2, Times.Exactly);
 
             A.CallTo(() => eventStore.CreateSubscription(A<IEventSubscriber>._, A<string>._, A<string>._))
                 .MustHaveHappened(2, Times.Exactly);
 
-            A.CallTo(() => eventSubscriber.OnErrorAsync(A<IEventSubscription>._, A<Exception>._))
+            A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, A<Exception>._))
                 .MustNotHaveHappened();
         }
 
@@ -64,26 +75,28 @@ namespace Squidex.Infrastructure.EventSourcing
             var ex = new InvalidOperationException();
 
             await OnErrorAsync(eventSubscription, ex);
-            await OnErrorAsync(null!, ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await OnErrorAsync(null!, ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await OnErrorAsync(null!, ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await OnErrorAsync(null!, ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await OnErrorAsync(null!, ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await sut.StopAsync();
+
-
+            sut.Unsubscribe();
-            A.CallTo(() => eventSubscriber.OnErrorAsync(sut, ex))
+
+            A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, ex))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_not_forward_error_when_exception_is_from_another_subscription()
+        public async Task Should_not_forward_error_when_exception_is_raised_after_unsubscribe()
         {
             var ex = new InvalidOperationException();
 
-            await OnErrorAsync(A.Fake<IEventSubscription>(), ex);
+            await OnErrorAsync(eventSubscription, ex);
-            await sut.StopAsync();
+
+            sut.Unsubscribe();
 
-            A.CallTo(() => eventSubscriber.OnErrorAsync(A<IEventSubscription>._, A<Exception>._))
+            A.CallTo(() => eventSubscriber.OnErrorAsync(eventSubscription, A<Exception>._))
                 .MustNotHaveHappened();
         }
 
@@ -93,22 +106,24 @@ namespace Squidex.Infrastructure.EventSourcing
             var ev = new StoredEvent("Stream", "1", 2, new EventData("Type", new EnvelopeHeaders(), "Payload"));
 
             await OnEventAsync(eventSubscription, ev);
-            await sut.StopAsync();
 
-            A.CallTo(() => eventSubscriber.OnEventAsync(sut, ev))
+            sut.Unsubscribe();
+
+            A.CallTo(() => eventSubscriber.OnEventAsync(eventSubscription, ev))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_not_forward_event_when_message_is_from_another_subscription()
+        public async Task Should_forward_event_when_message_is_from_another_subscription()
         {
             var ev = new StoredEvent("Stream", "1", 2, new EventData("Type", new EnvelopeHeaders(), "Payload"));
 
             await OnEventAsync(A.Fake<IEventSubscription>(), ev);
-            await sut.StopAsync();
+
+            sut.Unsubscribe();
 
             A.CallTo(() => eventSubscriber.OnEventAsync(A<IEventSubscription>._, A<StoredEvent>._))
-                .MustNotHaveHappened();
+                .MustHaveHappened();
         }
 
         private Task OnErrorAsync(IEventSubscription subscriber, Exception ex)

backend/tests/Squidex.Infrastructure.Tests/Migrations/MigratorTests.cs

@@ -149,9 +149,9 @@ namespace Squidex.Infrastructure.Migrations
             await Task.WhenAll(Enumerable.Repeat(0, 10).Select(x => Task.Run(() => sut.MigrateAsync())));
 
             A.CallTo(() => migrator_0_1.UpdateAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
             A.CallTo(() => migrator_1_2.UpdateAsync())
-                .MustHaveHappened(1, Times.Exactly);
+                .MustHaveHappenedOnceExactly();
         }
 
         private IMigration BuildMigration(int fromVersion, int toVersion)

backend/tests/Squidex.Infrastructure.Tests/MongoDb/MongoExtensionsTests.cs

@@ -78,7 +78,7 @@ namespace Squidex.Infrastructure.MongoDb
 
             var cursor = new Cursor<int>().Add(0, 1, 2, 3, 4, 5);
 
-            await cursor.ForEachPipelineAsync(x =>
+            await cursor.ForEachPipedAsync(x =>
             {
                 result.Add(x);
                 return Task.CompletedTask;
@@ -98,7 +98,7 @@ namespace Squidex.Infrastructure.MongoDb
             {
                 await Assert.ThrowsAsync<InvalidOperationException>(() =>
                 {
-                    return cursor.ForEachPipelineAsync(x =>
+                    return cursor.ForEachPipedAsync(x =>
                     {
                         result.Add(x);
                         return Task.CompletedTask;
@@ -120,7 +120,7 @@ namespace Squidex.Infrastructure.MongoDb
             {
                 await Assert.ThrowsAsync<InvalidOperationException>(() =>
                 {
-                    return cursor.ForEachPipelineAsync(x =>
+                    return cursor.ForEachPipedAsync(x =>
                     {
                         if (x == 2)
                         {
@@ -147,7 +147,7 @@ namespace Squidex.Infrastructure.MongoDb
                 {
                     await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
                     {
-                        return cursor.ForEachPipelineAsync(x =>
+                        return cursor.ForEachPipedAsync(x =>
                         {
                             if (x == 2)
                             {

backend/tests/Squidex.Infrastructure.Tests/Tasks/SingleThreadedDispatcherTests.cs

@@ -1,56 +0,0 @@
-// ==========================================================================
-//  Squidex Headless CMS
-// ==========================================================================
-//  Copyright (c) Squidex UG (haftungsbeschränkt)
-//  All rights reserved. Licensed under the MIT license.
-// ==========================================================================
-
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
-using Xunit;
-
-namespace Squidex.Infrastructure.Tasks
-{
-    public class SingleThreadedDispatcherTests
-    {
-        private readonly SingleThreadedDispatcher sut = new SingleThreadedDispatcher();
-
-        [Fact]
-        public async Task Should_handle_with_task_messages_sequentially()
-        {
-            var source = Enumerable.Range(1, 100);
-            var target = new List<int>();
-
-            foreach (var item in source)
-            {
-                sut.DispatchAsync(() =>
-                {
-                    target.Add(item);
-
-                    return Task.CompletedTask;
-                }).Forget();
-            }
-
-            await sut.StopAndWaitAsync();
-
-            Assert.Equal(source, target);
-        }
-
-        [Fact]
-        public async Task Should_handle_messages_sequentially()
-        {
-            var source = Enumerable.Range(1, 100);
-            var target = new List<int>();
-
-            foreach (var item in source)
-            {
-                sut.DispatchAsync(() => target.Add(item)).Forget();
-            }
-
-            await sut.StopAndWaitAsync();
-
-            Assert.Equal(source, target);
-        }
-    }
-}