Indexing improved and migrator.

src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs

@@ -9,6 +9,7 @@ using System;
 using System.IO;
 using System.IO.Compression;
 using System.Threading.Tasks;
+using Lucene.Net.Index;
 using Squidex.Infrastructure.Assets;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -18,7 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private const string ArchiveFile = "Archive.zip";
         private const string LockFile = "write.lock";
 
-        public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory)
+        public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory, IndexCommit commit)
         {
             using (var fileStream = new FileStream(
                    Path.Combine(directory.FullName, ArchiveFile),
@@ -30,8 +31,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             {
                 using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true))
                 {
-                    foreach (var file in directory.GetFiles())
+                    foreach (var fileName in commit.FileNames)
                     {
+                        var file = new FileInfo(Path.Combine(directory.FullName, fileName));
+
                         try
                         {
                             if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) &&

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs

@@ -37,11 +37,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private const string MetaDraft = "_dd";
         private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30);
         private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version);
+        private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(MetaDraft, "1"));
+        private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(MetaDraft, "0"));
+        private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
         private readonly IAssetStore assetStore;
         private IDisposable timer;
         private DirectoryInfo directory;
         private IndexWriter indexWriter;
         private IndexReader indexReader;
+        private IndexSearcher indexSearcher;
         private QueryParser queryParser;
         private HashSet<string> currentLanguages;
         private long updates;
@@ -64,8 +68,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             await assetStore.DownloadAsync(directory);
 
-            indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer));
-            indexReader = indexWriter.GetReader(true);
+            var config = new IndexWriterConfig(Version, Analyzer)
+            {
+                IndexDeletionPolicy = snapshotter
+            };
+
+            indexWriter = new IndexWriter(FSDirectory.Open(directory), config);
+
+            if (indexWriter.NumDocs > 0)
+            {
+                indexReader = indexWriter.GetReader(false);
+                indexSearcher = new IndexSearcher(indexReader);
+            }
         }
 
         public Task DeleteAsync(Guid id)
@@ -78,11 +92,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         public Task IndexAsync(Guid id, J<IndexData> data)
         {
             var docId = id.ToString();
-            var docDraft = data.Value.IsDraft.ToString();
+            var docDraft = data.Value.IsDraft ? "1" : "0";
             var docKey = $"{docId}_{docDraft}";
 
-            var query = new BooleanQuery();
-
             indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
 
             var languages = new Dictionary<string, StringBuilder>();
@@ -165,9 +177,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
                 if (indexReader != null)
                 {
-                    var filter = new TermsFilter(new Term(MetaDraft, context.IsDraft.ToString()));
+                    var filter = context.IsDraft ? DraftFilter : NoDraftFilter;
 
-                    var hits = new IndexSearcher(indexReader).Search(query, filter, MaxResults).ScoreDocs;
+                    var hits = indexSearcher.Search(query, filter, MaxResults).ScoreDocs;
 
                     foreach (var hit in hits)
                     {
@@ -236,13 +248,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             if (updates > 0 && indexWriter != null)
             {
-                indexWriter.Flush(true, true);
                 indexWriter.Commit();
+                indexWriter.Flush(true, true);
 
                 indexReader?.Dispose();
-                indexReader = indexWriter.GetReader(true);
+                indexReader = indexWriter.GetReader(false);
+                indexSearcher = new IndexSearcher(indexReader);
 
-                await assetStore.UploadDirectoryAsync(directory);
+                var commit = snapshotter.Snapshot();
+                try
+                {
+                    await assetStore.UploadDirectoryAsync(directory, commit);
+                }
+                finally
+                {
+                    snapshotter.Release(commit);
+                }
 
                 updates = 0;
             }

tools/Migrate_01/MigrationPath.cs

@@ -16,7 +16,7 @@ namespace Migrate_01
 {
     public sealed class MigrationPath : IMigrationPath
     {
-        private const int CurrentVersion = 14;
+        private const int CurrentVersion = 15;
         private readonly IServiceProvider serviceProvider;
 
         public MigrationPath(IServiceProvider serviceProvider)
@@ -72,7 +72,8 @@ namespace Migrate_01
             }
 
             // Version 11: Introduce content drafts.
-            if (version < 11)
+            // Version 15: Introduce custom full text search actors.
+            if (version < 15)
             {
                 yield return serviceProvider.GetService<DeleteContentCollections>();
                 yield return serviceProvider.GetRequiredService<RebuildContents>();

tools/Migrate_01/Migrations/DeleteContentCollections.cs

@@ -24,6 +24,8 @@ namespace Migrate_01.Migrations
         {
             await database.DropCollectionAsync("States_Contents");
             await database.DropCollectionAsync("States_Contents_Archive");
+            await database.DropCollectionAsync("State_Content_Draft");
+            await database.DropCollectionAsync("State_Content_Published");
         }
     }
 }

tools/Migrate_01/Rebuilder.cs

@@ -96,7 +96,11 @@ namespace Migrate_01
         {
             var handledIds = new HashSet<Guid>();
 
-            var worker = new ActionBlock<Guid>(action, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 32 });
+            var worker = new ActionBlock<Guid>(action,
+                new ExecutionDataflowBlockOptions
+                {
+                    MaxDegreeOfParallelism = 32
+                });
 
             await eventStore.QueryAsync(async storedEvent =>
             {