diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs index 621e141c9..d847ccf13 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/PersistenceHelper.cs @@ -9,6 +9,7 @@ using System; using System.IO; using System.IO.Compression; using System.Threading.Tasks; +using Lucene.Net.Index; using Squidex.Infrastructure.Assets; namespace Squidex.Domain.Apps.Entities.Contents.Text @@ -18,7 +19,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private const string ArchiveFile = "Archive.zip"; private const string LockFile = "write.lock"; - public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory) + public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory, IndexCommit commit) { using (var fileStream = new FileStream( Path.Combine(directory.FullName, ArchiveFile), @@ -30,8 +31,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) { - foreach (var file in directory.GetFiles()) + foreach (var fileName in commit.FileNames) { + var file = new FileInfo(Path.Combine(directory.FullName, fileName)); + try { if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index b0cc48f22..5e1f0e57d 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -37,11 +37,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private const string MetaDraft = "_dd"; private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30); private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); + private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(MetaDraft, "1")); + private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(MetaDraft, "0")); + private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); private readonly IAssetStore assetStore; private IDisposable timer; private DirectoryInfo directory; private IndexWriter indexWriter; private IndexReader indexReader; + private IndexSearcher indexSearcher; private QueryParser queryParser; private HashSet currentLanguages; private long updates; @@ -64,8 +68,18 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await assetStore.DownloadAsync(directory); - indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer)); - indexReader = indexWriter.GetReader(true); + var config = new IndexWriterConfig(Version, Analyzer) + { + IndexDeletionPolicy = snapshotter + }; + + indexWriter = new IndexWriter(FSDirectory.Open(directory), config); + + if (indexWriter.NumDocs > 0) + { + indexReader = indexWriter.GetReader(false); + indexSearcher = new IndexSearcher(indexReader); + } } public Task DeleteAsync(Guid id) @@ -78,11 +92,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public Task IndexAsync(Guid id, J data) { var docId = id.ToString(); - var docDraft = data.Value.IsDraft.ToString(); + var docDraft = data.Value.IsDraft ? "1" : "0"; var docKey = $"{docId}_{docDraft}"; - var query = new BooleanQuery(); - indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); var languages = new Dictionary(); @@ -165,9 +177,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text if (indexReader != null) { - var filter = new TermsFilter(new Term(MetaDraft, context.IsDraft.ToString())); + var filter = context.IsDraft ? DraftFilter : NoDraftFilter; - var hits = new IndexSearcher(indexReader).Search(query, filter, MaxResults).ScoreDocs; + var hits = indexSearcher.Search(query, filter, MaxResults).ScoreDocs; foreach (var hit in hits) { @@ -236,13 +248,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { if (updates > 0 && indexWriter != null) { - indexWriter.Flush(true, true); indexWriter.Commit(); + indexWriter.Flush(true, true); indexReader?.Dispose(); - indexReader = indexWriter.GetReader(true); + indexReader = indexWriter.GetReader(false); + indexSearcher = new IndexSearcher(indexReader); - await assetStore.UploadDirectoryAsync(directory); + var commit = snapshotter.Snapshot(); + try + { + await assetStore.UploadDirectoryAsync(directory, commit); + } + finally + { + snapshotter.Release(commit); + } updates = 0; } diff --git a/tools/Migrate_01/MigrationPath.cs b/tools/Migrate_01/MigrationPath.cs index 047afeab2..b67e2f653 100644 --- a/tools/Migrate_01/MigrationPath.cs +++ b/tools/Migrate_01/MigrationPath.cs @@ -16,7 +16,7 @@ namespace Migrate_01 { public sealed class MigrationPath : IMigrationPath { - private const int CurrentVersion = 14; + private const int CurrentVersion = 15; private readonly IServiceProvider serviceProvider; public MigrationPath(IServiceProvider serviceProvider) @@ -72,7 +72,8 @@ namespace Migrate_01 } // Version 11: Introduce content drafts. - if (version < 11) + // Version 15: Introduce custom full text search actors. + if (version < 15) { yield return serviceProvider.GetService(); yield return serviceProvider.GetRequiredService(); diff --git a/tools/Migrate_01/Migrations/DeleteContentCollections.cs b/tools/Migrate_01/Migrations/DeleteContentCollections.cs index 3cb3afe6e..b731e0d57 100644 --- a/tools/Migrate_01/Migrations/DeleteContentCollections.cs +++ b/tools/Migrate_01/Migrations/DeleteContentCollections.cs @@ -24,6 +24,8 @@ namespace Migrate_01.Migrations { await database.DropCollectionAsync("States_Contents"); await database.DropCollectionAsync("States_Contents_Archive"); + await database.DropCollectionAsync("State_Content_Draft"); + await database.DropCollectionAsync("State_Content_Published"); } } } diff --git a/tools/Migrate_01/Rebuilder.cs b/tools/Migrate_01/Rebuilder.cs index ee77eb829..c7a034eff 100644 --- a/tools/Migrate_01/Rebuilder.cs +++ b/tools/Migrate_01/Rebuilder.cs @@ -96,7 +96,11 @@ namespace Migrate_01 { var handledIds = new HashSet(); - var worker = new ActionBlock(action, new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = 32 }); + var worker = new ActionBlock(action, + new ExecutionDataflowBlockOptions + { + MaxDegreeOfParallelism = 32 + }); await eventStore.QueryAsync(async storedEvent => {