Flush performance improvement.

src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs

@@ -0,0 +1,69 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+using System;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Util;
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text
+{
+    internal sealed class IndexState
+    {
+        private const string MetaFor = "_fd";
+        private readonly IndexReader indexReader;
+        private readonly IndexWriter indexWriter;
+        private readonly BinaryDocValues binaryValues;
+
+        public IndexState(IndexReader indexReader, IndexWriter indexWriter)
+        {
+            this.indexReader = indexReader;
+            this.indexWriter = indexWriter;
+
+            if (indexReader != null)
+            {
+                binaryValues = MultiDocValues.GetBinaryValues(indexReader, MetaFor);
+            }
+        }
+
+        public void Index(Document document, byte forDraft, byte forPublished)
+        {
+            document.RemoveField(MetaFor);
+            document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
+        }
+
+        public void Index(Term term, Guid id, byte draft, byte forDraft, byte forPublished)
+        {
+            indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
+        }
+
+        public bool TryGet(int docId, out byte forDraft, out byte forPublished)
+        {
+            var forValue = new BytesRef();
+
+            forDraft = 0;
+            forPublished = 0;
+
+            binaryValues?.Get(docId, forValue);
+
+            if (forValue.Bytes.Length == 2)
+            {
+                forDraft = forValue.Bytes[0];
+                forPublished = forValue.Bytes[1];
+
+                return true;
+            }
+
+            return false;
+        }
+
+        private static BytesRef GetValue(byte forDraft, byte forPublished)
+        {
+            return new BytesRef(new byte[] { forDraft, forPublished });
+        }
+    }
+}

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs

@@ -12,7 +12,6 @@ using System.Text;
 using Lucene.Net.Documents;
 using Lucene.Net.Index;
 using Lucene.Net.Search;
-using Lucene.Net.Util;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Json.Objects;
@@ -23,48 +22,42 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     {
         private const string MetaId = "_id";
         private const string MetaKey = "_key";
-        private const string MetaFor = "_fd";
         private const int ForDraftIndex = 0;
         private const int ForPublishedIndex = 1;
         private readonly IndexWriter indexWriter;
         private readonly IndexSearcher indexSearcher;
-        private readonly BinaryDocValues indexValues;
+        private readonly IndexState indexState;
         private readonly Guid id;
 
-        public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id)
+        public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, IndexState indexState, Guid id)
         {
             this.indexWriter = indexWriter;
             this.indexSearcher = indexSearcher;
-            this.indexValues = indexValues;
+            this.indexState = indexState;
 
             this.id = id;
         }
 
-        public static BinaryDocValues CreateValues(IndexReader indexReader)
-        {
-            return MultiDocValues.GetBinaryValues(indexReader, MetaFor);
-        }
-
         public void Delete()
         {
             indexWriter.DeleteDocuments(new Term(MetaId, id.ToString()));
         }
 
-        public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result)
+        public static bool TryGetId(int docId, Scope scope, IndexReader reader, IndexState indexState, out Guid result)
         {
             result = Guid.Empty;
 
-            if (!TryGet(docId, values, out var @for))
+            if (!indexState.TryGet(docId, out var draft, out var published))
             {
                 return false;
             }
 
-            if (scope == Scope.Draft && @for[ForDraftIndex] != 1)
+            if (scope == Scope.Draft && draft != 1)
             {
                 return false;
             }
 
-            if (scope == Scope.Published && @for[ForPublishedIndex] != 1)
+            if (scope == Scope.Published && published != 1)
             {
                 return false;
             }
@@ -81,7 +74,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return true;
         }
 
-        public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft)
+        public void Index(NamedContentData dataDraft, NamedContentData data, bool onlyDraft)
         {
             var converted = CreateDocument(dataDraft);
 
@@ -95,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
             else
             {
-                var isPublished = IsForPublished(docId, values);
+                var isPublished = IsForPublished(docId);
 
                 if (!onlyDraft && docId > 0 && isPublished)
                 {
@@ -174,7 +167,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             var term = new Term(MetaKey, BuildKey(draft));
 
-            indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
+            indexState.Index(term, id, draft, forDraft, forPublished);
         }
 
         private int GetPublishedDocument()
@@ -190,18 +183,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             {
                 document.RemoveField(MetaId);
                 document.RemoveField(MetaKey);
-                document.RemoveField(MetaFor);
 
-                var docId = id.ToString();
+                var contentId = id.ToString();
-                var docKey = BuildKey(draft);
+                var contentKey = BuildKey(draft);
 
-                document.AddStringField(MetaId, docId, Field.Store.YES);
+                document.AddStringField(MetaId, contentId, Field.Store.YES);
-                document.AddStringField(MetaKey, docKey, Field.Store.YES);
+                document.AddStringField(MetaKey, contentKey, Field.Store.YES);
 
-                document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
+                indexState.Index(document, forDraft, forPublished);
 
-                // indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
+                indexWriter.UpdateDocument(new Term(MetaKey, contentKey), document);
-                indexWriter.UpdateDocument(new Term(MetaKey, docKey), document);
             }
         }
 
@@ -227,32 +218,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
-        private static bool IsForPublished(int docId, BinaryDocValues values)
+        private bool IsForPublished(int docId)
-        {
-            return TryGet(docId, values, out var @for) && @for[1] == 1;
-        }
-
-        private static bool TryGet(int docId, BinaryDocValues values, out byte[] result)
-        {
-            var forValue = new BytesRef();
-
-            values?.Get(docId, forValue);
-
-            if (forValue.Bytes.Length == 2)
-            {
-                result = forValue.Bytes;
-                return true;
-            }
-            else
-            {
-                result = null;
-                return false;
-            }
-        }
-
-        private static BytesRef GetValue(byte forDraft, byte forPublished)
         {
-            return new BytesRef(new byte[] { forDraft, forPublished });
+            return indexState.TryGet(docId, out _, out var p) && p == 1;
         }
 
         private string BuildKey(byte draft)

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs

@@ -38,11 +38,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private IndexWriter indexWriter;
         private IndexReader indexReader;
         private IndexSearcher indexSearcher;
-        private BinaryDocValues indexValues;
+        private IndexState indexState;
         private QueryParser queryParser;
         private HashSet<string> currentLanguages;
         private long updates;
-        private long updatesNotWritten;
 
         public TextIndexerGrain(IAssetStore assetStore)
         {
@@ -71,15 +70,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             if (indexWriter.NumDocs > 0)
             {
-                indexReader = indexWriter.GetReader(false);
+                OpenReader();
-                indexSearcher = new IndexSearcher(indexReader);
+            }
-                indexValues = TextIndexContent.CreateValues(indexReader);
+            else
+            {
+                indexState = new IndexState(indexReader, indexWriter);
             }
         }
 
         public Task DeleteAsync(Guid id)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
 
             content.Delete();
 
@@ -88,16 +89,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
 
-            content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft);
+            content.Index(data.Value.DataDraft, data.Value.Data, onlyDraft);
 
             return TryFlushAsync();
         }
 
         public Task CopyAsync(Guid id, bool fromDraft)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
 
             content.Copy(fromDraft);
 
@@ -118,7 +119,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
                     foreach (var hit in hits)
                     {
-                        if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id))
+                        if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexState, out var id))
                         {
                             result.Add(id);
                         }
@@ -153,21 +154,20 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private async Task TryFlushAsync()
         {
             updates++;
-            updatesNotWritten++;
 
             if (updates >= MaxUpdates)
             {
-                await FlushAsync(true);
+                await FlushAsync();
             }
             else
             {
-                await FlushAsync();
+                OpenReader();
 
                 timer?.Dispose();
 
                 try
                 {
-                    timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay);
+                    timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay);
                 }
                 catch (InvalidOperationException)
                 {
@@ -176,23 +176,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
-        public async Task FlushAsync(bool write = false)
+        public async Task FlushAsync()
         {
             if (updates > 0 && indexWriter != null)
             {
                 indexWriter.Commit();
                 indexWriter.Flush(true, true);
 
-                indexReader?.Dispose();
+                OpenReader();
-                indexReader = indexWriter.GetReader(false);
-                indexSearcher = new IndexSearcher(indexReader);
-                indexValues = TextIndexContent.CreateValues(indexReader);
-
-                updates = 0;
-            }
 
-            if (updatesNotWritten > 0 && write)
-            {
                 var commit = snapshotter.Snapshot();
                 try
                 {
@@ -203,7 +195,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                     snapshotter.Release(commit);
                 }
 
-                updatesNotWritten = 0;
+                updates = 0;
             }
 
             timer?.Dispose();
@@ -211,7 +203,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public async Task DeactivateAsync(bool deleteFolder = false)
         {
-            await FlushAsync(true);
+            await FlushAsync();
 
             indexWriter?.Dispose();
             indexWriter = null;
@@ -224,5 +216,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                 directory.Delete(true);
             }
         }
+
+        private void OpenReader()
+        {
+            indexReader?.Dispose();
+            indexReader = indexWriter.GetReader(true);
+            indexSearcher = new IndexSearcher(indexReader);
+            indexState = new IndexState(indexReader, indexWriter);
+        }
     }
 }

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs

@@ -213,7 +213,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true);
             await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true);
-            await sut.FlushAsync();
         }
 
         private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false)