From 9984937116ae445956f00c7cb61aecb6cb968fa4 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 29 Mar 2019 20:25:37 +0100 Subject: [PATCH] Flush performance improvement. --- .../Contents/Text/IndexState.cs | 69 +++++++++++++++++++ .../Contents/Text/TextIndexContent.cs | 68 +++++------------- .../Contents/Text/TextIndexerGrain.cs | 52 +++++++------- .../Contents/Text/TextIndexerGrainTests.cs | 1 - 4 files changed, 113 insertions(+), 77 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs new file mode 100644 index 000000000..257f09a15 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs @@ -0,0 +1,69 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +using System; +using Lucene.Net.Documents; +using Lucene.Net.Index; +using Lucene.Net.Util; + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + internal sealed class IndexState + { + private const string MetaFor = "_fd"; + private readonly IndexReader indexReader; + private readonly IndexWriter indexWriter; + private readonly BinaryDocValues binaryValues; + + public IndexState(IndexReader indexReader, IndexWriter indexWriter) + { + this.indexReader = indexReader; + this.indexWriter = indexWriter; + + if (indexReader != null) + { + binaryValues = MultiDocValues.GetBinaryValues(indexReader, MetaFor); + } + } + + public void Index(Document document, byte forDraft, byte forPublished) + { + document.RemoveField(MetaFor); + document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished)); + } + + public void Index(Term term, Guid id, byte draft, byte forDraft, byte forPublished) + { + indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished)); + } + + public bool TryGet(int docId, out byte forDraft, out byte forPublished) + { + var forValue = new BytesRef(); + + forDraft = 0; + forPublished = 0; + + binaryValues?.Get(docId, forValue); + + if (forValue.Bytes.Length == 2) + { + forDraft = forValue.Bytes[0]; + forPublished = forValue.Bytes[1]; + + return true; + } + + return false; + } + + private static BytesRef GetValue(byte forDraft, byte forPublished) + { + return new BytesRef(new byte[] { forDraft, forPublished }); + } + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs index 6d0f36076..e79d23791 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs @@ -12,7 +12,6 @@ using System.Text; using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; -using Lucene.Net.Util; using Squidex.Domain.Apps.Core.Contents; using Squidex.Infrastructure; using Squidex.Infrastructure.Json.Objects; @@ -23,48 +22,42 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { private const string MetaId = "_id"; private const string MetaKey = "_key"; - private const string MetaFor = "_fd"; private const int ForDraftIndex = 0; private const int ForPublishedIndex = 1; private readonly IndexWriter indexWriter; private readonly IndexSearcher indexSearcher; - private readonly BinaryDocValues indexValues; + private readonly IndexState indexState; private readonly Guid id; - public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id) + public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, IndexState indexState, Guid id) { this.indexWriter = indexWriter; this.indexSearcher = indexSearcher; - this.indexValues = indexValues; + this.indexState = indexState; this.id = id; } - public static BinaryDocValues CreateValues(IndexReader indexReader) - { - return MultiDocValues.GetBinaryValues(indexReader, MetaFor); - } - public void Delete() { indexWriter.DeleteDocuments(new Term(MetaId, id.ToString())); } - public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result) + public static bool TryGetId(int docId, Scope scope, IndexReader reader, IndexState indexState, out Guid result) { result = Guid.Empty; - if (!TryGet(docId, values, out var @for)) + if (!indexState.TryGet(docId, out var draft, out var published)) { return false; } - if (scope == Scope.Draft && @for[ForDraftIndex] != 1) + if (scope == Scope.Draft && draft != 1) { return false; } - if (scope == Scope.Published && @for[ForPublishedIndex] != 1) + if (scope == Scope.Published && published != 1) { return false; } @@ -81,7 +74,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return true; } - public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft) + public void Index(NamedContentData dataDraft, NamedContentData data, bool onlyDraft) { var converted = CreateDocument(dataDraft); @@ -95,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } else { - var isPublished = IsForPublished(docId, values); + var isPublished = IsForPublished(docId); if (!onlyDraft && docId > 0 && isPublished) { @@ -174,7 +167,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { var term = new Term(MetaKey, BuildKey(draft)); - indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished)); + indexState.Index(term, id, draft, forDraft, forPublished); } private int GetPublishedDocument() @@ -190,18 +183,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { document.RemoveField(MetaId); document.RemoveField(MetaKey); - document.RemoveField(MetaFor); - var docId = id.ToString(); - var docKey = BuildKey(draft); + var contentId = id.ToString(); + var contentKey = BuildKey(draft); - document.AddStringField(MetaId, docId, Field.Store.YES); - document.AddStringField(MetaKey, docKey, Field.Store.YES); + document.AddStringField(MetaId, contentId, Field.Store.YES); + document.AddStringField(MetaKey, contentKey, Field.Store.YES); - document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished)); + indexState.Index(document, forDraft, forPublished); - // indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); - indexWriter.UpdateDocument(new Term(MetaKey, docKey), document); + indexWriter.UpdateDocument(new Term(MetaKey, contentKey), document); } } @@ -227,32 +218,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } - private static bool IsForPublished(int docId, BinaryDocValues values) - { - return TryGet(docId, values, out var @for) && @for[1] == 1; - } - - private static bool TryGet(int docId, BinaryDocValues values, out byte[] result) - { - var forValue = new BytesRef(); - - values?.Get(docId, forValue); - - if (forValue.Bytes.Length == 2) - { - result = forValue.Bytes; - return true; - } - else - { - result = null; - return false; - } - } - - private static BytesRef GetValue(byte forDraft, byte forPublished) + private bool IsForPublished(int docId) { - return new BytesRef(new byte[] { forDraft, forPublished }); + return indexState.TryGet(docId, out _, out var p) && p == 1; } private string BuildKey(byte draft) diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index 1af1d4dc7..b60ac7b89 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -38,11 +38,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private IndexWriter indexWriter; private IndexReader indexReader; private IndexSearcher indexSearcher; - private BinaryDocValues indexValues; + private IndexState indexState; private QueryParser queryParser; private HashSet currentLanguages; private long updates; - private long updatesNotWritten; public TextIndexerGrain(IAssetStore assetStore) { @@ -71,15 +70,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text if (indexWriter.NumDocs > 0) { - indexReader = indexWriter.GetReader(false); - indexSearcher = new IndexSearcher(indexReader); - indexValues = TextIndexContent.CreateValues(indexReader); + OpenReader(); + } + else + { + indexState = new IndexState(indexReader, indexWriter); } } public Task DeleteAsync(Guid id) { - var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id); content.Delete(); @@ -88,16 +89,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public Task IndexAsync(Guid id, J data, bool onlyDraft) { - var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id); - content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft); + content.Index(data.Value.DataDraft, data.Value.Data, onlyDraft); return TryFlushAsync(); } public Task CopyAsync(Guid id, bool fromDraft) { - var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id); content.Copy(fromDraft); @@ -118,7 +119,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text foreach (var hit in hits) { - if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id)) + if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexState, out var id)) { result.Add(id); } @@ -153,21 +154,20 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private async Task TryFlushAsync() { updates++; - updatesNotWritten++; if (updates >= MaxUpdates) { - await FlushAsync(true); + await FlushAsync(); } else { - await FlushAsync(); + OpenReader(); timer?.Dispose(); try { - timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay); + timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay); } catch (InvalidOperationException) { @@ -176,23 +176,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } - public async Task FlushAsync(bool write = false) + public async Task FlushAsync() { if (updates > 0 && indexWriter != null) { indexWriter.Commit(); indexWriter.Flush(true, true); - indexReader?.Dispose(); - indexReader = indexWriter.GetReader(false); - indexSearcher = new IndexSearcher(indexReader); - indexValues = TextIndexContent.CreateValues(indexReader); - - updates = 0; - } + OpenReader(); - if (updatesNotWritten > 0 && write) - { var commit = snapshotter.Snapshot(); try { @@ -203,7 +195,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text snapshotter.Release(commit); } - updatesNotWritten = 0; + updates = 0; } timer?.Dispose(); @@ -211,7 +203,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public async Task DeactivateAsync(bool deleteFolder = false) { - await FlushAsync(true); + await FlushAsync(); indexWriter?.Dispose(); indexWriter = null; @@ -224,5 +216,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text directory.Delete(true); } } + + private void OpenReader() + { + indexReader?.Dispose(); + indexReader = indexWriter.GetReader(true); + indexSearcher = new IndexSearcher(indexReader); + indexState = new IndexState(indexReader, indexWriter); + } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs index f1f84a48e..0a29b4978 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs @@ -213,7 +213,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true); await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true); - await sut.FlushAsync(); } private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false)