tsai
/
squidex
mirror of https://github.com/Squidex/squidex.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Flush performance improvement.

pull/351/head
Sebastian Stehle 7 years ago
parent
4c6a36725b
commit
9984937116
4 changed files with 113 additions and 77 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 69
      src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs
  2. 68
      src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs
  3. 52
      src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs
  4. 1
      tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs

69
src/Squidex.Domain.Apps.Entities/Contents/Text/IndexState.cs
View File

@ -0,0 +1,69 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
using System;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Util;
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
internal sealed class IndexState
{
private const string MetaFor = "_fd";
private readonly IndexReader indexReader;
private readonly IndexWriter indexWriter;
private readonly BinaryDocValues binaryValues;
public IndexState(IndexReader indexReader, IndexWriter indexWriter)
{
this.indexReader = indexReader;
this.indexWriter = indexWriter;
if (indexReader != null)
{
binaryValues = MultiDocValues.GetBinaryValues(indexReader, MetaFor);
}
}
public void Index(Document document, byte forDraft, byte forPublished)
{
document.RemoveField(MetaFor);
document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
}
public void Index(Term term, Guid id, byte draft, byte forDraft, byte forPublished)
{
indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
}
public bool TryGet(int docId, out byte forDraft, out byte forPublished)
{
var forValue = new BytesRef();
forDraft = 0;
forPublished = 0;
binaryValues?.Get(docId, forValue);
if (forValue.Bytes.Length == 2)
{
forDraft = forValue.Bytes[0];
forPublished = forValue.Bytes[1];
return true;
}
return false;
}
private static BytesRef GetValue(byte forDraft, byte forPublished)
{
return new BytesRef(new byte[] { forDraft, forPublished });
}
}
}

68
src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs
View File

@ -12,7 +12,6 @@ using System.Text;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Search;
using Lucene.Net.Util;
using Squidex.Domain.Apps.Core.Contents;
using Squidex.Infrastructure;
using Squidex.Infrastructure.Json.Objects;
@ -23,48 +22,42 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{
private const string MetaId = "_id";
private const string MetaKey = "_key";
private const string MetaFor = "_fd";
private const int ForDraftIndex = 0;
private const int ForPublishedIndex = 1;
private readonly IndexWriter indexWriter;
private readonly IndexSearcher indexSearcher;
private readonly BinaryDocValues indexValues;
private readonly IndexState indexState;
private readonly Guid id;
public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id)
public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, IndexState indexState, Guid id)
{
this.indexWriter = indexWriter;
this.indexSearcher = indexSearcher;
this.indexValues = indexValues;
this.indexState = indexState;
this.id = id;
}
public static BinaryDocValues CreateValues(IndexReader indexReader)
{
return MultiDocValues.GetBinaryValues(indexReader, MetaFor);
}
public void Delete()
{
indexWriter.DeleteDocuments(new Term(MetaId, id.ToString()));
}
public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result)
public static bool TryGetId(int docId, Scope scope, IndexReader reader, IndexState indexState, out Guid result)
{
result = Guid.Empty;
if (!TryGet(docId, values, out var @for))
if (!indexState.TryGet(docId, out var draft, out var published))
{
return false;
}
if (scope == Scope.Draft && @for[ForDraftIndex] != 1)
if (scope == Scope.Draft && draft != 1)
{
return false;
}
if (scope == Scope.Published && @for[ForPublishedIndex] != 1)
if (scope == Scope.Published && published != 1)
{
return false;
}
@ -81,7 +74,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
return true;
}
public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft)
public void Index(NamedContentData dataDraft, NamedContentData data, bool onlyDraft)
{
var converted = CreateDocument(dataDraft);
@ -95,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
}
else
{
var isPublished = IsForPublished(docId, values);
var isPublished = IsForPublished(docId);
if (!onlyDraft && docId > 0 && isPublished)
{
@ -174,7 +167,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{
var term = new Term(MetaKey, BuildKey(draft));
indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
indexState.Index(term, id, draft, forDraft, forPublished);
}
private int GetPublishedDocument()
@ -190,18 +183,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{
document.RemoveField(MetaId);
document.RemoveField(MetaKey);
document.RemoveField(MetaFor);
var docId = id.ToString();
var docKey = BuildKey(draft);
var contentId = id.ToString();
var contentKey = BuildKey(draft);
document.AddStringField(MetaId, docId, Field.Store.YES);
document.AddStringField(MetaKey, docKey, Field.Store.YES);
document.AddStringField(MetaId, contentId, Field.Store.YES);
document.AddStringField(MetaKey, contentKey, Field.Store.YES);
document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
indexState.Index(document, forDraft, forPublished);
// indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
indexWriter.UpdateDocument(new Term(MetaKey, docKey), document);
indexWriter.UpdateDocument(new Term(MetaKey, contentKey), document);
}
}
@ -227,32 +218,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
}
}
private static bool IsForPublished(int docId, BinaryDocValues values)
{
return TryGet(docId, values, out var @for) && @for[1] == 1;
}
private static bool TryGet(int docId, BinaryDocValues values, out byte[] result)
{
var forValue = new BytesRef();
values?.Get(docId, forValue);
if (forValue.Bytes.Length == 2)
{
result = forValue.Bytes;
return true;
}
else
{
result = null;
return false;
}
}
private static BytesRef GetValue(byte forDraft, byte forPublished)
private bool IsForPublished(int docId)
{
return new BytesRef(new byte[] { forDraft, forPublished });
return indexState.TryGet(docId, out _, out var p) && p == 1;
}
private string BuildKey(byte draft)

52
src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs
View File

@ -38,11 +38,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private IndexWriter indexWriter;
private IndexReader indexReader;
private IndexSearcher indexSearcher;
private BinaryDocValues indexValues;
private IndexState indexState;
private QueryParser queryParser;
private HashSet<string> currentLanguages;
private long updates;
private long updatesNotWritten;
public TextIndexerGrain(IAssetStore assetStore)
{
@ -71,15 +70,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
if (indexWriter.NumDocs > 0)
{
indexReader = indexWriter.GetReader(false);
indexSearcher = new IndexSearcher(indexReader);
indexValues = TextIndexContent.CreateValues(indexReader);
OpenReader();
}
else
{
indexState = new IndexState(indexReader, indexWriter);
}
}
public Task DeleteAsync(Guid id)
{
var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
content.Delete();
@ -88,16 +89,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft)
{
var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft);
content.Index(data.Value.DataDraft, data.Value.Data, onlyDraft);
return TryFlushAsync();
}
public Task CopyAsync(Guid id, bool fromDraft)
{
var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
var content = new TextIndexContent(indexWriter, indexSearcher, indexState, id);
content.Copy(fromDraft);
@ -118,7 +119,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
foreach (var hit in hits)
{
if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id))
if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexState, out var id))
{
result.Add(id);
}
@ -153,21 +154,20 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private async Task TryFlushAsync()
{
updates++;
updatesNotWritten++;
if (updates >= MaxUpdates)
{
await FlushAsync(true);
await FlushAsync();
}
else
{
await FlushAsync();
OpenReader();
timer?.Dispose();
try
{
timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay);
timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay);
}
catch (InvalidOperationException)
{
@ -176,23 +176,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
}
}
public async Task FlushAsync(bool write = false)
public async Task FlushAsync()
{
if (updates > 0 && indexWriter != null)
{
indexWriter.Commit();
indexWriter.Flush(true, true);
indexReader?.Dispose();
indexReader = indexWriter.GetReader(false);
indexSearcher = new IndexSearcher(indexReader);
indexValues = TextIndexContent.CreateValues(indexReader);
updates = 0;
}
OpenReader();
if (updatesNotWritten > 0 && write)
{
var commit = snapshotter.Snapshot();
try
{
@ -203,7 +195,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
snapshotter.Release(commit);
}
updatesNotWritten = 0;
updates = 0;
}
timer?.Dispose();
@ -211,7 +203,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
public async Task DeactivateAsync(bool deleteFolder = false)
{
await FlushAsync(true);
await FlushAsync();
indexWriter?.Dispose();
indexWriter = null;
@ -224,5 +216,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
directory.Delete(true);
}
}
private void OpenReader()
{
indexReader?.Dispose();
indexReader = indexWriter.GetReader(true);
indexSearcher = new IndexSearcher(indexReader);
indexState = new IndexState(indexReader, indexWriter);
}
}
}

1
tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs
View File

@ -213,7 +213,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true);
await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true);
await sut.FlushAsync();
}
private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false)

Write Preview
Loading…
Cancel
Save