From 68c16308ede38e3d94b401ebe2e73652db7a7d8f Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 29 Mar 2019 15:45:26 +0100 Subject: [PATCH] Finalized. --- .../Contents/ContentFieldData.cs | 5 + .../Templates/CreateBlogCommandMiddleware.cs | 8 +- .../CreateProfileCommandMiddleware.cs | 6 +- .../Contents/Text/TextIndexContent.cs | 125 ++++++++++++++---- .../Contents/Text/TextIndexerGrain.cs | 22 ++- .../Contents/Text/TextIndexerGrainTests.cs | 10 +- 6 files changed, 123 insertions(+), 53 deletions(-) diff --git a/src/Squidex.Domain.Apps.Core.Model/Contents/ContentFieldData.cs b/src/Squidex.Domain.Apps.Core.Model/Contents/ContentFieldData.cs index 41afa54f3..8bc60388a 100644 --- a/src/Squidex.Domain.Apps.Core.Model/Contents/ContentFieldData.cs +++ b/src/Squidex.Domain.Apps.Core.Model/Contents/ContentFieldData.cs @@ -19,6 +19,11 @@ namespace Squidex.Domain.Apps.Core.Contents { } + public ContentFieldData AddValue(object value) + { + return AddJsonValue(InvariantPartitioning.Instance.Master.Key, JsonValue.Create(value)); + } + public ContentFieldData AddValue(string key, object value) { return AddJsonValue(key, JsonValue.Create(value)); diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateBlogCommandMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateBlogCommandMiddleware.cs index 35576796a..95ac51bae 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateBlogCommandMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateBlogCommandMiddleware.cs @@ -60,10 +60,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.Templates new NamedContentData() .AddField("title", new ContentFieldData() - .AddValue("iv", "My first post with Squidex")) + .AddValue("My first post with Squidex")) .AddField("text", new ContentFieldData() - .AddValue("iv", "Just created a blog with Squidex. I love it!")), + .AddValue("Just created a blog with Squidex. I love it!")), Publish = true }); } @@ -79,10 +79,10 @@ namespace Squidex.Domain.Apps.Entities.Apps.Templates new NamedContentData() .AddField("title", new ContentFieldData() - .AddValue("iv", "About Me")) + .AddValue("About Me")) .AddField("text", new ContentFieldData() - .AddValue("iv", "I love Squidex and SciFi!")), + .AddValue("I love Squidex and SciFi!")), Publish = true }); } diff --git a/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateProfileCommandMiddleware.cs b/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateProfileCommandMiddleware.cs index 85cac0c86..59e46b4f5 100644 --- a/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateProfileCommandMiddleware.cs +++ b/src/Squidex.Domain.Apps.Entities/Apps/Templates/CreateProfileCommandMiddleware.cs @@ -64,13 +64,13 @@ namespace Squidex.Domain.Apps.Entities.Apps.Templates new NamedContentData() .AddField("firstName", new ContentFieldData() - .AddValue("iv", "John")) + .AddValue("John")) .AddField("lastName", new ContentFieldData() - .AddValue("iv", "Doe")) + .AddValue("Doe")) .AddField("profession", new ContentFieldData() - .AddValue("iv", "Software Developer")), + .AddValue("Software Developer")), Publish = true }); } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs index ec480ea70..d6becb142 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs @@ -11,6 +11,7 @@ using System.Text; using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Search; +using Lucene.Net.Util; using Squidex.Domain.Apps.Core.Contents; using Squidex.Infrastructure; using Squidex.Infrastructure.Json.Objects; @@ -19,10 +20,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { internal sealed class TextIndexContent { - public const string MetaId = "_id"; - public const string MetaKey = "_key"; - public const string MetaDraft = "_dd"; - + private const string MetaId = "_id"; + private const string MetaKey = "_key"; + private const string MetaFor = "_fd"; + private const int ForDraftIndex = 0; + private const int ForPublishedIndex = 1; private readonly IndexWriter indexWriter; private readonly IndexSearcher indexSearcher; private readonly Guid id; @@ -35,11 +37,67 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text this.id = id; } - public void Index(NamedContentData data, bool isDraft) + public static BinaryDocValues CreateValues(IndexReader indexReader) + { + return MultiDocValues.GetBinaryValues(indexReader, MetaFor); + } + + public static bool TryGetId(int docId, bool forDraft, IndexReader reader, BinaryDocValues values, out Guid result) + { + result = Guid.Empty; + + var forValue = new BytesRef(); + + values.Get(docId, forValue); + + if (forValue.Bytes.Length != 2) + { + return false; + } + + if (forDraft && forValue.Bytes[ForDraftIndex] != 1) + { + return false; + } + + if (!forDraft && forValue.Bytes[ForPublishedIndex] != 1) + { + return false; + } + + var document = reader.Document(docId); + + var id = document.Get(MetaId); + + if (!Guid.TryParse(id, out result)) + { + return false; + } + + return true; + } + + public void Index(NamedContentData data, bool onlyDraft) { var converted = CreateDocument(data); - Upsert(converted, isDraft); + Upsert(converted, 1, 1, 0); + + var existing = GetDocument(1); + + if (IsForPublished(existing)) + { + Upsert(converted, 0, 0, 1); + } + else if (existing == null) + { + Upsert(converted, 0, 0, 0); + } + } + + private static bool IsForPublished(Document existing) + { + return existing?.GetField(MetaFor)?.GetBinaryValue().Bytes[ForPublishedIndex] == 1; } public void Delete() @@ -49,9 +107,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public void Copy(bool fromDraft) { - var published = GetDocument(fromDraft); - - Upsert(published, !fromDraft); + if (fromDraft) + { + Update(1, 1, 0); + Update(0, 0, 1); + } + else + { + Update(1, 1, 1); + Update(0, 0, 0); + } } private Document CreateDocument(NamedContentData data) @@ -98,14 +163,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return document; } - private Document GetDocument(bool draft) + private void Update(byte draft, byte forDraft, byte forPublished) + { + var term = new Term(MetaKey, BuildKey(draft)); + + indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished)); + } + + private Document GetDocument(byte draft) { if (indexSearcher == null) { return null; } - var docs = indexSearcher.Search(new TermQuery(new Term(MetaKey, BuildKey(BuildValue(draft)))), 1); + var docs = indexSearcher.Search(new TermQuery(new Term(MetaKey, BuildKey(draft))), 1); if (docs.ScoreDocs.Length > 0) { @@ -115,38 +187,27 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return null; } - private void Upsert(Document document, bool draft) + private void Upsert(Document document, byte draft, byte forDraft, byte forPublished) { if (document != null) { document.RemoveField(MetaId); document.RemoveField(MetaKey); - document.RemoveField(MetaDraft); - - var docDraft = BuildValue(draft); + document.RemoveField(MetaFor); var docId = id.ToString(); - var docKey = BuildKey(docDraft); + var docKey = BuildKey(draft); document.AddStringField(MetaId, docId, Field.Store.YES); document.AddStringField(MetaKey, docKey, Field.Store.YES); - document.AddStringField(MetaDraft, docDraft, Field.Store.YES); + + document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished)); indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); indexWriter.AddDocument(document); } } - private static string BuildValue(bool draft) - { - return draft ? "1" : "0"; - } - - private string BuildKey(string draft) - { - return $"{id}_{draft}"; - } - private static void AppendJsonText(IJsonValue value, Action appendText) { if (value.Type == JsonValueType.String) @@ -168,5 +229,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } } + + private static BytesRef GetValue(byte forDraft, byte forPublished) + { + return new BytesRef(new byte[] { forDraft, forPublished }); + } + + private string BuildKey(byte draft) + { + return $"{id}_{draft}"; + } } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index 4fb74d535..cb07d9463 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -31,8 +31,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private const int MaxUpdates = 100; private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30); private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); - private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, true.ToString())); - private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, false.ToString())); + private static readonly string[] Invariant = { InvariantPartitioning.Instance.Master.Key }; private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); private readonly IAssetStore assetStore; private IDisposable timer; @@ -40,6 +39,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private IndexWriter indexWriter; private IndexReader indexReader; private IndexSearcher indexSearcher; + private BinaryDocValues indexValues; private QueryParser queryParser; private HashSet currentLanguages; private long updates; @@ -73,6 +73,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { indexReader = indexWriter.GetReader(false); indexSearcher = new IndexSearcher(indexReader); + indexValues = TextIndexContent.CreateValues(indexReader); } } @@ -113,19 +114,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text if (indexReader != null) { - var filter = context.IsDraft ? DraftFilter : NoDraftFilter; - - var hits = indexSearcher.Search(query, filter, MaxResults).ScoreDocs; + var hits = indexSearcher.Search(query, MaxResults).ScoreDocs; foreach (var hit in hits) { - var document = indexReader.Document(hit.Doc); - - var idField = document.GetField(TextIndexContent.MetaId)?.GetStringValue(); - - if (idField != null && Guid.TryParse(idField, out var guid)) + if (TextIndexContent.TryGetId(hit.Doc, context.IsDraft, indexReader, indexValues, out var id)) { - result.Add(guid); + result.Add(id); } } } @@ -138,9 +133,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { if (queryParser == null || !currentLanguages.SetEquals(context.Languages)) { - var fields = - context.Languages - .Union(Enumerable.Repeat(InvariantPartitioning.Instance.Master.Key, 1)).ToArray(); + var fields = context.Languages.Union(Invariant).ToArray(); queryParser = new MultiFieldQueryParser(Version, fields, Analyzer); @@ -190,6 +183,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text indexReader?.Dispose(); indexReader = indexWriter.GetReader(false); indexSearcher = new IndexSearcher(indexReader); + indexValues = TextIndexContent.CreateValues(indexReader); var commit = snapshotter.Snapshot(); try diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs index 69f366f41..b53853422 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs @@ -152,7 +152,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text [Fact] public async Task Should_also_retrieve_published_content_after_copy() { - await AddLocalizedContent(); + await AddInvariantContent(); context.IsDraft = false; @@ -182,8 +182,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text new ContentFieldData() .AddValue("en", "City and Surroundings und sonstiges")); - await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, false); - await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, false); + await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, true); + await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, true); await sut.FlushAsync(); } @@ -201,8 +201,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text new ContentFieldData() .AddValue("iv", "World")); - await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, false); - await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, false); + await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, true); + await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, true); await sut.FlushAsync(); }