From 4c6a36725b10957642c457fe3e6c826f7de8a5a5 Mon Sep 17 00:00:00 2001 From: Sebastian Stehle Date: Fri, 29 Mar 2019 19:47:42 +0100 Subject: [PATCH] Test for improved flushing. --- .../Contents/MongoContentRepository.cs | 2 +- .../Contents/Text/GrainTextIndexer.cs | 14 +- .../Contents/Text/ITextIndexer.cs | 2 +- .../Contents/Text/Scope.cs | 15 ++ .../Contents/Text/SearchContext.cs | 2 +- .../Contents/Text/TextIndexContent.cs | 120 ++++++----- .../Contents/Text/TextIndexerGrain.cs | 36 ++-- .../Contents/Text/GrainTextIndexerTests.cs | 12 +- .../Contents/Text/TextIndexerGrainTests.cs | 194 +++++++++++------- .../Pipeline/AppResolverTests.cs | 3 - .../Migrations/BuildFullTextIndices.cs | 2 +- 11 files changed, 244 insertions(+), 158 deletions(-) create mode 100644 src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs index 03e74bf11..dfe3b615f 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs @@ -71,7 +71,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents { var useDraft = UseDraft(status); - var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft); + var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft ? Scope.Draft : Scope.Published); if (fullTextIds?.Count == 0) { diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs index 81477f7aa..7ee503460 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs @@ -56,7 +56,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return Task.CompletedTask; } - public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft) + public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData dataDraft, NamedContentData data) { var index = grainFactory.GetGrain(schemaId); @@ -64,7 +64,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { try { - await index.IndexAsync(id, new IndexData { Data = data, DataDraft = dataDraft }, false); + await index.IndexAsync(id, new IndexData { DataDraft = dataDraft, Data = data }, false); } catch (Exception ex) { @@ -126,10 +126,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private J Data(NamedContentData data) { - return new IndexData { Data = data }; + return new IndexData { DataDraft = data }; } - public async Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false) + public async Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published) { if (string.IsNullOrWhiteSpace(queryText)) { @@ -140,17 +140,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text using (Profiler.TraceMethod()) { - var context = CreateContext(app, useDraft); + var context = CreateContext(app, scope); return await index.SearchAsync(queryText, context); } } - private static SearchContext CreateContext(IAppEntity app, bool useDraft) + private static SearchContext CreateContext(IAppEntity app, Scope scope) { var languages = new HashSet(app.LanguagesConfig.Select(x => x.Key)); - return new SearchContext { Languages = languages, IsDraft = useDraft }; + return new SearchContext { Languages = languages, Scope = scope }; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs index 1e2d62618..69eb6449c 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs @@ -17,6 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { Task IndexAsync(Guid schemaId, Guid contentId, NamedContentData draftData, NamedContentData data); - Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false); + Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published); } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs new file mode 100644 index 000000000..2d302c600 --- /dev/null +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs @@ -0,0 +1,15 @@ +// ========================================================================== +// Squidex Headless CMS +// ========================================================================== +// Copyright (c) Squidex UG (haftungsbeschraenkt) +// All rights reserved. Licensed under the MIT license. +// ========================================================================== + +namespace Squidex.Domain.Apps.Entities.Contents.Text +{ + public enum Scope + { + Draft, + Published + } +} diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs index 01bd8f78a..619cc7a1d 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs @@ -11,7 +11,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { public sealed class SearchContext { - public bool IsDraft { get; set; } + public Scope Scope { get; set; } public HashSet Languages { get; set; } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs index ff97da642..6d0f36076 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs @@ -7,6 +7,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using Lucene.Net.Documents; using Lucene.Net.Index; @@ -27,12 +28,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private const int ForPublishedIndex = 1; private readonly IndexWriter indexWriter; private readonly IndexSearcher indexSearcher; + private readonly BinaryDocValues indexValues; private readonly Guid id; - public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, Guid id) + public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id) { this.indexWriter = indexWriter; this.indexSearcher = indexSearcher; + this.indexValues = indexValues; this.id = id; } @@ -42,34 +45,35 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return MultiDocValues.GetBinaryValues(indexReader, MetaFor); } - public static bool TryGetId(int docId, bool forDraft, IndexReader reader, BinaryDocValues values, out Guid result) + public void Delete() { - result = Guid.Empty; - - var forValue = new BytesRef(); + indexWriter.DeleteDocuments(new Term(MetaId, id.ToString())); + } - values.Get(docId, forValue); + public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result) + { + result = Guid.Empty; - if (forValue.Bytes.Length != 2) + if (!TryGet(docId, values, out var @for)) { return false; } - if (forDraft && forValue.Bytes[ForDraftIndex] != 1) + if (scope == Scope.Draft && @for[ForDraftIndex] != 1) { return false; } - if (!forDraft && forValue.Bytes[ForPublishedIndex] != 1) + if (scope == Scope.Published && @for[ForPublishedIndex] != 1) { return false; } var document = reader.Document(docId); - var id = document.Get(MetaId); + var idString = document.Get(MetaId); - if (!Guid.TryParse(id, out result)) + if (!Guid.TryParse(idString, out result)) { return false; } @@ -77,51 +81,48 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return true; } - public void Index(NamedContentData data, NamedContentData dataDraft, bool onlyDraft) + public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft) { - var converted = CreateDocument(data); + var converted = CreateDocument(dataDraft); Upsert(converted, 1, 1, 0); - var existing = GetDocument(1); - - if (dataDraft != null) - { - converted = CreateDocument(dataDraft); + var docId = GetPublishedDocument(); - Upsert(converted, 0, 0, 1); - } - else if (IsForPublished(existing)) + if (data != null) { - Upsert(converted, 0, 0, 1); + Upsert(CreateDocument(data), 0, 0, 1); } - else if (existing == null) + else { - Upsert(converted, 0, 0, 0); - } - } - - private static bool IsForPublished(Document existing) - { - return existing?.GetField(MetaFor)?.GetBinaryValue().Bytes[ForPublishedIndex] == 1; - } + var isPublished = IsForPublished(docId, values); - public void Delete() - { - indexWriter.DeleteDocuments(new Term(MetaId, id.ToString())); + if (!onlyDraft && docId > 0 && isPublished) + { + Upsert(converted, 0, 0, 1); + } + else if (!onlyDraft) + { + Upsert(converted, 0, 0, 0); + } + else + { + Update(0, 0, isPublished ? (byte)1 : (byte)0); + } + } } public void Copy(bool fromDraft) { if (fromDraft) { - Update(1, 0, 0); - Update(0, 1, 1); + Update(1, 1, 0); + Update(0, 0, 1); } else { - Update(1, 1, 1); - Update(0, 0, 0); + Update(1, 0, 0); + Update(0, 1, 1); } } @@ -176,21 +177,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished)); } - private Document GetDocument(byte draft) + private int GetPublishedDocument() { - if (indexSearcher == null) - { - return null; - } + var docs = indexSearcher?.Search(new TermQuery(new Term(MetaKey, BuildKey(0))), 1); - var docs = indexSearcher.Search(new TermQuery(new Term(MetaKey, BuildKey(draft))), 1); - - if (docs.ScoreDocs.Length > 0) - { - return indexSearcher.Doc(docs.ScoreDocs[0].Doc); - } - - return null; + return docs?.ScoreDocs.FirstOrDefault()?.Doc ?? 0; } private void Upsert(Document document, byte draft, byte forDraft, byte forPublished) @@ -209,8 +200,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished)); - indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); - indexWriter.AddDocument(document); + // indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); + indexWriter.UpdateDocument(new Term(MetaKey, docKey), document); } } @@ -236,6 +227,29 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } + private static bool IsForPublished(int docId, BinaryDocValues values) + { + return TryGet(docId, values, out var @for) && @for[1] == 1; + } + + private static bool TryGet(int docId, BinaryDocValues values, out byte[] result) + { + var forValue = new BytesRef(); + + values?.Get(docId, forValue); + + if (forValue.Bytes.Length == 2) + { + result = forValue.Bytes; + return true; + } + else + { + result = null; + return false; + } + } + private static BytesRef GetValue(byte forDraft, byte forPublished) { return new BytesRef(new byte[] { forDraft, forPublished }); diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index cb0f0a202..1af1d4dc7 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -12,7 +12,6 @@ using System.Linq; using System.Threading.Tasks; using Lucene.Net.Analysis; using Lucene.Net.Index; -using Lucene.Net.Queries; using Lucene.Net.QueryParsers.Classic; using Lucene.Net.Search; using Lucene.Net.Store; @@ -43,6 +42,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private QueryParser queryParser; private HashSet currentLanguages; private long updates; + private long updatesNotWritten; public TextIndexerGrain(IAssetStore assetStore) { @@ -79,7 +79,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public Task DeleteAsync(Guid id) { - var content = new TextIndexContent(indexWriter, indexSearcher, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); content.Delete(); @@ -88,16 +88,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public Task IndexAsync(Guid id, J data, bool onlyDraft) { - var content = new TextIndexContent(indexWriter, indexSearcher, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); - content.Index(data.Value.Data, data.Value.DataDraft, onlyDraft); + content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft); return TryFlushAsync(); } public Task CopyAsync(Guid id, bool fromDraft) { - var content = new TextIndexContent(indexWriter, indexSearcher, id); + var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id); content.Copy(fromDraft); @@ -118,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text foreach (var hit in hits) { - if (TextIndexContent.TryGetId(hit.Doc, context.IsDraft, indexReader, indexValues, out var id)) + if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id)) { result.Add(id); } @@ -153,18 +153,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private async Task TryFlushAsync() { updates++; + updatesNotWritten++; if (updates >= MaxUpdates) { - await FlushAsync(); + await FlushAsync(true); } else { + await FlushAsync(); + timer?.Dispose(); try { - timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay); + timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay); } catch (InvalidOperationException) { @@ -173,7 +176,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } } - public async Task FlushAsync() + public async Task FlushAsync(bool write = false) { if (updates > 0 && indexWriter != null) { @@ -185,6 +188,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text indexSearcher = new IndexSearcher(indexReader); indexValues = TextIndexContent.CreateValues(indexReader); + updates = 0; + } + + if (updatesNotWritten > 0 && write) + { var commit = snapshotter.Snapshot(); try { @@ -195,17 +203,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text snapshotter.Release(commit); } - updates = 0; - } - else - { - timer?.Dispose(); + updatesNotWritten = 0; } + + timer?.Dispose(); } public async Task DeactivateAsync(bool deleteFolder = false) { - await TryFlushAsync(); + await FlushAsync(true); indexWriter?.Dispose(); indexWriter = null; diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs index 281f4413f..9b44668f5 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs @@ -44,7 +44,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text var data = new NamedContentData(); var dataDraft = new NamedContentData(); - await sut.IndexAsync(schemaId, contentId, data, dataDraft); + await sut.IndexAsync(schemaId, contentId, dataDraft, data); A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data && x.Value.DataDraft == dataDraft), false)) .MustHaveHappened(); @@ -66,7 +66,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await sut.On(E(new ContentCreated { Data = data })); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), true)) + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.DataDraft == data), true)) .MustHaveHappened(); } @@ -77,7 +77,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await sut.On(E(new ContentUpdated { Data = data })); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), false)) + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.DataDraft == data), false)) .MustHaveHappened(); } @@ -88,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await sut.On(E(new ContentUpdateProposed { Data = data })); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), true)) + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.DataDraft == data), true)) .MustHaveHappened(); } @@ -161,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text A.CallTo(() => grain.SearchAsync("Search", A.Ignored)) .Returns(foundIds); - var ids = await sut.SearchAsync("Search", GetApp(), schemaId, true); + var ids = await sut.SearchAsync("Search", GetApp(), schemaId, Scope.Draft); Assert.Equal(foundIds, ids); } @@ -169,7 +169,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text [Fact] public async Task Should_not_call_grain_when_input_is_empty() { - var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, false); + var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, Scope.Published); Assert.Null(ids); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs index b53853422..f1f84a48e 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs @@ -28,8 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { context = new SearchContext { - Languages = new HashSet { "de", "en" }, - IsDraft = true + Languages = new HashSet { "de", "en" } }; sut = new TextIndexerGrain(assetStore); @@ -41,10 +40,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text sut.OnDeactivateAsync().Wait(); } + [Fact] + public async Task Should_throw_exception_for_invalid_query() + { + await Assert.ThrowsAsync(() => sut.SearchAsync("~hello", context)); + } + [Fact] public async Task Should_read_index_and_retrieve() { - await AddInvariantContent(); + await AddInvariantContent("Hello", "World", false); await sut.DeactivateAsync(); @@ -53,11 +58,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { await other.ActivateAsync(schemaId); - var foundHello = await other.SearchAsync("Hello", context); - var foundWorld = await other.SearchAsync("World", context); - - Assert.Equal(ids1, foundHello); - Assert.Equal(ids2, foundWorld); + await TestSearchAsync(ids1, "Hello", grain: other); + await TestSearchAsync(ids2, "World", grain: other); } finally { @@ -68,104 +70,131 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text [Fact] public async Task Should_index_invariant_content_and_retrieve() { - await AddInvariantContent(); - - var foundHello = await sut.SearchAsync("Hello", context); - var foundWorld = await sut.SearchAsync("World", context); + await AddInvariantContent("Hello", "World", false); - Assert.Equal(ids1, foundHello); - Assert.Equal(ids2, foundWorld); + await TestSearchAsync(ids1, "Hello"); + await TestSearchAsync(ids2, "World"); } [Fact] public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() { - await AddInvariantContent(); + await AddInvariantContent("Hello", "World", false); - var foundHello = await sut.SearchAsync("helo~", context); - var foundWorld = await sut.SearchAsync("wold~", context); - - Assert.Equal(ids1, foundHello); - Assert.Equal(ids2, foundWorld); + await TestSearchAsync(ids1, "helo~"); + await TestSearchAsync(ids2, "wold~"); } [Fact] - public async Task Should_delete_documents_from_index() + public async Task Should_update_draft_only() { - await AddInvariantContent(); - - await sut.DeleteAsync(ids1[0]); - await sut.FlushAsync(); + await AddInvariantContent("Hello", "World", false); + await AddInvariantContent("Hallo", "Welt", false); - var helloIds = await sut.SearchAsync("Hello", context); + await TestSearchAsync(null, "Hello", Scope.Draft); + await TestSearchAsync(null, "Hello", Scope.Published); - var worldIds = await sut.SearchAsync("World", context); - - Assert.Empty(helloIds); - Assert.Equal(ids2, worldIds); + await TestSearchAsync(ids1, "Hallo", Scope.Draft); + await TestSearchAsync(null, "Hallo", Scope.Published); } [Fact] - public async Task Should_search_by_field() + public async Task Should_also_update_published_after_copy() { - await AddLocalizedContent(); + await AddInvariantContent("Hello", "World", false); - var emptyGerman = await sut.SearchAsync("de:City", context); - var emptyEnglish = await sut.SearchAsync("en:Stadt", context); + await CopyAsync(true); - Assert.Empty(emptyGerman); - Assert.Empty(emptyEnglish); + await AddInvariantContent("Hallo", "Welt", false); + + await TestSearchAsync(null, "Hello", Scope.Draft); + await TestSearchAsync(null, "Hello", Scope.Published); + + await TestSearchAsync(ids1, "Hallo", Scope.Draft); + await TestSearchAsync(ids1, "Hallo", Scope.Published); } [Fact] - public async Task Should_index_localized_content_and_retrieve() + public async Task Should_simulate_content_reversion() { - await AddLocalizedContent(); + await AddInvariantContent("Hello", "World", false); + + await CopyAsync(true); + + await AddInvariantContent("Hallo", "Welt", true); + + await TestSearchAsync(null, "Hello", Scope.Draft); + await TestSearchAsync(ids1, "Hello", Scope.Published); - var german1 = await sut.SearchAsync("Stadt", context); - var german2 = await sut.SearchAsync("and", context); + await TestSearchAsync(ids1, "Hallo", Scope.Draft); + await TestSearchAsync(null, "Hallo", Scope.Published); - var germanStopwordsIds = await sut.SearchAsync("und", context); + await CopyAsync(false); - Assert.Equal(ids1, german1); - Assert.Equal(ids1, german2); - Assert.Equal(ids2, germanStopwordsIds); + await TestSearchAsync(ids1, "Hello", Scope.Draft); + await TestSearchAsync(ids1, "Hello", Scope.Published); - var english1 = await sut.SearchAsync("City", context); - var english2 = await sut.SearchAsync("und", context); + await TestSearchAsync(null, "Hallo", Scope.Draft); + await TestSearchAsync(null, "Hallo", Scope.Published); - var englishStopwordsIds = await sut.SearchAsync("and", context); + await AddInvariantContent("Hallo", "Welt", true); - Assert.Equal(ids2, english1); - Assert.Equal(ids2, english2); - Assert.Equal(ids1, englishStopwordsIds); + await TestSearchAsync(null, "Hello", Scope.Draft); + await TestSearchAsync(ids1, "Hello", Scope.Published); + + await TestSearchAsync(ids1, "Hallo", Scope.Draft); + await TestSearchAsync(null, "Hallo", Scope.Published); } [Fact] - public async Task Should_throw_exception_for_invalid_query() + public async Task Should_also_retrieve_published_content_after_copy() { - await AddInvariantContent(); + await AddInvariantContent("Hello", "World", false); - await Assert.ThrowsAsync(() => sut.SearchAsync("~hello", context)); + await TestSearchAsync(ids1, "Hello", Scope.Draft); + await TestSearchAsync(null, "Hello", Scope.Published); + + await CopyAsync(false); + + await TestSearchAsync(ids1, "Hello", Scope.Draft); + await TestSearchAsync(ids1, "Hello", Scope.Published); } [Fact] - public async Task Should_also_retrieve_published_content_after_copy() + public async Task Should_delete_documents_from_index() { - await AddInvariantContent(); + await AddInvariantContent("Hello", "World", false); - context.IsDraft = false; + await TestSearchAsync(ids1, "Hello"); + await TestSearchAsync(ids2, "World"); - var foundHello1 = await sut.SearchAsync("Hello", context); + await DeleteAsync(ids1[0]); - Assert.Empty(foundHello1); + await TestSearchAsync(null, "Hello"); + await TestSearchAsync(ids2, "World"); + } - await sut.CopyAsync(ids1[0], true); - await sut.FlushAsync(); + [Fact] + public async Task Should_search_by_field() + { + await AddLocalizedContent(); - var foundHello2 = await sut.SearchAsync("Hello", context); + await TestSearchAsync(null, "de:city"); + await TestSearchAsync(null, "en:Stadt"); + } - Assert.Equal(ids1, foundHello2); + [Fact] + public async Task Should_index_localized_content_and_retrieve() + { + await AddLocalizedContent(); + + await TestSearchAsync(ids1, "Stadt"); + await TestSearchAsync(ids1, "and"); + await TestSearchAsync(ids2, "und"); + + await TestSearchAsync(ids2, "City"); + await TestSearchAsync(ids2, "und"); + await TestSearchAsync(ids1, "and"); } private async Task AddLocalizedContent() @@ -182,29 +211,54 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text new ContentFieldData() .AddValue("en", "City and Surroundings und sonstiges")); - await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, true); - await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, true); + await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true); + await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true); await sut.FlushAsync(); } - private async Task AddInvariantContent() + private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false) { var data1 = new NamedContentData() .AddField("test", new ContentFieldData() - .AddValue("iv", "Hello")); + .AddValue("iv", text1)); var data2 = new NamedContentData() .AddField("test", new ContentFieldData() - .AddValue("iv", "World")); + .AddValue("iv", text2)); - await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, true); - await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, true); + await sut.IndexAsync(ids1[0], new IndexData { DataDraft = data1 }, onlyDraft); + await sut.IndexAsync(ids2[0], new IndexData { DataDraft = data2 }, onlyDraft); + } - await sut.FlushAsync(); + private async Task DeleteAsync(Guid id) + { + await sut.DeleteAsync(id); + } + + private async Task CopyAsync(bool fromDraft) + { + await sut.CopyAsync(ids1[0], fromDraft); + await sut.CopyAsync(ids2[0], fromDraft); + } + + private async Task TestSearchAsync(List expected, string text, Scope target = Scope.Draft, TextIndexerGrain grain = null) + { + context.Scope = target; + + var result = await (grain ?? sut).SearchAsync(text, context); + + if (expected != null) + { + Assert.Equal(expected, result); + } + else + { + Assert.Empty(result); + } } } } diff --git a/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs b/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs index a1585f58c..4aebe79cb 100644 --- a/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs +++ b/tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs @@ -8,10 +8,8 @@ using System.Collections.Generic; using System.Linq; using System.Security.Claims; -using System.Security.Principal; using System.Threading.Tasks; using FakeItEasy; -using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc.Abstractions; @@ -22,7 +20,6 @@ using Squidex.Domain.Apps.Core.Apps; using Squidex.Domain.Apps.Entities; using Squidex.Domain.Apps.Entities.Apps; using Squidex.Infrastructure.Security; -using Squidex.Shared; using Squidex.Shared.Identity; using Xunit; diff --git a/tools/Migrate_01/Migrations/BuildFullTextIndices.cs b/tools/Migrate_01/Migrations/BuildFullTextIndices.cs index 42732e496..de7e9f754 100644 --- a/tools/Migrate_01/Migrations/BuildFullTextIndices.cs +++ b/tools/Migrate_01/Migrations/BuildFullTextIndices.cs @@ -39,7 +39,7 @@ namespace Migrate_01.Migrations var worker = new ActionBlock(state => { - return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.Data, state.DataDraft); + return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.DataDraft, state.Data); }, new ExecutionDataflowBlockOptions {