Test for improved flushing.

src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs

@@ -71,7 +71,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
             {
                 var useDraft = UseDraft(status);
 
-                var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft);
+                var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft ? Scope.Draft : Scope.Published);
 
                 if (fullTextIds?.Count == 0)
                 {

src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs

@@ -56,7 +56,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return Task.CompletedTask;
         }
 
-        public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft)
+        public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData dataDraft, NamedContentData data)
         {
             var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId);
 
@@ -64,7 +64,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             {
                 try
                 {
-                    await index.IndexAsync(id, new IndexData { Data = data, DataDraft = dataDraft }, false);
+                    await index.IndexAsync(id, new IndexData { DataDraft = dataDraft, Data = data }, false);
                 }
                 catch (Exception ex)
                 {
@@ -126,10 +126,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         private J<IndexData> Data(NamedContentData data)
         {
-            return new IndexData { Data = data };
+            return new IndexData { DataDraft = data };
         }
 
-        public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false)
+        public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published)
         {
             if (string.IsNullOrWhiteSpace(queryText))
             {
@@ -140,17 +140,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             using (Profiler.TraceMethod<GrainTextIndexer>())
             {
-                var context = CreateContext(app, useDraft);
+                var context = CreateContext(app, scope);
 
                 return await index.SearchAsync(queryText, context);
             }
         }
 
-        private static SearchContext CreateContext(IAppEntity app, bool useDraft)
+        private static SearchContext CreateContext(IAppEntity app, Scope scope)
         {
             var languages = new HashSet<string>(app.LanguagesConfig.Select(x => x.Key));
 
-            return new SearchContext { Languages = languages, IsDraft = useDraft };
+            return new SearchContext { Languages = languages, Scope = scope };
         }
     }
 }

src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs

@@ -17,6 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     {
         Task IndexAsync(Guid schemaId, Guid contentId, NamedContentData draftData, NamedContentData data);
 
-        Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false);
+        Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published);
     }
 }

src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs

@@ -0,0 +1,15 @@
+// ==========================================================================
+//  Squidex Headless CMS
+// ==========================================================================
+//  Copyright (c) Squidex UG (haftungsbeschraenkt)
+//  All rights reserved. Licensed under the MIT license.
+// ==========================================================================
+
+namespace Squidex.Domain.Apps.Entities.Contents.Text
+{
+    public enum Scope
+    {
+        Draft,
+        Published
+    }
+}

src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs

@@ -11,7 +11,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
     public sealed class SearchContext
     {
-        public bool IsDraft { get; set; }
+        public Scope Scope { get; set; }
 
         public HashSet<string> Languages { get; set; }
     }

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs

@@ -7,6 +7,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Text;
 using Lucene.Net.Documents;
 using Lucene.Net.Index;
@@ -27,12 +28,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private const int ForPublishedIndex = 1;
         private readonly IndexWriter indexWriter;
         private readonly IndexSearcher indexSearcher;
+        private readonly BinaryDocValues indexValues;
         private readonly Guid id;
 
-        public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, Guid id)
+        public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id)
         {
             this.indexWriter = indexWriter;
             this.indexSearcher = indexSearcher;
+            this.indexValues = indexValues;
 
             this.id = id;
         }
@@ -42,34 +45,35 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return MultiDocValues.GetBinaryValues(indexReader, MetaFor);
         }
 
-        public static bool TryGetId(int docId, bool forDraft, IndexReader reader, BinaryDocValues values, out Guid result)
+        public void Delete()
         {
-            result = Guid.Empty;
-
-            var forValue = new BytesRef();
+            indexWriter.DeleteDocuments(new Term(MetaId, id.ToString()));
+        }
 
-            values.Get(docId, forValue);
+        public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result)
+        {
+            result = Guid.Empty;
 
-            if (forValue.Bytes.Length != 2)
+            if (!TryGet(docId, values, out var @for))
             {
                 return false;
             }
 
-            if (forDraft && forValue.Bytes[ForDraftIndex] != 1)
+            if (scope == Scope.Draft && @for[ForDraftIndex] != 1)
             {
                 return false;
             }
 
-            if (!forDraft && forValue.Bytes[ForPublishedIndex] != 1)
+            if (scope == Scope.Published && @for[ForPublishedIndex] != 1)
             {
                 return false;
             }
 
             var document = reader.Document(docId);
 
-            var id = document.Get(MetaId);
+            var idString = document.Get(MetaId);
 
-            if (!Guid.TryParse(id, out result))
+            if (!Guid.TryParse(idString, out result))
             {
                 return false;
             }
@@ -77,51 +81,48 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return true;
         }
 
-        public void Index(NamedContentData data, NamedContentData dataDraft, bool onlyDraft)
+        public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft)
         {
-            var converted = CreateDocument(data);
+            var converted = CreateDocument(dataDraft);
 
             Upsert(converted, 1, 1, 0);
 
-            var existing = GetDocument(1);
-
-            if (dataDraft != null)
-            {
-                converted = CreateDocument(dataDraft);
+            var docId = GetPublishedDocument();
 
-                Upsert(converted, 0, 0, 1);
-            }
-            else if (IsForPublished(existing))
+            if (data != null)
             {
-                Upsert(converted, 0, 0, 1);
+                Upsert(CreateDocument(data), 0, 0, 1);
             }
-            else if (existing == null)
+            else
             {
-                Upsert(converted, 0, 0, 0);
-            }
-        }
-
-        private static bool IsForPublished(Document existing)
-        {
-            return existing?.GetField(MetaFor)?.GetBinaryValue().Bytes[ForPublishedIndex] == 1;
-        }
+                var isPublished = IsForPublished(docId, values);
 
-        public void Delete()
-        {
-            indexWriter.DeleteDocuments(new Term(MetaId, id.ToString()));
+                if (!onlyDraft && docId > 0 && isPublished)
+                {
+                    Upsert(converted, 0, 0, 1);
+                }
+                else if (!onlyDraft)
+                {
+                    Upsert(converted, 0, 0, 0);
+                }
+                else
+                {
+                    Update(0, 0, isPublished ? (byte)1 : (byte)0);
+                }
+            }
         }
 
         public void Copy(bool fromDraft)
         {
             if (fromDraft)
             {
-                Update(1, 0, 0);
-                Update(0, 1, 1);
+                Update(1, 1, 0);
+                Update(0, 0, 1);
             }
             else
             {
-                Update(1, 1, 1);
-                Update(0, 0, 0);
+                Update(1, 0, 0);
+                Update(0, 1, 1);
             }
         }
 
@@ -176,21 +177,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
         }
 
-        private Document GetDocument(byte draft)
+        private int GetPublishedDocument()
         {
-            if (indexSearcher == null)
-            {
-                return null;
-            }
+            var docs = indexSearcher?.Search(new TermQuery(new Term(MetaKey, BuildKey(0))), 1);
 
-            var docs = indexSearcher.Search(new TermQuery(new Term(MetaKey, BuildKey(draft))), 1);
-
-            if (docs.ScoreDocs.Length > 0)
-            {
-                return indexSearcher.Doc(docs.ScoreDocs[0].Doc);
-            }
-
-            return null;
+            return docs?.ScoreDocs.FirstOrDefault()?.Doc ?? 0;
         }
 
         private void Upsert(Document document, byte draft, byte forDraft, byte forPublished)
@@ -209,8 +200,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
                 document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
 
-                indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
-                indexWriter.AddDocument(document);
+                // indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
+                indexWriter.UpdateDocument(new Term(MetaKey, docKey), document);
             }
         }
 
@@ -236,6 +227,29 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
+        private static bool IsForPublished(int docId, BinaryDocValues values)
+        {
+            return TryGet(docId, values, out var @for) && @for[1] == 1;
+        }
+
+        private static bool TryGet(int docId, BinaryDocValues values, out byte[] result)
+        {
+            var forValue = new BytesRef();
+
+            values?.Get(docId, forValue);
+
+            if (forValue.Bytes.Length == 2)
+            {
+                result = forValue.Bytes;
+                return true;
+            }
+            else
+            {
+                result = null;
+                return false;
+            }
+        }
+
         private static BytesRef GetValue(byte forDraft, byte forPublished)
         {
             return new BytesRef(new byte[] { forDraft, forPublished });

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs

@@ -12,7 +12,6 @@ using System.Linq;
 using System.Threading.Tasks;
 using Lucene.Net.Analysis;
 using Lucene.Net.Index;
-using Lucene.Net.Queries;
 using Lucene.Net.QueryParsers.Classic;
 using Lucene.Net.Search;
 using Lucene.Net.Store;
@@ -43,6 +42,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private QueryParser queryParser;
         private HashSet<string> currentLanguages;
         private long updates;
+        private long updatesNotWritten;
 
         public TextIndexerGrain(IAssetStore assetStore)
         {
@@ -79,7 +79,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public Task DeleteAsync(Guid id)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
 
             content.Delete();
 
@@ -88,16 +88,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
         public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
 
-            content.Index(data.Value.Data, data.Value.DataDraft, onlyDraft);
+            content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft);
 
             return TryFlushAsync();
         }
 
         public Task CopyAsync(Guid id, bool fromDraft)
         {
-            var content = new TextIndexContent(indexWriter, indexSearcher, id);
+            var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
 
             content.Copy(fromDraft);
 
@@ -118,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
                     foreach (var hit in hits)
                     {
-                        if (TextIndexContent.TryGetId(hit.Doc, context.IsDraft, indexReader, indexValues, out var id))
+                        if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id))
                         {
                             result.Add(id);
                         }
@@ -153,18 +153,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private async Task TryFlushAsync()
         {
             updates++;
+            updatesNotWritten++;
 
             if (updates >= MaxUpdates)
             {
-                await FlushAsync();
+                await FlushAsync(true);
             }
             else
             {
+                await FlushAsync();
+
                 timer?.Dispose();
 
                 try
                 {
-                    timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay);
+                    timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay);
                 }
                 catch (InvalidOperationException)
                 {
@@ -173,7 +176,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             }
         }
 
-        public async Task FlushAsync()
+        public async Task FlushAsync(bool write = false)
         {
             if (updates > 0 && indexWriter != null)
             {
@@ -185,6 +188,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                 indexSearcher = new IndexSearcher(indexReader);
                 indexValues = TextIndexContent.CreateValues(indexReader);
 
+                updates = 0;
+            }
+
+            if (updatesNotWritten > 0 && write)
+            {
                 var commit = snapshotter.Snapshot();
                 try
                 {
@@ -195,17 +203,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                     snapshotter.Release(commit);
                 }
 
-                updates = 0;
-            }
-            else
-            {
-                timer?.Dispose();
+                updatesNotWritten = 0;
             }
+
+            timer?.Dispose();
         }
 
         public async Task DeactivateAsync(bool deleteFolder = false)
         {
-            await TryFlushAsync();
+            await FlushAsync(true);
 
             indexWriter?.Dispose();
             indexWriter = null;

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs

@@ -44,7 +44,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             var data = new NamedContentData();
             var dataDraft = new NamedContentData();
 
-            await sut.IndexAsync(schemaId, contentId, data, dataDraft);
+            await sut.IndexAsync(schemaId, contentId, dataDraft, data);
 
             A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && x.Value.DataDraft == dataDraft), false))
                 .MustHaveHappened();
@@ -66,7 +66,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             await sut.On(E(new ContentCreated { Data = data }));
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true))
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), true))
                 .MustHaveHappened();
         }
 
@@ -77,7 +77,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             await sut.On(E(new ContentUpdated { Data = data }));
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), false))
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), false))
                 .MustHaveHappened();
         }
 
@@ -88,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             await sut.On(E(new ContentUpdateProposed { Data = data }));
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true))
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), true))
                 .MustHaveHappened();
         }
 
@@ -161,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             A.CallTo(() => grain.SearchAsync("Search", A<SearchContext>.Ignored))
                 .Returns(foundIds);
 
-            var ids = await sut.SearchAsync("Search", GetApp(), schemaId, true);
+            var ids = await sut.SearchAsync("Search", GetApp(), schemaId, Scope.Draft);
 
             Assert.Equal(foundIds, ids);
         }
@@ -169,7 +169,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         [Fact]
         public async Task Should_not_call_grain_when_input_is_empty()
         {
-            var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, false);
+            var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, Scope.Published);
 
             Assert.Null(ids);
 

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs

@@ -28,8 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             context = new SearchContext
             {
-                Languages = new HashSet<string> { "de", "en" },
-                IsDraft = true
+                Languages = new HashSet<string> { "de", "en" }
             };
 
             sut = new TextIndexerGrain(assetStore);
@@ -41,10 +40,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             sut.OnDeactivateAsync().Wait();
         }
 
+        [Fact]
+        public async Task Should_throw_exception_for_invalid_query()
+        {
+            await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context));
+        }
+
         [Fact]
         public async Task Should_read_index_and_retrieve()
         {
-            await AddInvariantContent();
+            await AddInvariantContent("Hello", "World", false);
 
             await sut.DeactivateAsync();
 
@@ -53,11 +58,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             {
                 await other.ActivateAsync(schemaId);
 
-                var foundHello = await other.SearchAsync("Hello", context);
-                var foundWorld = await other.SearchAsync("World", context);
-
-                Assert.Equal(ids1, foundHello);
-                Assert.Equal(ids2, foundWorld);
+                await TestSearchAsync(ids1, "Hello", grain: other);
+                await TestSearchAsync(ids2, "World", grain: other);
             }
             finally
             {
@@ -68,104 +70,131 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         [Fact]
         public async Task Should_index_invariant_content_and_retrieve()
         {
-            await AddInvariantContent();
-
-            var foundHello = await sut.SearchAsync("Hello", context);
-            var foundWorld = await sut.SearchAsync("World", context);
+            await AddInvariantContent("Hello", "World", false);
 
-            Assert.Equal(ids1, foundHello);
-            Assert.Equal(ids2, foundWorld);
+            await TestSearchAsync(ids1, "Hello");
+            await TestSearchAsync(ids2, "World");
         }
 
         [Fact]
         public async Task Should_index_invariant_content_and_retrieve_with_fuzzy()
         {
-            await AddInvariantContent();
+            await AddInvariantContent("Hello", "World", false);
 
-            var foundHello = await sut.SearchAsync("helo~", context);
-            var foundWorld = await sut.SearchAsync("wold~", context);
-
-            Assert.Equal(ids1, foundHello);
-            Assert.Equal(ids2, foundWorld);
+            await TestSearchAsync(ids1, "helo~");
+            await TestSearchAsync(ids2, "wold~");
         }
 
         [Fact]
-        public async Task Should_delete_documents_from_index()
+        public async Task Should_update_draft_only()
         {
-            await AddInvariantContent();
-
-            await sut.DeleteAsync(ids1[0]);
-            await sut.FlushAsync();
+            await AddInvariantContent("Hello", "World", false);
+            await AddInvariantContent("Hallo", "Welt", false);
 
-            var helloIds = await sut.SearchAsync("Hello", context);
+            await TestSearchAsync(null, "Hello", Scope.Draft);
+            await TestSearchAsync(null, "Hello", Scope.Published);
 
-            var worldIds = await sut.SearchAsync("World", context);
-
-            Assert.Empty(helloIds);
-            Assert.Equal(ids2, worldIds);
+            await TestSearchAsync(ids1, "Hallo", Scope.Draft);
+            await TestSearchAsync(null, "Hallo", Scope.Published);
         }
 
         [Fact]
-        public async Task Should_search_by_field()
+        public async Task Should_also_update_published_after_copy()
         {
-            await AddLocalizedContent();
+            await AddInvariantContent("Hello", "World", false);
 
-            var emptyGerman = await sut.SearchAsync("de:City", context);
-            var emptyEnglish = await sut.SearchAsync("en:Stadt", context);
+            await CopyAsync(true);
 
-            Assert.Empty(emptyGerman);
-            Assert.Empty(emptyEnglish);
+            await AddInvariantContent("Hallo", "Welt", false);
+
+            await TestSearchAsync(null, "Hello", Scope.Draft);
+            await TestSearchAsync(null, "Hello", Scope.Published);
+
+            await TestSearchAsync(ids1, "Hallo", Scope.Draft);
+            await TestSearchAsync(ids1, "Hallo", Scope.Published);
         }
 
         [Fact]
-        public async Task Should_index_localized_content_and_retrieve()
+         public async Task Should_simulate_content_reversion()
         {
-            await AddLocalizedContent();
+            await AddInvariantContent("Hello", "World", false);
+
+            await CopyAsync(true);
+
+            await AddInvariantContent("Hallo", "Welt", true);
+
+            await TestSearchAsync(null, "Hello", Scope.Draft);
+            await TestSearchAsync(ids1, "Hello", Scope.Published);
 
-            var german1 = await sut.SearchAsync("Stadt", context);
-            var german2 = await sut.SearchAsync("and", context);
+            await TestSearchAsync(ids1, "Hallo", Scope.Draft);
+            await TestSearchAsync(null, "Hallo", Scope.Published);
 
-            var germanStopwordsIds = await sut.SearchAsync("und", context);
+            await CopyAsync(false);
 
-            Assert.Equal(ids1, german1);
-            Assert.Equal(ids1, german2);
-            Assert.Equal(ids2, germanStopwordsIds);
+            await TestSearchAsync(ids1, "Hello", Scope.Draft);
+            await TestSearchAsync(ids1, "Hello", Scope.Published);
 
-            var english1 = await sut.SearchAsync("City", context);
-            var english2 = await sut.SearchAsync("und", context);
+            await TestSearchAsync(null, "Hallo", Scope.Draft);
+            await TestSearchAsync(null, "Hallo", Scope.Published);
 
-            var englishStopwordsIds = await sut.SearchAsync("and", context);
+            await AddInvariantContent("Hallo", "Welt", true);
 
-            Assert.Equal(ids2, english1);
-            Assert.Equal(ids2, english2);
-            Assert.Equal(ids1, englishStopwordsIds);
+            await TestSearchAsync(null, "Hello", Scope.Draft);
+            await TestSearchAsync(ids1, "Hello", Scope.Published);
+
+            await TestSearchAsync(ids1, "Hallo", Scope.Draft);
+            await TestSearchAsync(null, "Hallo", Scope.Published);
         }
 
         [Fact]
-        public async Task Should_throw_exception_for_invalid_query()
+        public async Task Should_also_retrieve_published_content_after_copy()
         {
-            await AddInvariantContent();
+            await AddInvariantContent("Hello", "World", false);
 
-            await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context));
+            await TestSearchAsync(ids1, "Hello", Scope.Draft);
+            await TestSearchAsync(null, "Hello", Scope.Published);
+
+            await CopyAsync(false);
+
+            await TestSearchAsync(ids1, "Hello", Scope.Draft);
+            await TestSearchAsync(ids1, "Hello", Scope.Published);
         }
 
         [Fact]
-        public async Task Should_also_retrieve_published_content_after_copy()
+        public async Task Should_delete_documents_from_index()
         {
-            await AddInvariantContent();
+            await AddInvariantContent("Hello", "World", false);
 
-            context.IsDraft = false;
+            await TestSearchAsync(ids1, "Hello");
+            await TestSearchAsync(ids2, "World");
 
-            var foundHello1 = await sut.SearchAsync("Hello", context);
+            await DeleteAsync(ids1[0]);
 
-            Assert.Empty(foundHello1);
+            await TestSearchAsync(null, "Hello");
+            await TestSearchAsync(ids2, "World");
+        }
 
-            await sut.CopyAsync(ids1[0], true);
-            await sut.FlushAsync();
+        [Fact]
+        public async Task Should_search_by_field()
+        {
+            await AddLocalizedContent();
 
-            var foundHello2 = await sut.SearchAsync("Hello", context);
+            await TestSearchAsync(null, "de:city");
+            await TestSearchAsync(null, "en:Stadt");
+        }
 
-            Assert.Equal(ids1, foundHello2);
+        [Fact]
+        public async Task Should_index_localized_content_and_retrieve()
+        {
+            await AddLocalizedContent();
+
+            await TestSearchAsync(ids1, "Stadt");
+            await TestSearchAsync(ids1, "and");
+            await TestSearchAsync(ids2, "und");
+
+            await TestSearchAsync(ids2, "City");
+            await TestSearchAsync(ids2, "und");
+            await TestSearchAsync(ids1, "and");
         }
 
         private async Task AddLocalizedContent()
@@ -182,29 +211,54 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                         new ContentFieldData()
                             .AddValue("en", "City and Surroundings und sonstiges"));
 
-            await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, true);
-            await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, true);
+            await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true);
+            await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true);
             await sut.FlushAsync();
         }
 
-        private async Task AddInvariantContent()
+        private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false)
         {
             var data1 =
                 new NamedContentData()
                     .AddField("test",
                         new ContentFieldData()
-                            .AddValue("iv", "Hello"));
+                            .AddValue("iv", text1));
 
             var data2 =
                 new NamedContentData()
                     .AddField("test",
                         new ContentFieldData()
-                            .AddValue("iv", "World"));
+                            .AddValue("iv", text2));
 
-            await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, true);
-            await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, true);
+            await sut.IndexAsync(ids1[0], new IndexData { DataDraft = data1 }, onlyDraft);
+            await sut.IndexAsync(ids2[0], new IndexData { DataDraft = data2 }, onlyDraft);
+        }
 
-            await sut.FlushAsync();
+        private async Task DeleteAsync(Guid id)
+        {
+            await sut.DeleteAsync(id);
+        }
+
+        private async Task CopyAsync(bool fromDraft)
+        {
+            await sut.CopyAsync(ids1[0], fromDraft);
+            await sut.CopyAsync(ids2[0], fromDraft);
+        }
+
+        private async Task TestSearchAsync(List<Guid> expected, string text, Scope target = Scope.Draft, TextIndexerGrain grain = null)
+        {
+            context.Scope = target;
+
+            var result = await (grain ?? sut).SearchAsync(text, context);
+
+            if (expected != null)
+            {
+                Assert.Equal(expected, result);
+            }
+            else
+            {
+                Assert.Empty(result);
+            }
         }
     }
 }

tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs

@@ -8,10 +8,8 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Security.Claims;
-using System.Security.Principal;
 using System.Threading.Tasks;
 using FakeItEasy;
-using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Mvc.Abstractions;
@@ -22,7 +20,6 @@ using Squidex.Domain.Apps.Core.Apps;
 using Squidex.Domain.Apps.Entities;
 using Squidex.Domain.Apps.Entities.Apps;
 using Squidex.Infrastructure.Security;
-using Squidex.Shared;
 using Squidex.Shared.Identity;
 using Xunit;
 

tools/Migrate_01/Migrations/BuildFullTextIndices.cs

@@ -39,7 +39,7 @@ namespace Migrate_01.Migrations
 
             var worker = new ActionBlock<ContentState>(state =>
                 {
-                    return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.Data, state.DataDraft);
+                    return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.DataDraft, state.Data);
                 },
                 new ExecutionDataflowBlockOptions
                 {