Test

src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs

@@ -75,15 +75,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
                 });
 
                 await contents.UpsertAsync(content, oldVersion);
-
-                if (value.IsDeleted)
-                {
-                    await indexer.DeleteAsync(value.SchemaId.Id, value.Id);
-                }
-                else
-                {
-                    await indexer.IndexAsync(value.SchemaId.Id, value.Id, value.Data, value.DataDraft);
-                }
             }
         }
 

src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs

@@ -86,6 +86,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.State
         protected void On(ContentStatusChanged @event)
         {
             ScheduleJob = null;
+
             Status = @event.Status;
 
             if (@event.Status == Status.Published)

src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs

@@ -12,16 +12,30 @@ using System.Threading.Tasks;
 using Orleans;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Apps;
+using Squidex.Domain.Apps.Events.Contents;
 using Squidex.Infrastructure;
+using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.Log;
+using Squidex.Infrastructure.Orleans;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
-    public sealed class GrainTextIndexer : ITextIndexer
+    public sealed class GrainTextIndexer : ITextIndexer, IEventConsumer
     {
+        private readonly RetryWindow retryWindow = new RetryWindow(TimeSpan.FromMinutes(5), 5);
         private readonly IGrainFactory grainFactory;
         private readonly ISemanticLog log;
 
+        public string Name
+        {
+            get { return "TextIndexer"; }
+        }
+
+        public string EventsFilter
+        {
+            get { return "^content-"; }
+        }
+
         public GrainTextIndexer(IGrainFactory grainFactory, ISemanticLog log)
         {
             Guard.NotNull(grainFactory, nameof(grainFactory));
@@ -32,52 +46,65 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             this.log = log;
         }
 
-        public async Task DeleteAsync(Guid schemaId, Guid id)
+        public Task ClearAsync()
         {
-            var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId);
+            return Task.CompletedTask;
-
-            using (Profiler.TraceMethod<GrainTextIndexer>())
-            {
-                try
-                {
-                    await index.DeleteAsync(id);
-                }
-                catch (Exception ex)
-                {
-                    log.LogError(ex, w => w
-                        .WriteProperty("action", "DeleteTextEntry")
-                        .WriteProperty("status", "Failed"));
-                }
-            }
         }
 
-        public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft)
+        public async Task On(Envelope<IEvent> @event)
         {
-            var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId);
+            try
-
-            using (Profiler.TraceMethod<GrainTextIndexer>())
             {
-                try
+                if (@event.Payload is ContentEvent contentEvent)
                 {
-                    if (data != null)
+                    var index = grainFactory.GetGrain<ITextIndexerGrain>(contentEvent.SchemaId.Id);
-                    {
+
-                        await index.IndexAsync(id, new IndexData { Data = data });
+                    var id = contentEvent.ContentId;
-                    }
 
-                    if (dataDraft != null)
+                    switch (@event.Payload)
                     {
-                        await index.IndexAsync(id, new IndexData { Data = dataDraft, IsDraft = true });
+                        case ContentDeleted contentDeleted:
+                            await index.DeleteAsync(id);
+                            break;
+                        case ContentCreated contentCreated:
+                            await index.IndexAsync(id, Data(contentCreated.Data), true);
+                            break;
+                        case ContentUpdateProposed contentCreated:
+                            await index.IndexAsync(id, Data(contentCreated.Data), true);
+                            break;
+                        case ContentUpdated contentUpdated:
+                            await index.IndexAsync(id, Data(contentUpdated.Data), false);
+                            break;
+                        case ContentChangesPublished contentChangesPublished:
+                            await index.CopyAsync(id, false);
+                            break;
+                        case ContentChangesDiscarded contentChangesDiscarded:
+                        case ContentStatusChanged contentStatusChanged when contentStatusChanged.Status == Status.Published:
+                            await index.CopyAsync(id, true);
+                            break;
                     }
                 }
-                catch (Exception ex)
+            }
+            catch (Exception ex)
+            {
+                if (retryWindow.CanRetryAfterFailure())
                 {
                     log.LogError(ex, w => w
-                        .WriteProperty("action", "UpdateTextEntry")
+                        .WriteProperty("action", "DeleteTextEntry")
                         .WriteProperty("status", "Failed"));
                 }
+                else
+                {
+                    throw;
+                }
             }
         }
 
+        private J<IndexData> Data(NamedContentData data)
+        {
+            return new IndexData { Data = data };
+        }
+
         public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false)
         {
             if (string.IsNullOrWhiteSpace(queryText))

src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs

@@ -8,17 +8,12 @@
 using System;
 using System.Collections.Generic;
 using System.Threading.Tasks;
-using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Apps;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
 {
     public interface ITextIndexer
     {
-        Task DeleteAsync(Guid schemaId, Guid id);
-
-        Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft);
-
         Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false);
     }
 }

src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexerGrain.cs

@@ -17,7 +17,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     {
         Task DeleteAsync(Guid id);
 
-        Task IndexAsync(Guid id, J<IndexData> data);
+        Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft);
+
+        Task CopyAsync(Guid id, bool fromDraft);
 
         Task<List<Guid>> SearchAsync(string queryText, SearchContext context);
     }

src/Squidex.Domain.Apps.Entities/Contents/Text/IndexData.cs

@@ -12,7 +12,5 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
     public sealed class IndexData
     {
         public NamedContentData Data { get; set; }
-
-        public bool IsDraft { get; set; }
     }
 }

src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs

@@ -9,10 +9,8 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading.Tasks;
 using Lucene.Net.Analysis;
-using Lucene.Net.Documents;
 using Lucene.Net.Index;
 using Lucene.Net.Queries;
 using Lucene.Net.QueryParsers.Classic;
@@ -22,7 +20,6 @@ using Lucene.Net.Util;
 using Squidex.Domain.Apps.Core;
 using Squidex.Infrastructure;
 using Squidex.Infrastructure.Assets;
-using Squidex.Infrastructure.Json.Objects;
 using Squidex.Infrastructure.Orleans;
 
 namespace Squidex.Domain.Apps.Entities.Contents.Text
@@ -34,8 +31,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         private const int MaxUpdates = 100;
         private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30);
         private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version);
-        private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, "1"));
+        private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, true.ToString()));
-        private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, "0"));
+        private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, false.ToString()));
         private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
         private readonly IAssetStore assetStore;
         private IDisposable timer;
@@ -88,35 +85,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             return TryFlushAsync();
         }
 
-        public Task IndexAsync(Guid id, J<IndexData> data)
+        public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft)
         {
             var content = new TextIndexContent(indexWriter, indexSearcher, id);
 
-            content.Index(data.Value.Data, data.Value.IsDraft);
+            content.Index(data.Value.Data, onlyDraft);
 
             return TryFlushAsync();
         }
 
-        private static void AppendJsonText(IJsonValue value, Action<string> appendText)
+        public Task CopyAsync(Guid id, bool fromDraft)
         {
-            if (value.Type == JsonValueType.String)
+            var content = new TextIndexContent(indexWriter, indexSearcher, id);
-            {
+
-                appendText(value.ToString());
+            content.Copy(fromDraft);
-            }
+
-            else if (value is JsonArray array)
+            return TryFlushAsync();
-            {
-                foreach (var item in array)
-                {
-                    AppendJsonText(item, appendText);
-                }
-            }
-            else if (value is JsonObject obj)
-            {
-                foreach (var item in obj.Values)
-                {
-                    AppendJsonText(item, appendText);
-                }
-            }
         }
 
         public Task<List<Guid>> SearchAsync(string queryText, SearchContext context)

src/Squidex/Config/Domain/EntitiesServices.cs

@@ -108,7 +108,7 @@ namespace Squidex.Config.Domain
                 .As<ITagService>();
 
             services.AddSingletonAs<GrainTextIndexer>()
-                .As<ITextIndexer>();
+                .As<ITextIndexer>().As<IEventConsumer>();
 
             services.AddSingletonAs<FileTypeTagGenerator>()
                 .As<ITagGenerator<CreateAsset>>();

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs

@@ -13,7 +13,9 @@ using Orleans;
 using Squidex.Domain.Apps.Core.Apps;
 using Squidex.Domain.Apps.Core.Contents;
 using Squidex.Domain.Apps.Entities.Apps;
+using Squidex.Domain.Apps.Events.Contents;
 using Squidex.Infrastructure;
+using Squidex.Infrastructure.EventSourcing;
 using Squidex.Infrastructure.Log;
 using Squidex.Infrastructure.Orleans;
 using Xunit;
@@ -37,64 +39,78 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         }
 
         [Fact]
-        public async Task Should_call_grain_when_deleting_entry()
+        public async Task Should_call_grain_when_content_deleted()
         {
-            await sut.DeleteAsync(schemaId, contentId);
+            await sut.On(E(new ContentDeleted()));
 
             A.CallTo(() => grain.DeleteAsync(contentId))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_catch_exception_when_deleting_failed()
+        public async Task Should_call_grain_when_content_created()
         {
-            A.CallTo(() => grain.DeleteAsync(contentId))
+            var data = new NamedContentData();
-                .Throws(new InvalidOperationException());
 
-            await sut.DeleteAsync(schemaId, contentId);
+            await sut.On(E(new ContentCreated { Data = data }));
+
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true))
+                .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_call_grain_when_indexing_data()
+        public async Task Should_call_grain_when_content_updated()
         {
             var data = new NamedContentData();
-            var dataDraft = new NamedContentData();
 
-            await sut.IndexAsync(schemaId, contentId, data, dataDraft);
+            await sut.On(E(new ContentUpdated { Data = data }));
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft)))
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), false))
                 .MustHaveHappened();
+        }
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft)))
+        [Fact]
+        public async Task Should_call_grain_when_content_change_proposed()
+        {
+            var data = new NamedContentData();
+
+            await sut.On(E(new ContentUpdateProposed { Data = data }));
+
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_not_call_grain_when_data_is_null()
+        public async Task Should_call_grain_when_content_change_published()
         {
-            var dataDraft = new NamedContentData();
+            var data = new NamedContentData();
-
-            await sut.IndexAsync(schemaId, contentId, null, dataDraft);
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => !x.Value.IsDraft)))
+            await sut.On(E(new ContentChangesPublished()));
-                .MustNotHaveHappened();
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft)))
+            A.CallTo(() => grain.CopyAsync(contentId, false))
                 .MustHaveHappened();
         }
 
         [Fact]
-        public async Task Should_not_call_grain_when_data_draft_is_null()
+        public async Task Should_call_grain_when_content_change_discarded()
         {
             var data = new NamedContentData();
 
-            await sut.IndexAsync(schemaId, contentId, data, null);
+            await sut.On(E(new ContentChangesDiscarded()));
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft)))
+            A.CallTo(() => grain.CopyAsync(contentId, true))
                 .MustHaveHappened();
+        }
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.IsDraft)))
+        [Fact]
-                .MustNotHaveHappened();
+        public async Task Should_call_grain_when_content_published()
+        {
+            var data = new NamedContentData();
+
+            await sut.On(E(new ContentStatusChanged { Status = Status.Published }));
+
+            A.CallTo(() => grain.CopyAsync(contentId, true))
+                .MustHaveHappened();
         }
 
         [Fact]
@@ -102,10 +118,27 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             var data = new NamedContentData();
 
-            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.Ignored))
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.Ignored, false))
                 .Throws(new InvalidOperationException());
 
-            await sut.IndexAsync(schemaId, contentId, data, null);
+            await sut.On(E(new ContentCreated { Data = data }));
+        }
+
+        [Fact]
+        public async Task Should_not_catch_exception_when_indexing_failed_often()
+        {
+            var data = new NamedContentData();
+
+            A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.Ignored, A<bool>.Ignored))
+                .Throws(new InvalidOperationException());
+
+            await Assert.ThrowsAsync<InvalidOperationException>(async () =>
+            {
+                for (var i = 0; i < 10; i++)
+                {
+                    await sut.On(E(new ContentCreated { Data = data }));
+                }
+            });
         }
 
         [Fact]
@@ -140,5 +173,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
 
             return app;
         }
+
+        private Envelope<IEvent> E(ContentEvent contentEvent)
+        {
+            contentEvent.ContentId = contentId;
+            contentEvent.SchemaId = NamedId.Of(schemaId, "my-schema");
+
+            return new Envelope<IEvent>(contentEvent);
+        }
     }
 }

tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs

@@ -28,7 +28,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
         {
             context = new SearchContext
             {
-                Languages = new HashSet<string> { "de", "en" }
+                Languages = new HashSet<string> { "de", "en" },
+                IsDraft = true
             };
 
             sut = new TextIndexerGrain(assetStore);
@@ -148,6 +149,25 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
             await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context));
         }
 
+        [Fact]
+        public async Task Should_also_retrieve_published_content_after_copy()
+        {
+            await AddLocalizedContent();
+
+            context.IsDraft = false;
+
+            var foundHello1 = await sut.SearchAsync("Hello", context);
+
+            Assert.Empty(foundHello1);
+
+            await sut.CopyAsync(ids1[0], true);
+            await sut.FlushAsync();
+
+            var foundHello2 = await sut.SearchAsync("Hello", context);
+
+            Assert.Equal(ids1, foundHello2);
+        }
+
         private async Task AddLocalizedContent()
         {
             var germanData =
@@ -162,8 +182,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                         new ContentFieldData()
                             .AddValue("en", "City and Surroundings und sonstiges"));
 
-            await sut.IndexAsync(ids1[0], new IndexData { Data = germanData });
+            await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, false);
-            await sut.IndexAsync(ids2[0], new IndexData { Data = englishData });
+            await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, false);
             await sut.FlushAsync();
         }
 
@@ -181,8 +201,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
                         new ContentFieldData()
                             .AddValue("iv", "World"));
 
-            await sut.IndexAsync(ids1[0], new IndexData { Data = data1 });
+            await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, false);
-            await sut.IndexAsync(ids2[0], new IndexData { Data = data2 });
+            await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, false);
 
             await sut.FlushAsync();
         }

tools/Migrate_01/MigrationPath.cs

@@ -101,7 +101,6 @@ namespace Migrate_01
             if (version < 15)
             {
                 yield return serviceProvider.GetService<RestructureContentCollection>();
-                yield return serviceProvider.GetService<BuildFullTextIndices>();
             }
 
             yield return serviceProvider.GetRequiredService<StartEventConsumers>();