diff --git a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs index c45d01aa5..67aa7e213 100644 --- a/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs +++ b/src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository_SnapshotStore.cs @@ -75,15 +75,6 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents }); await contents.UpsertAsync(content, oldVersion); - - if (value.IsDeleted) - { - await indexer.DeleteAsync(value.SchemaId.Id, value.Id); - } - else - { - await indexer.IndexAsync(value.SchemaId.Id, value.Id, value.Data, value.DataDraft); - } } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs b/src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs index fd375704b..558a49e8c 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/State/ContentState.cs @@ -86,6 +86,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.State protected void On(ContentStatusChanged @event) { ScheduleJob = null; + Status = @event.Status; if (@event.Status == Status.Published) diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs index e5bcfa60c..697668f76 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs @@ -12,16 +12,30 @@ using System.Threading.Tasks; using Orleans; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Apps; +using Squidex.Domain.Apps.Events.Contents; using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Log; +using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Contents.Text { - public sealed class GrainTextIndexer : ITextIndexer + public sealed class GrainTextIndexer : ITextIndexer, IEventConsumer { + private readonly RetryWindow retryWindow = new RetryWindow(TimeSpan.FromMinutes(5), 5); private readonly IGrainFactory grainFactory; private readonly ISemanticLog log; + public string Name + { + get { return "TextIndexer"; } + } + + public string EventsFilter + { + get { return "^content-"; } + } + public GrainTextIndexer(IGrainFactory grainFactory, ISemanticLog log) { Guard.NotNull(grainFactory, nameof(grainFactory)); @@ -32,52 +46,65 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text this.log = log; } - public async Task DeleteAsync(Guid schemaId, Guid id) + public Task ClearAsync() { - var index = grainFactory.GetGrain(schemaId); - - using (Profiler.TraceMethod()) - { - try - { - await index.DeleteAsync(id); - } - catch (Exception ex) - { - log.LogError(ex, w => w - .WriteProperty("action", "DeleteTextEntry") - .WriteProperty("status", "Failed")); - } - } + return Task.CompletedTask; } - public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft) + public async Task On(Envelope @event) { - var index = grainFactory.GetGrain(schemaId); - - using (Profiler.TraceMethod()) + try { - try + if (@event.Payload is ContentEvent contentEvent) { - if (data != null) - { - await index.IndexAsync(id, new IndexData { Data = data }); - } + var index = grainFactory.GetGrain(contentEvent.SchemaId.Id); + + var id = contentEvent.ContentId; - if (dataDraft != null) + switch (@event.Payload) { - await index.IndexAsync(id, new IndexData { Data = dataDraft, IsDraft = true }); + case ContentDeleted contentDeleted: + await index.DeleteAsync(id); + break; + case ContentCreated contentCreated: + await index.IndexAsync(id, Data(contentCreated.Data), true); + break; + case ContentUpdateProposed contentCreated: + await index.IndexAsync(id, Data(contentCreated.Data), true); + break; + case ContentUpdated contentUpdated: + await index.IndexAsync(id, Data(contentUpdated.Data), false); + break; + case ContentChangesPublished contentChangesPublished: + await index.CopyAsync(id, false); + break; + case ContentChangesDiscarded contentChangesDiscarded: + case ContentStatusChanged contentStatusChanged when contentStatusChanged.Status == Status.Published: + await index.CopyAsync(id, true); + break; } } - catch (Exception ex) + } + catch (Exception ex) + { + if (retryWindow.CanRetryAfterFailure()) { log.LogError(ex, w => w - .WriteProperty("action", "UpdateTextEntry") + .WriteProperty("action", "DeleteTextEntry") .WriteProperty("status", "Failed")); } + else + { + throw; + } } } + private J Data(NamedContentData data) + { + return new IndexData { Data = data }; + } + public async Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false) { if (string.IsNullOrWhiteSpace(queryText)) diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs index b350b8ee9..77270fefc 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs @@ -8,17 +8,12 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; -using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Apps; namespace Squidex.Domain.Apps.Entities.Contents.Text { public interface ITextIndexer { - Task DeleteAsync(Guid schemaId, Guid id); - - Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft); - Task> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false); } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexerGrain.cs index dd1d4c5c8..6f9b71b49 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexerGrain.cs @@ -17,7 +17,9 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { Task DeleteAsync(Guid id); - Task IndexAsync(Guid id, J data); + Task IndexAsync(Guid id, J data, bool onlyDraft); + + Task CopyAsync(Guid id, bool fromDraft); Task> SearchAsync(string queryText, SearchContext context); } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexData.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexData.cs index 7911be4a7..240cc0fda 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexData.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/IndexData.cs @@ -12,7 +12,5 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text public sealed class IndexData { public NamedContentData Data { get; set; } - - public bool IsDraft { get; set; } } } diff --git a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs index 66aa6de17..4fb74d535 100644 --- a/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs +++ b/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs @@ -9,10 +9,8 @@ using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; using System.Threading.Tasks; using Lucene.Net.Analysis; -using Lucene.Net.Documents; using Lucene.Net.Index; using Lucene.Net.Queries; using Lucene.Net.QueryParsers.Classic; @@ -22,7 +20,6 @@ using Lucene.Net.Util; using Squidex.Domain.Apps.Core; using Squidex.Infrastructure; using Squidex.Infrastructure.Assets; -using Squidex.Infrastructure.Json.Objects; using Squidex.Infrastructure.Orleans; namespace Squidex.Domain.Apps.Entities.Contents.Text @@ -34,8 +31,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text private const int MaxUpdates = 100; private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30); private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); - private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, "1")); - private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, "0")); + private static readonly TermsFilter DraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, true.ToString())); + private static readonly TermsFilter NoDraftFilter = new TermsFilter(new Term(TextIndexContent.MetaDraft, false.ToString())); private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); private readonly IAssetStore assetStore; private IDisposable timer; @@ -88,35 +85,22 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return TryFlushAsync(); } - public Task IndexAsync(Guid id, J data) + public Task IndexAsync(Guid id, J data, bool onlyDraft) { var content = new TextIndexContent(indexWriter, indexSearcher, id); - content.Index(data.Value.Data, data.Value.IsDraft); + content.Index(data.Value.Data, onlyDraft); return TryFlushAsync(); } - private static void AppendJsonText(IJsonValue value, Action appendText) + public Task CopyAsync(Guid id, bool fromDraft) { - if (value.Type == JsonValueType.String) - { - appendText(value.ToString()); - } - else if (value is JsonArray array) - { - foreach (var item in array) - { - AppendJsonText(item, appendText); - } - } - else if (value is JsonObject obj) - { - foreach (var item in obj.Values) - { - AppendJsonText(item, appendText); - } - } + var content = new TextIndexContent(indexWriter, indexSearcher, id); + + content.Copy(fromDraft); + + return TryFlushAsync(); } public Task> SearchAsync(string queryText, SearchContext context) diff --git a/src/Squidex/Config/Domain/EntitiesServices.cs b/src/Squidex/Config/Domain/EntitiesServices.cs index b1deb7ae3..d74410de7 100644 --- a/src/Squidex/Config/Domain/EntitiesServices.cs +++ b/src/Squidex/Config/Domain/EntitiesServices.cs @@ -108,7 +108,7 @@ namespace Squidex.Config.Domain .As(); services.AddSingletonAs() - .As(); + .As().As(); services.AddSingletonAs() .As>(); diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs index 35ebe6607..665ea2a4e 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs @@ -13,7 +13,9 @@ using Orleans; using Squidex.Domain.Apps.Core.Apps; using Squidex.Domain.Apps.Core.Contents; using Squidex.Domain.Apps.Entities.Apps; +using Squidex.Domain.Apps.Events.Contents; using Squidex.Infrastructure; +using Squidex.Infrastructure.EventSourcing; using Squidex.Infrastructure.Log; using Squidex.Infrastructure.Orleans; using Xunit; @@ -37,64 +39,78 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text } [Fact] - public async Task Should_call_grain_when_deleting_entry() + public async Task Should_call_grain_when_content_deleted() { - await sut.DeleteAsync(schemaId, contentId); + await sut.On(E(new ContentDeleted())); A.CallTo(() => grain.DeleteAsync(contentId)) .MustHaveHappened(); } [Fact] - public async Task Should_catch_exception_when_deleting_failed() + public async Task Should_call_grain_when_content_created() { - A.CallTo(() => grain.DeleteAsync(contentId)) - .Throws(new InvalidOperationException()); + var data = new NamedContentData(); - await sut.DeleteAsync(schemaId, contentId); + await sut.On(E(new ContentCreated { Data = data })); + + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), true)) + .MustHaveHappened(); } [Fact] - public async Task Should_call_grain_when_indexing_data() + public async Task Should_call_grain_when_content_updated() { var data = new NamedContentData(); - var dataDraft = new NamedContentData(); - await sut.IndexAsync(schemaId, contentId, data, dataDraft); + await sut.On(E(new ContentUpdated { Data = data })); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), false)) .MustHaveHappened(); + } - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) + [Fact] + public async Task Should_call_grain_when_content_change_proposed() + { + var data = new NamedContentData(); + + await sut.On(E(new ContentUpdateProposed { Data = data })); + + A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data), true)) .MustHaveHappened(); } [Fact] - public async Task Should_not_call_grain_when_data_is_null() + public async Task Should_call_grain_when_content_change_published() { - var dataDraft = new NamedContentData(); - - await sut.IndexAsync(schemaId, contentId, null, dataDraft); + var data = new NamedContentData(); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => !x.Value.IsDraft))) - .MustNotHaveHappened(); + await sut.On(E(new ContentChangesPublished())); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) + A.CallTo(() => grain.CopyAsync(contentId, false)) .MustHaveHappened(); } [Fact] - public async Task Should_not_call_grain_when_data_draft_is_null() + public async Task Should_call_grain_when_content_change_discarded() { var data = new NamedContentData(); - await sut.IndexAsync(schemaId, contentId, data, null); + await sut.On(E(new ContentChangesDiscarded())); - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) + A.CallTo(() => grain.CopyAsync(contentId, true)) .MustHaveHappened(); + } - A.CallTo(() => grain.IndexAsync(contentId, A>.That.Matches(x => x.Value.IsDraft))) - .MustNotHaveHappened(); + [Fact] + public async Task Should_call_grain_when_content_published() + { + var data = new NamedContentData(); + + await sut.On(E(new ContentStatusChanged { Status = Status.Published })); + + A.CallTo(() => grain.CopyAsync(contentId, true)) + .MustHaveHappened(); } [Fact] @@ -102,10 +118,27 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { var data = new NamedContentData(); - A.CallTo(() => grain.IndexAsync(contentId, A>.Ignored)) + A.CallTo(() => grain.IndexAsync(contentId, A>.Ignored, false)) .Throws(new InvalidOperationException()); - await sut.IndexAsync(schemaId, contentId, data, null); + await sut.On(E(new ContentCreated { Data = data })); + } + + [Fact] + public async Task Should_not_catch_exception_when_indexing_failed_often() + { + var data = new NamedContentData(); + + A.CallTo(() => grain.IndexAsync(contentId, A>.Ignored, A.Ignored)) + .Throws(new InvalidOperationException()); + + await Assert.ThrowsAsync(async () => + { + for (var i = 0; i < 10; i++) + { + await sut.On(E(new ContentCreated { Data = data })); + } + }); } [Fact] @@ -140,5 +173,13 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text return app; } + + private Envelope E(ContentEvent contentEvent) + { + contentEvent.ContentId = contentId; + contentEvent.SchemaId = NamedId.Of(schemaId, "my-schema"); + + return new Envelope(contentEvent); + } } } diff --git a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs index b44c3bbbe..69f366f41 100644 --- a/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs +++ b/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs @@ -28,7 +28,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text { context = new SearchContext { - Languages = new HashSet { "de", "en" } + Languages = new HashSet { "de", "en" }, + IsDraft = true }; sut = new TextIndexerGrain(assetStore); @@ -148,6 +149,25 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text await Assert.ThrowsAsync(() => sut.SearchAsync("~hello", context)); } + [Fact] + public async Task Should_also_retrieve_published_content_after_copy() + { + await AddLocalizedContent(); + + context.IsDraft = false; + + var foundHello1 = await sut.SearchAsync("Hello", context); + + Assert.Empty(foundHello1); + + await sut.CopyAsync(ids1[0], true); + await sut.FlushAsync(); + + var foundHello2 = await sut.SearchAsync("Hello", context); + + Assert.Equal(ids1, foundHello2); + } + private async Task AddLocalizedContent() { var germanData = @@ -162,8 +182,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text new ContentFieldData() .AddValue("en", "City and Surroundings und sonstiges")); - await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }); - await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }); + await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, false); + await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, false); await sut.FlushAsync(); } @@ -181,8 +201,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text new ContentFieldData() .AddValue("iv", "World")); - await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }); - await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }); + await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, false); + await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, false); await sut.FlushAsync(); } diff --git a/tools/Migrate_01/MigrationPath.cs b/tools/Migrate_01/MigrationPath.cs index 4879a20d4..7c28de24d 100644 --- a/tools/Migrate_01/MigrationPath.cs +++ b/tools/Migrate_01/MigrationPath.cs @@ -101,7 +101,6 @@ namespace Migrate_01 if (version < 15) { yield return serviceProvider.GetService(); - yield return serviceProvider.GetService(); } yield return serviceProvider.GetRequiredService();