tsai
/
squidex
mirror of https://github.com/Squidex/squidex.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Test for improved flushing.

pull/351/head
Sebastian Stehle 8 years ago
parent
15ba7ee3ae
commit
4c6a36725b
11 changed files with 244 additions and 158 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs
  2. 14
      src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs
  3. 2
      src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs
  4. 15
      src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs
  5. 2
      src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs
  6. 120
      src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs
  7. 36
      src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs
  8. 12
      tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs
  9. 194
      tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs
  10. 3
      tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs
  11. 2
      tools/Migrate_01/Migrations/BuildFullTextIndices.cs

2
src/Squidex.Domain.Apps.Entities.MongoDb/Contents/MongoContentRepository.cs
View File

@ -71,7 +71,7 @@ namespace Squidex.Domain.Apps.Entities.MongoDb.Contents
{ {
var useDraft = UseDraft(status); var useDraft = UseDraft(status);
var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft); var fullTextIds = await indexer.SearchAsync(query.FullText, app, schema.Id, useDraft ? Scope.Draft : Scope.Published);
if (fullTextIds?.Count == 0) if (fullTextIds?.Count == 0)
{ {

14
src/Squidex.Domain.Apps.Entities/Contents/Text/GrainTextIndexer.cs
View File

@ -56,7 +56,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
return Task.CompletedTask; return Task.CompletedTask;
} }
public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft) public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData dataDraft, NamedContentData data)
{ {
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId);
@ -64,7 +64,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
try try
{ {
await index.IndexAsync(id, new IndexData { Data = data, DataDraft = dataDraft }, false); await index.IndexAsync(id, new IndexData { DataDraft = dataDraft, Data = data }, false);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -126,10 +126,10 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private J<IndexData> Data(NamedContentData data) private J<IndexData> Data(NamedContentData data)
{ {
return new IndexData { Data = data }; return new IndexData { DataDraft = data };
} }
public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false) public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published)
{ {
if (string.IsNullOrWhiteSpace(queryText)) if (string.IsNullOrWhiteSpace(queryText))
{ {
@ -140,17 +140,17 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
using (Profiler.TraceMethod<GrainTextIndexer>()) using (Profiler.TraceMethod<GrainTextIndexer>())
{ {
var context = CreateContext(app, useDraft); var context = CreateContext(app, scope);
return await index.SearchAsync(queryText, context); return await index.SearchAsync(queryText, context);
} }
} }
private static SearchContext CreateContext(IAppEntity app, bool useDraft) private static SearchContext CreateContext(IAppEntity app, Scope scope)
{ {
var languages = new HashSet<string>(app.LanguagesConfig.Select(x => x.Key)); var languages = new HashSet<string>(app.LanguagesConfig.Select(x => x.Key));
return new SearchContext { Languages = languages, IsDraft = useDraft }; return new SearchContext { Languages = languages, Scope = scope };
} }
} }
} }

2
src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndexer.cs
View File

@ -17,6 +17,6 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
Task IndexAsync(Guid schemaId, Guid contentId, NamedContentData draftData, NamedContentData data); Task IndexAsync(Guid schemaId, Guid contentId, NamedContentData draftData, NamedContentData data);
Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false); Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, Scope scope = Scope.Published);
} }
} }

15
src/Squidex.Domain.Apps.Entities/Contents/Text/Scope.cs
View File

@ -0,0 +1,15 @@
// ==========================================================================
// Squidex Headless CMS
// ==========================================================================
// Copyright (c) Squidex UG (haftungsbeschraenkt)
// All rights reserved. Licensed under the MIT license.
// ==========================================================================
namespace Squidex.Domain.Apps.Entities.Contents.Text
{
public enum Scope
{
Draft,
Published
}
}

2
src/Squidex.Domain.Apps.Entities/Contents/Text/SearchContext.cs
View File

@ -11,7 +11,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
public sealed class SearchContext public sealed class SearchContext
{ {
public bool IsDraft { get; set; } public Scope Scope { get; set; }
public HashSet<string> Languages { get; set; } public HashSet<string> Languages { get; set; }
} }

120
src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexContent.cs
View File

@ -7,6 +7,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using System.Text; using System.Text;
using Lucene.Net.Documents; using Lucene.Net.Documents;
using Lucene.Net.Index; using Lucene.Net.Index;
@ -27,12 +28,14 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private const int ForPublishedIndex = 1; private const int ForPublishedIndex = 1;
private readonly IndexWriter indexWriter; private readonly IndexWriter indexWriter;
private readonly IndexSearcher indexSearcher; private readonly IndexSearcher indexSearcher;
private readonly BinaryDocValues indexValues;
private readonly Guid id; private readonly Guid id;
public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, Guid id) public TextIndexContent(IndexWriter indexWriter, IndexSearcher indexSearcher, BinaryDocValues indexValues, Guid id)
{ {
this.indexWriter = indexWriter; this.indexWriter = indexWriter;
this.indexSearcher = indexSearcher; this.indexSearcher = indexSearcher;
this.indexValues = indexValues;
this.id = id; this.id = id;
} }
@ -42,34 +45,35 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
return MultiDocValues.GetBinaryValues(indexReader, MetaFor); return MultiDocValues.GetBinaryValues(indexReader, MetaFor);
} }
public static bool TryGetId(int docId, bool forDraft, IndexReader reader, BinaryDocValues values, out Guid result) public void Delete()
{ {
result = Guid.Empty; indexWriter.DeleteDocuments(new Term(MetaId, id.ToString()));
}
var forValue = new BytesRef();
values.Get(docId, forValue); public static bool TryGetId(int docId, Scope scope, IndexReader reader, BinaryDocValues values, out Guid result)
{
result = Guid.Empty;
if (forValue.Bytes.Length != 2) if (!TryGet(docId, values, out var @for))
{ {
return false; return false;
} }
if (forDraft && forValue.Bytes[ForDraftIndex] != 1) if (scope == Scope.Draft && @for[ForDraftIndex] != 1)
{ {
return false; return false;
} }
if (!forDraft && forValue.Bytes[ForPublishedIndex] != 1) if (scope == Scope.Published && @for[ForPublishedIndex] != 1)
{ {
return false; return false;
} }
var document = reader.Document(docId); var document = reader.Document(docId);
var id = document.Get(MetaId); var idString = document.Get(MetaId);
if (!Guid.TryParse(id, out result)) if (!Guid.TryParse(idString, out result))
{ {
return false; return false;
} }
@ -77,51 +81,48 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
return true; return true;
} }
public void Index(NamedContentData data, NamedContentData dataDraft, bool onlyDraft) public void Index(NamedContentData dataDraft, NamedContentData data, BinaryDocValues values, bool onlyDraft)
{ {
var converted = CreateDocument(data); var converted = CreateDocument(dataDraft);
Upsert(converted, 1, 1, 0); Upsert(converted, 1, 1, 0);
var existing = GetDocument(1); var docId = GetPublishedDocument();
if (dataDraft != null)
{
converted = CreateDocument(dataDraft);
Upsert(converted, 0, 0, 1); if (data != null)
}
else if (IsForPublished(existing))
{ {
Upsert(converted, 0, 0, 1); Upsert(CreateDocument(data), 0, 0, 1);
} }
else if (existing == null) else
{ {
Upsert(converted, 0, 0, 0); var isPublished = IsForPublished(docId, values);
}
}
private static bool IsForPublished(Document existing)
{
return existing?.GetField(MetaFor)?.GetBinaryValue().Bytes[ForPublishedIndex] == 1;
}
public void Delete() if (!onlyDraft && docId > 0 && isPublished)
{ {
indexWriter.DeleteDocuments(new Term(MetaId, id.ToString())); Upsert(converted, 0, 0, 1);
}
else if (!onlyDraft)
{
Upsert(converted, 0, 0, 0);
}
else
{
Update(0, 0, isPublished ? (byte)1 : (byte)0);
}
}
} }
public void Copy(bool fromDraft) public void Copy(bool fromDraft)
{ {
if (fromDraft) if (fromDraft)
{ {
Update(1, 0, 0); Update(1, 1, 0);
Update(0, 1, 1); Update(0, 0, 1);
} }
else else
{ {
Update(1, 1, 1); Update(1, 0, 0);
Update(0, 0, 0); Update(0, 1, 1);
} }
} }
@ -176,21 +177,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished)); indexWriter.UpdateBinaryDocValue(term, MetaFor, GetValue(forDraft, forPublished));
} }
private Document GetDocument(byte draft) private int GetPublishedDocument()
{ {
if (indexSearcher == null) var docs = indexSearcher?.Search(new TermQuery(new Term(MetaKey, BuildKey(0))), 1);
{
return null;
}
var docs = indexSearcher.Search(new TermQuery(new Term(MetaKey, BuildKey(draft))), 1); return docs?.ScoreDocs.FirstOrDefault()?.Doc ?? 0;
if (docs.ScoreDocs.Length > 0)
{
return indexSearcher.Doc(docs.ScoreDocs[0].Doc);
}
return null;
} }
private void Upsert(Document document, byte draft, byte forDraft, byte forPublished) private void Upsert(Document document, byte draft, byte forDraft, byte forPublished)
@ -209,8 +200,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished)); document.AddBinaryDocValuesField(MetaFor, GetValue(forDraft, forPublished));
indexWriter.DeleteDocuments(new Term(MetaKey, docKey)); // indexWriter.DeleteDocuments(new Term(MetaKey, docKey));
indexWriter.AddDocument(document); indexWriter.UpdateDocument(new Term(MetaKey, docKey), document);
} }
} }
@ -236,6 +227,29 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
} }
} }
private static bool IsForPublished(int docId, BinaryDocValues values)
{
return TryGet(docId, values, out var @for) && @for[1] == 1;
}
private static bool TryGet(int docId, BinaryDocValues values, out byte[] result)
{
var forValue = new BytesRef();
values?.Get(docId, forValue);
if (forValue.Bytes.Length == 2)
{
result = forValue.Bytes;
return true;
}
else
{
result = null;
return false;
}
}
private static BytesRef GetValue(byte forDraft, byte forPublished) private static BytesRef GetValue(byte forDraft, byte forPublished)
{ {
return new BytesRef(new byte[] { forDraft, forPublished }); return new BytesRef(new byte[] { forDraft, forPublished });

36
src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexerGrain.cs
View File

@ -12,7 +12,6 @@ using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using Lucene.Net.Analysis; using Lucene.Net.Analysis;
using Lucene.Net.Index; using Lucene.Net.Index;
using Lucene.Net.Queries;
using Lucene.Net.QueryParsers.Classic; using Lucene.Net.QueryParsers.Classic;
using Lucene.Net.Search; using Lucene.Net.Search;
using Lucene.Net.Store; using Lucene.Net.Store;
@ -43,6 +42,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private QueryParser queryParser; private QueryParser queryParser;
private HashSet<string> currentLanguages; private HashSet<string> currentLanguages;
private long updates; private long updates;
private long updatesNotWritten;
public TextIndexerGrain(IAssetStore assetStore) public TextIndexerGrain(IAssetStore assetStore)
{ {
@ -79,7 +79,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
public Task DeleteAsync(Guid id) public Task DeleteAsync(Guid id)
{ {
var content = new TextIndexContent(indexWriter, indexSearcher, id); var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
content.Delete(); content.Delete();
@ -88,16 +88,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft) public Task IndexAsync(Guid id, J<IndexData> data, bool onlyDraft)
{ {
var content = new TextIndexContent(indexWriter, indexSearcher, id); var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
content.Index(data.Value.Data, data.Value.DataDraft, onlyDraft); content.Index(data.Value.DataDraft, data.Value.Data, indexValues, onlyDraft);
return TryFlushAsync(); return TryFlushAsync();
} }
public Task CopyAsync(Guid id, bool fromDraft) public Task CopyAsync(Guid id, bool fromDraft)
{ {
var content = new TextIndexContent(indexWriter, indexSearcher, id); var content = new TextIndexContent(indexWriter, indexSearcher, indexValues, id);
content.Copy(fromDraft); content.Copy(fromDraft);
@ -118,7 +118,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
foreach (var hit in hits) foreach (var hit in hits)
{ {
if (TextIndexContent.TryGetId(hit.Doc, context.IsDraft, indexReader, indexValues, out var id)) if (TextIndexContent.TryGetId(hit.Doc, context.Scope, indexReader, indexValues, out var id))
{ {
result.Add(id); result.Add(id);
} }
@ -153,18 +153,21 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
private async Task TryFlushAsync() private async Task TryFlushAsync()
{ {
updates++; updates++;
updatesNotWritten++;
if (updates >= MaxUpdates) if (updates >= MaxUpdates)
{ {
await FlushAsync(); await FlushAsync(true);
} }
else else
{ {
await FlushAsync();
timer?.Dispose(); timer?.Dispose();
try try
{ {
timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay); timer = RegisterTimer(_ => FlushAsync(true), null, CommitDelay, CommitDelay);
} }
catch (InvalidOperationException) catch (InvalidOperationException)
{ {
@ -173,7 +176,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
} }
} }
public async Task FlushAsync() public async Task FlushAsync(bool write = false)
{ {
if (updates > 0 && indexWriter != null) if (updates > 0 && indexWriter != null)
{ {
@ -185,6 +188,11 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
indexSearcher = new IndexSearcher(indexReader); indexSearcher = new IndexSearcher(indexReader);
indexValues = TextIndexContent.CreateValues(indexReader); indexValues = TextIndexContent.CreateValues(indexReader);
updates = 0;
}
if (updatesNotWritten > 0 && write)
{
var commit = snapshotter.Snapshot(); var commit = snapshotter.Snapshot();
try try
{ {
@ -195,17 +203,15 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
snapshotter.Release(commit); snapshotter.Release(commit);
} }
updates = 0; updatesNotWritten = 0;
}
else
{
timer?.Dispose();
} }
timer?.Dispose();
} }
public async Task DeactivateAsync(bool deleteFolder = false) public async Task DeactivateAsync(bool deleteFolder = false)
{ {
await TryFlushAsync(); await FlushAsync(true);
indexWriter?.Dispose(); indexWriter?.Dispose();
indexWriter = null; indexWriter = null;

12
tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/GrainTextIndexerTests.cs
View File

@ -44,7 +44,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
var data = new NamedContentData(); var data = new NamedContentData();
var dataDraft = new NamedContentData(); var dataDraft = new NamedContentData();
await sut.IndexAsync(schemaId, contentId, data, dataDraft); await sut.IndexAsync(schemaId, contentId, dataDraft, data);
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && x.Value.DataDraft == dataDraft), false)) A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && x.Value.DataDraft == dataDraft), false))
.MustHaveHappened(); .MustHaveHappened();
@ -66,7 +66,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
await sut.On(E(new ContentCreated { Data = data })); await sut.On(E(new ContentCreated { Data = data }));
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true)) A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), true))
.MustHaveHappened(); .MustHaveHappened();
} }
@ -77,7 +77,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
await sut.On(E(new ContentUpdated { Data = data })); await sut.On(E(new ContentUpdated { Data = data }));
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), false)) A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), false))
.MustHaveHappened(); .MustHaveHappened();
} }
@ -88,7 +88,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
await sut.On(E(new ContentUpdateProposed { Data = data })); await sut.On(E(new ContentUpdateProposed { Data = data }));
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data), true)) A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.DataDraft == data), true))
.MustHaveHappened(); .MustHaveHappened();
} }
@ -161,7 +161,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
A.CallTo(() => grain.SearchAsync("Search", A<SearchContext>.Ignored)) A.CallTo(() => grain.SearchAsync("Search", A<SearchContext>.Ignored))
.Returns(foundIds); .Returns(foundIds);
var ids = await sut.SearchAsync("Search", GetApp(), schemaId, true); var ids = await sut.SearchAsync("Search", GetApp(), schemaId, Scope.Draft);
Assert.Equal(foundIds, ids); Assert.Equal(foundIds, ids);
} }
@ -169,7 +169,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
[Fact] [Fact]
public async Task Should_not_call_grain_when_input_is_empty() public async Task Should_not_call_grain_when_input_is_empty()
{ {
var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, false); var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, Scope.Published);
Assert.Null(ids); Assert.Null(ids);

194
tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerGrainTests.cs
View File

@ -28,8 +28,7 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
context = new SearchContext context = new SearchContext
{ {
Languages = new HashSet<string> { "de", "en" }, Languages = new HashSet<string> { "de", "en" }
IsDraft = true
}; };
sut = new TextIndexerGrain(assetStore); sut = new TextIndexerGrain(assetStore);
@ -41,10 +40,16 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
sut.OnDeactivateAsync().Wait(); sut.OnDeactivateAsync().Wait();
} }
[Fact]
public async Task Should_throw_exception_for_invalid_query()
{
await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context));
}
[Fact] [Fact]
public async Task Should_read_index_and_retrieve() public async Task Should_read_index_and_retrieve()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
await sut.DeactivateAsync(); await sut.DeactivateAsync();
@ -53,11 +58,8 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
{ {
await other.ActivateAsync(schemaId); await other.ActivateAsync(schemaId);
var foundHello = await other.SearchAsync("Hello", context); await TestSearchAsync(ids1, "Hello", grain: other);
var foundWorld = await other.SearchAsync("World", context); await TestSearchAsync(ids2, "World", grain: other);
Assert.Equal(ids1, foundHello);
Assert.Equal(ids2, foundWorld);
} }
finally finally
{ {
@ -68,104 +70,131 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
[Fact] [Fact]
public async Task Should_index_invariant_content_and_retrieve() public async Task Should_index_invariant_content_and_retrieve()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
var foundHello = await sut.SearchAsync("Hello", context);
var foundWorld = await sut.SearchAsync("World", context);
Assert.Equal(ids1, foundHello); await TestSearchAsync(ids1, "Hello");
Assert.Equal(ids2, foundWorld); await TestSearchAsync(ids2, "World");
} }
[Fact] [Fact]
public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() public async Task Should_index_invariant_content_and_retrieve_with_fuzzy()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
var foundHello = await sut.SearchAsync("helo~", context); await TestSearchAsync(ids1, "helo~");
var foundWorld = await sut.SearchAsync("wold~", context); await TestSearchAsync(ids2, "wold~");
Assert.Equal(ids1, foundHello);
Assert.Equal(ids2, foundWorld);
} }
[Fact] [Fact]
public async Task Should_delete_documents_from_index() public async Task Should_update_draft_only()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
await AddInvariantContent("Hallo", "Welt", false);
await sut.DeleteAsync(ids1[0]);
await sut.FlushAsync();
var helloIds = await sut.SearchAsync("Hello", context); await TestSearchAsync(null, "Hello", Scope.Draft);
await TestSearchAsync(null, "Hello", Scope.Published);
var worldIds = await sut.SearchAsync("World", context); await TestSearchAsync(ids1, "Hallo", Scope.Draft);
await TestSearchAsync(null, "Hallo", Scope.Published);
Assert.Empty(helloIds);
Assert.Equal(ids2, worldIds);
} }
[Fact] [Fact]
public async Task Should_search_by_field() public async Task Should_also_update_published_after_copy()
{ {
await AddLocalizedContent(); await AddInvariantContent("Hello", "World", false);
var emptyGerman = await sut.SearchAsync("de:City", context); await CopyAsync(true);
var emptyEnglish = await sut.SearchAsync("en:Stadt", context);
Assert.Empty(emptyGerman); await AddInvariantContent("Hallo", "Welt", false);
Assert.Empty(emptyEnglish);
await TestSearchAsync(null, "Hello", Scope.Draft);
await TestSearchAsync(null, "Hello", Scope.Published);
await TestSearchAsync(ids1, "Hallo", Scope.Draft);
await TestSearchAsync(ids1, "Hallo", Scope.Published);
} }
[Fact] [Fact]
public async Task Should_index_localized_content_and_retrieve() public async Task Should_simulate_content_reversion()
{ {
await AddLocalizedContent(); await AddInvariantContent("Hello", "World", false);
await CopyAsync(true);
await AddInvariantContent("Hallo", "Welt", true);
await TestSearchAsync(null, "Hello", Scope.Draft);
await TestSearchAsync(ids1, "Hello", Scope.Published);
var german1 = await sut.SearchAsync("Stadt", context); await TestSearchAsync(ids1, "Hallo", Scope.Draft);
var german2 = await sut.SearchAsync("and", context); await TestSearchAsync(null, "Hallo", Scope.Published);
var germanStopwordsIds = await sut.SearchAsync("und", context); await CopyAsync(false);
Assert.Equal(ids1, german1); await TestSearchAsync(ids1, "Hello", Scope.Draft);
Assert.Equal(ids1, german2); await TestSearchAsync(ids1, "Hello", Scope.Published);
Assert.Equal(ids2, germanStopwordsIds);
var english1 = await sut.SearchAsync("City", context); await TestSearchAsync(null, "Hallo", Scope.Draft);
var english2 = await sut.SearchAsync("und", context); await TestSearchAsync(null, "Hallo", Scope.Published);
var englishStopwordsIds = await sut.SearchAsync("and", context); await AddInvariantContent("Hallo", "Welt", true);
Assert.Equal(ids2, english1); await TestSearchAsync(null, "Hello", Scope.Draft);
Assert.Equal(ids2, english2); await TestSearchAsync(ids1, "Hello", Scope.Published);
Assert.Equal(ids1, englishStopwordsIds);
await TestSearchAsync(ids1, "Hallo", Scope.Draft);
await TestSearchAsync(null, "Hallo", Scope.Published);
} }
[Fact] [Fact]
public async Task Should_throw_exception_for_invalid_query() public async Task Should_also_retrieve_published_content_after_copy()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context)); await TestSearchAsync(ids1, "Hello", Scope.Draft);
await TestSearchAsync(null, "Hello", Scope.Published);
await CopyAsync(false);
await TestSearchAsync(ids1, "Hello", Scope.Draft);
await TestSearchAsync(ids1, "Hello", Scope.Published);
} }
[Fact] [Fact]
public async Task Should_also_retrieve_published_content_after_copy() public async Task Should_delete_documents_from_index()
{ {
await AddInvariantContent(); await AddInvariantContent("Hello", "World", false);
context.IsDraft = false; await TestSearchAsync(ids1, "Hello");
await TestSearchAsync(ids2, "World");
var foundHello1 = await sut.SearchAsync("Hello", context); await DeleteAsync(ids1[0]);
Assert.Empty(foundHello1); await TestSearchAsync(null, "Hello");
await TestSearchAsync(ids2, "World");
}
await sut.CopyAsync(ids1[0], true); [Fact]
await sut.FlushAsync(); public async Task Should_search_by_field()
{
await AddLocalizedContent();
var foundHello2 = await sut.SearchAsync("Hello", context); await TestSearchAsync(null, "de:city");
await TestSearchAsync(null, "en:Stadt");
}
Assert.Equal(ids1, foundHello2); [Fact]
public async Task Should_index_localized_content_and_retrieve()
{
await AddLocalizedContent();
await TestSearchAsync(ids1, "Stadt");
await TestSearchAsync(ids1, "and");
await TestSearchAsync(ids2, "und");
await TestSearchAsync(ids2, "City");
await TestSearchAsync(ids2, "und");
await TestSearchAsync(ids1, "and");
} }
private async Task AddLocalizedContent() private async Task AddLocalizedContent()
@ -182,29 +211,54 @@ namespace Squidex.Domain.Apps.Entities.Contents.Text
new ContentFieldData() new ContentFieldData()
.AddValue("en", "City and Surroundings und sonstiges")); .AddValue("en", "City and Surroundings und sonstiges"));
await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }, true); await sut.IndexAsync(ids1[0], new IndexData { DataDraft = germanData }, true);
await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }, true); await sut.IndexAsync(ids2[0], new IndexData { DataDraft = englishData }, true);
await sut.FlushAsync(); await sut.FlushAsync();
} }
private async Task AddInvariantContent() private async Task AddInvariantContent(string text1, string text2, bool onlyDraft = false)
{ {
var data1 = var data1 =
new NamedContentData() new NamedContentData()
.AddField("test", .AddField("test",
new ContentFieldData() new ContentFieldData()
.AddValue("iv", "Hello")); .AddValue("iv", text1));
var data2 = var data2 =
new NamedContentData() new NamedContentData()
.AddField("test", .AddField("test",
new ContentFieldData() new ContentFieldData()
.AddValue("iv", "World")); .AddValue("iv", text2));
await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }, true); await sut.IndexAsync(ids1[0], new IndexData { DataDraft = data1 }, onlyDraft);
await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }, true); await sut.IndexAsync(ids2[0], new IndexData { DataDraft = data2 }, onlyDraft);
}
await sut.FlushAsync(); private async Task DeleteAsync(Guid id)
{
await sut.DeleteAsync(id);
}
private async Task CopyAsync(bool fromDraft)
{
await sut.CopyAsync(ids1[0], fromDraft);
await sut.CopyAsync(ids2[0], fromDraft);
}
private async Task TestSearchAsync(List<Guid> expected, string text, Scope target = Scope.Draft, TextIndexerGrain grain = null)
{
context.Scope = target;
var result = await (grain ?? sut).SearchAsync(text, context);
if (expected != null)
{
Assert.Equal(expected, result);
}
else
{
Assert.Empty(result);
}
} }
} }
} }

3
tests/Squidex.Web.Tests/Pipeline/AppResolverTests.cs
View File

@ -8,10 +8,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Security.Claims; using System.Security.Claims;
using System.Security.Principal;
using System.Threading.Tasks; using System.Threading.Tasks;
using FakeItEasy; using FakeItEasy;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.Mvc.Abstractions; using Microsoft.AspNetCore.Mvc.Abstractions;
@ -22,7 +20,6 @@ using Squidex.Domain.Apps.Core.Apps;
using Squidex.Domain.Apps.Entities; using Squidex.Domain.Apps.Entities;
using Squidex.Domain.Apps.Entities.Apps; using Squidex.Domain.Apps.Entities.Apps;
using Squidex.Infrastructure.Security; using Squidex.Infrastructure.Security;
using Squidex.Shared;
using Squidex.Shared.Identity; using Squidex.Shared.Identity;
using Xunit; using Xunit;

2
tools/Migrate_01/Migrations/BuildFullTextIndices.cs
View File

@ -39,7 +39,7 @@ namespace Migrate_01.Migrations
var worker = new ActionBlock<ContentState>(state => var worker = new ActionBlock<ContentState>(state =>
{ {
return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.Data, state.DataDraft); return textIndexer.IndexAsync(state.SchemaId.Id, state.Id, state.DataDraft, state.Data);
}, },
new ExecutionDataflowBlockOptions new ExecutionDataflowBlockOptions
{ {

Write Preview
Loading…
Cancel
Save