mirror of https://github.com/Squidex/squidex.git
Sebastian Stehle
7 years ago
19 changed files with 703 additions and 32 deletions
@ -0,0 +1,65 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Analysis.Standard; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class MultiLanguageAnalyzer : AnalyzerWrapper |
|||
{ |
|||
private readonly StandardAnalyzer fallbackAnalyzer; |
|||
private readonly Dictionary<string, Analyzer> analyzers = new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase); |
|||
|
|||
public MultiLanguageAnalyzer(LuceneVersion version) |
|||
: base(PER_FIELD_REUSE_STRATEGY) |
|||
{ |
|||
fallbackAnalyzer = new StandardAnalyzer(version); |
|||
|
|||
foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) |
|||
{ |
|||
if (typeof(Analyzer).IsAssignableFrom(type)) |
|||
{ |
|||
var language = type.Namespace.Split('.').Last(); |
|||
|
|||
if (language.Length == 2) |
|||
{ |
|||
try |
|||
{ |
|||
var analyzer = Activator.CreateInstance(type, version); |
|||
|
|||
analyzers[language] = (Analyzer)analyzer; |
|||
} |
|||
catch (MissingMethodException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected override Analyzer GetWrappedAnalyzer(string fieldName) |
|||
{ |
|||
if (fieldName.Length > 0) |
|||
{ |
|||
var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; |
|||
|
|||
return analyzer; |
|||
} |
|||
else |
|||
{ |
|||
return fallbackAnalyzer; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,90 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using System.IO.Compression; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Infrastructure.Assets; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public static class PersistenceHelper |
|||
{ |
|||
private const string ArchiveFile = "Archive.zip"; |
|||
private const string LockFile = "write.lock"; |
|||
|
|||
public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory) |
|||
{ |
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) |
|||
{ |
|||
foreach (var file in directory.GetFiles()) |
|||
{ |
|||
try |
|||
{ |
|||
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && !file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
zipArchive.CreateEntryFromFile(file.FullName, file.Name); |
|||
} |
|||
} |
|||
catch (IOException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
await assetStore.UploadAsync(directory.Name, 0, string.Empty, fileStream, true); |
|||
} |
|||
} |
|||
|
|||
public static async Task DownloadAsync(this IAssetStore assetStore, DirectoryInfo directory) |
|||
{ |
|||
if (directory.Exists) |
|||
{ |
|||
directory.Delete(true); |
|||
} |
|||
|
|||
directory.Create(); |
|||
|
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
try |
|||
{ |
|||
await assetStore.DownloadAsync(directory.Name, 0, string.Empty, fileStream); |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) |
|||
{ |
|||
zipArchive.ExtractToDirectory(directory.FullName); |
|||
} |
|||
} |
|||
catch (AssetNotFoundException) |
|||
{ |
|||
return; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,47 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Events.Contents; |
|||
using Squidex.Infrastructure.EventSourcing; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class TextIndexer : IEventConsumer |
|||
{ |
|||
public string Name |
|||
{ |
|||
get { return GetType().Name; } |
|||
} |
|||
|
|||
public string EventsFilter |
|||
{ |
|||
get { return "^content-"; } |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return TaskHelper.Done; |
|||
} |
|||
|
|||
public Task On(Envelope<IEvent> @event) |
|||
{ |
|||
switch (@event.Payload) |
|||
{ |
|||
case ContentCreated contentCreated: |
|||
break; |
|||
case ContentUpdated contentUpdated: |
|||
break; |
|||
case ContentUpdateProposed contentUpdateProposed: |
|||
break; |
|||
} |
|||
|
|||
return Task.CompletedTask; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,268 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.IO; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Analysis.Standard; |
|||
using Lucene.Net.Documents; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.QueryParsers.Classic; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Domain.Apps.Core; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Core.Schemas; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Assets; |
|||
using Squidex.Infrastructure.Json.Objects; |
|||
using Squidex.Infrastructure.Orleans; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class TextIndexerGrain : GrainOfGuid |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private const int MaxResults = 2000; |
|||
private const int MaxUpdates = 100; |
|||
private static readonly HashSet<string> IdFields = new HashSet<string>(); |
|||
private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly IAssetStore assetStore; |
|||
private DirectoryInfo directory; |
|||
private IndexWriter indexWriter; |
|||
private IndexReader indexReader; |
|||
private QueryParser queryParser; |
|||
private int currentAppVersion; |
|||
private int currentSchemaVersion; |
|||
private int updates; |
|||
|
|||
public TextIndexerGrain(IAssetStore assetStore) |
|||
{ |
|||
Guard.NotNull(assetStore, nameof(assetStore)); |
|||
|
|||
this.assetStore = assetStore; |
|||
} |
|||
|
|||
public override Task OnActivateAsync() |
|||
{ |
|||
RegisterTimer(_ => FlushAsync(), null, TimeSpan.Zero, TimeSpan.FromMinutes(10)); |
|||
|
|||
return base.OnActivateAsync(); |
|||
} |
|||
|
|||
public override async Task OnDeactivateAsync() |
|||
{ |
|||
await DeactivateAsync(true); |
|||
} |
|||
|
|||
protected override async Task OnActivateAsync(Guid key) |
|||
{ |
|||
directory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), $"Index_{key}")); |
|||
|
|||
await assetStore.DownloadAsync(directory); |
|||
|
|||
indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer)); |
|||
indexReader = indexWriter.GetReader(true); |
|||
} |
|||
|
|||
public Task DeleteContentAsync(Guid id) |
|||
{ |
|||
indexWriter.DeleteDocuments(new Term("id", id.ToString())); |
|||
|
|||
return TryFlushAsync(); |
|||
} |
|||
|
|||
public Task AddContentAsync(Guid id, NamedContentData data, bool isUpdate, bool isDraft) |
|||
{ |
|||
var idString = id.ToString(); |
|||
|
|||
if (isUpdate) |
|||
{ |
|||
indexWriter.DeleteDocuments(new Term("id", idString)); |
|||
} |
|||
|
|||
var document = new Document(); |
|||
|
|||
document.AddStringField("id", idString, Field.Store.YES); |
|||
document.AddInt32Field("draft", isDraft ? 1 : 0, Field.Store.YES); |
|||
|
|||
foreach (var field in data) |
|||
{ |
|||
foreach (var fieldValue in field.Value) |
|||
{ |
|||
var value = fieldValue.Value; |
|||
|
|||
if (value.Type == JsonValueType.String) |
|||
{ |
|||
var fieldName = BuildFieldName(fieldValue.Key, field.Key); |
|||
|
|||
document.AddTextField(fieldName, fieldValue.Value.ToString(), Field.Store.YES); |
|||
} |
|||
else if (value.Type == JsonValueType.Object) |
|||
{ |
|||
foreach (var property in (JsonObject)value) |
|||
{ |
|||
if (property.Value.Type == JsonValueType.String) |
|||
{ |
|||
var fieldName = BuildFieldName(fieldValue.Key, field.Key, property.Key); |
|||
|
|||
document.AddTextField(fieldName, property.Value.ToString(), Field.Store.YES); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
indexWriter.AddDocument(document); |
|||
|
|||
return TryFlushAsync(); |
|||
} |
|||
|
|||
public Task<List<Guid>> SearchAsync(string term, int appVersion, int schemaVersion, J<Schema> schema, List<string> languages) |
|||
{ |
|||
var query = BuildQuery(term, appVersion, schemaVersion, schema, languages); |
|||
|
|||
var result = new List<Guid>(); |
|||
|
|||
if (indexReader != null) |
|||
{ |
|||
var hits = new IndexSearcher(indexReader).Search(query, MaxResults).ScoreDocs; |
|||
|
|||
foreach (var hit in hits) |
|||
{ |
|||
var document = indexReader.Document(hit.Doc); |
|||
|
|||
var idField = document.GetField("id")?.GetStringValue(); |
|||
|
|||
if (idField != null && Guid.TryParse(idField, out var guid)) |
|||
{ |
|||
result.Add(guid); |
|||
} |
|||
} |
|||
} |
|||
|
|||
return Task.FromResult(result); |
|||
} |
|||
|
|||
private Query BuildQuery(string query, int appVersion, int schemaVersion, J<Schema> schema, List<string> language) |
|||
{ |
|||
if (queryParser == null || currentAppVersion != appVersion || currentSchemaVersion != schemaVersion) |
|||
{ |
|||
var fields = BuildFields(schema, language); |
|||
|
|||
queryParser = new MultiFieldQueryParser(Version, fields, Analyzer); |
|||
|
|||
currentAppVersion = appVersion; |
|||
currentSchemaVersion = schemaVersion; |
|||
} |
|||
|
|||
return queryParser.Parse(query); |
|||
} |
|||
|
|||
private string[] BuildFields(Schema schema, IEnumerable<string> languages) |
|||
{ |
|||
var fieldNames = new List<string>(); |
|||
|
|||
var iv = InvariantPartitioning.Instance.Master.Key; |
|||
|
|||
foreach (var field in schema.Fields) |
|||
{ |
|||
if (field.RawProperties is StringFieldProperties) |
|||
{ |
|||
if (field.Partitioning.Equals(Partitioning.Invariant)) |
|||
{ |
|||
fieldNames.Add(BuildFieldName(iv, field.Name)); |
|||
} |
|||
else |
|||
{ |
|||
foreach (var language in languages) |
|||
{ |
|||
fieldNames.Add(BuildFieldName(language, field.Name)); |
|||
} |
|||
} |
|||
} |
|||
else if (field is IArrayField arrayField) |
|||
{ |
|||
foreach (var nested in arrayField.Fields) |
|||
{ |
|||
if (nested.RawProperties is StringFieldProperties) |
|||
{ |
|||
if (field.Partitioning.Equals(Partitioning.Invariant)) |
|||
{ |
|||
fieldNames.Add(BuildFieldName(iv, field.Name, nested.Name)); |
|||
} |
|||
else |
|||
{ |
|||
foreach (var language in languages) |
|||
{ |
|||
fieldNames.Add(BuildFieldName(language, field.Name, nested.Name)); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return fieldNames.ToArray(); |
|||
} |
|||
|
|||
private async Task TryFlushAsync() |
|||
{ |
|||
updates++; |
|||
|
|||
if (updates >= MaxUpdates) |
|||
{ |
|||
await FlushAsync(); |
|||
} |
|||
} |
|||
|
|||
public async Task FlushAsync() |
|||
{ |
|||
if (updates > 0 && indexWriter != null) |
|||
{ |
|||
indexWriter.Flush(true, true); |
|||
indexWriter.Commit(); |
|||
|
|||
indexReader?.Dispose(); |
|||
indexReader = indexWriter.GetReader(true); |
|||
|
|||
await assetStore.UploadDirectoryAsync(directory); |
|||
|
|||
updates = 0; |
|||
} |
|||
} |
|||
|
|||
public async Task DeactivateAsync(bool deleteFolder = false) |
|||
{ |
|||
await TryFlushAsync(); |
|||
|
|||
indexWriter?.Dispose(); |
|||
indexWriter = null; |
|||
|
|||
indexReader?.Dispose(); |
|||
indexReader = null; |
|||
|
|||
if (deleteFolder && directory.Exists) |
|||
{ |
|||
directory.Delete(true); |
|||
} |
|||
} |
|||
|
|||
private static string BuildFieldName(string language, string name) |
|||
{ |
|||
return $"{language}_{name}"; |
|||
} |
|||
|
|||
private static string BuildFieldName(string language, string name, string nested) |
|||
{ |
|||
return $"{language}_{name}_{nested}"; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,165 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Core; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Core.Schemas; |
|||
using Squidex.Infrastructure.Assets; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerGrainTests : IDisposable |
|||
{ |
|||
private readonly Schema schema = |
|||
new Schema("test") |
|||
.AddString(1, "test", Partitioning.Invariant) |
|||
.AddString(2, "localized", Partitioning.Language); |
|||
private readonly List<string> languages = new List<string> { "de", "en" }; |
|||
private readonly Guid schemaId = Guid.NewGuid(); |
|||
private readonly List<Guid> ids1 = new List<Guid> { Guid.NewGuid() }; |
|||
private readonly List<Guid> ids2 = new List<Guid> { Guid.NewGuid() }; |
|||
private readonly IAssetStore assetStore = new MemoryAssetStore(); |
|||
private readonly TextIndexerGrain sut; |
|||
|
|||
public TextIndexerGrainTests() |
|||
{ |
|||
sut = new TextIndexerGrain(assetStore); |
|||
sut.ActivateAsync(schemaId).Wait(); |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
sut.OnDeactivateAsync().Wait(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_read_index_and_retrieve() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
await sut.DeactivateAsync(); |
|||
|
|||
var other = new TextIndexerGrain(assetStore); |
|||
try |
|||
{ |
|||
await other.ActivateAsync(schemaId); |
|||
|
|||
var helloIds = await other.SearchAsync("Hello", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids1, helloIds); |
|||
|
|||
var worldIds = await other.SearchAsync("World", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids2, worldIds); |
|||
} |
|||
finally |
|||
{ |
|||
await other.OnDeactivateAsync(); |
|||
} |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_index_invariant_content_and_retrieve() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
var helloIds = await sut.SearchAsync("Hello", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids1, helloIds); |
|||
|
|||
var worldIds = await sut.SearchAsync("World", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids2, worldIds); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_delete_documents_from_index() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
await sut.DeleteContentAsync(ids1[0]); |
|||
await sut.FlushAsync(); |
|||
|
|||
var helloIds = await sut.SearchAsync("Hello", 0, 0, schema, languages); |
|||
|
|||
Assert.Empty(helloIds); |
|||
|
|||
var worldIds = await sut.SearchAsync("World", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids2, worldIds); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_index_localized_content_and_retrieve() |
|||
{ |
|||
await AddLocalizedContent(); |
|||
|
|||
var german1 = await sut.SearchAsync("Stadt", 0, 0, schema, languages); |
|||
var german2 = await sut.SearchAsync("and", 0, 0, schema, languages); |
|||
|
|||
var germanStopwordsIds = await sut.SearchAsync("und", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids1, german1); |
|||
Assert.Equal(ids1, german2); |
|||
|
|||
Assert.Equal(ids2, germanStopwordsIds); |
|||
|
|||
var english1 = await sut.SearchAsync("City", 0, 0, schema, languages); |
|||
var english2 = await sut.SearchAsync("und", 0, 0, schema, languages); |
|||
|
|||
var englishStopwordsIds = await sut.SearchAsync("and", 0, 0, schema, languages); |
|||
|
|||
Assert.Equal(ids2, english1); |
|||
Assert.Equal(ids2, english2); |
|||
|
|||
Assert.Equal(ids1, englishStopwordsIds); |
|||
} |
|||
|
|||
private async Task AddLocalizedContent() |
|||
{ |
|||
var germanData = |
|||
new NamedContentData() |
|||
.AddField("localized", |
|||
new ContentFieldData() |
|||
.AddValue("de", "Stadt und Umgebung and whatever")); |
|||
|
|||
var englishData = |
|||
new NamedContentData() |
|||
.AddField("localized", |
|||
new ContentFieldData() |
|||
.AddValue("en", "City and Surroundings und sonstiges")); |
|||
|
|||
await sut.AddContentAsync(ids1[0], germanData, false, false); |
|||
await sut.AddContentAsync(ids2[0], englishData, false, false); |
|||
await sut.FlushAsync(); |
|||
} |
|||
|
|||
private async Task AddInvariantContent() |
|||
{ |
|||
var data1 = |
|||
new NamedContentData() |
|||
.AddField("test", |
|||
new ContentFieldData() |
|||
.AddValue("iv", "Hello")); |
|||
|
|||
var data2 = |
|||
new NamedContentData() |
|||
.AddField("test", |
|||
new ContentFieldData() |
|||
.AddValue("iv", "World")); |
|||
|
|||
await sut.AddContentAsync(ids1[0], data1, false, false); |
|||
await sut.AddContentAsync(ids2[0], data2, false, false); |
|||
|
|||
await sut.FlushAsync(); |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue