mirror of https://github.com/Squidex/squidex.git
Browse Source
# Conflicts: # backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoIndexStorage.cs # backend/src/Squidex.Domain.Apps.Entities.MongoDb/FullText/MongoTextIndexerState.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Elastic/ElasticSearchTextIndex.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/ITextIndex.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/ILuceneTextIndexGrain.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/IndexManager_Impl.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndex.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/LuceneTextIndexGrain.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/AssetIndexStorage.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/FileIndexStorage.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/Lucene/Storage/IIndexStorage.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/CachingTextIndexerState.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/ITextIndexerState.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/State/InMemoryTextIndexerState.cs # backend/src/Squidex.Domain.Apps.Entities/Contents/Text/TextIndexingProcess.cs # backend/src/Squidex.Domain.Apps.Entities/Rules/RuleEnqueuer.cs # backend/src/Squidex.Infrastructure/EventSourcing/RetrySubscription.cs # backend/tests/Squidex.Domain.Apps.Core.Tests/Operations/ValidateContent/ValidationTestExtensions.cs # backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/CachingTextIndexerStateTests.cs # backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/LuceneIndexFactory.cs # backend/tests/Squidex.Domain.Apps.Entities.Tests/Contents/Text/TextIndexerTestsBase.cs # backend/tests/Squidex.Domain.Apps.Entities.Tests/Rules/RuleEnqueuerTests.cs # backend/tests/Squidex.Infrastructure.Tests/EventSourcing/Grains/EventConsumerGrainTests.cspull/590/head
Sebastian
5 years ago
99 changed files with 1878 additions and 2636 deletions
@ -1,123 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.IO; |
|||
using System.IO.Compression; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Store; |
|||
using MongoDB.Driver.GridFS; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; |
|||
using Squidex.Infrastructure; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoIndexStorage : IIndexStorage |
|||
{ |
|||
private readonly IGridFSBucket<string> bucket; |
|||
|
|||
public MongoIndexStorage(IGridFSBucket<string> bucket) |
|||
{ |
|||
Guard.NotNull(bucket, nameof(bucket)); |
|||
|
|||
this.bucket = bucket; |
|||
} |
|||
|
|||
public async Task<LuceneDirectory> CreateDirectoryAsync(DomainId ownerId) |
|||
{ |
|||
var fileId = $"index_{ownerId}"; |
|||
|
|||
var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), fileId)); |
|||
|
|||
if (directoryInfo.Exists) |
|||
{ |
|||
directoryInfo.Delete(true); |
|||
} |
|||
|
|||
directoryInfo.Create(); |
|||
|
|||
try |
|||
{ |
|||
using (var stream = await bucket.OpenDownloadStreamAsync(fileId)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Read, true)) |
|||
{ |
|||
foreach (var entry in zipArchive.Entries) |
|||
{ |
|||
var file = new FileInfo(Path.Combine(directoryInfo.FullName, entry.Name)); |
|||
|
|||
using (var entryStream = entry.Open()) |
|||
{ |
|||
using (var fileStream = file.OpenWrite()) |
|||
{ |
|||
await entryStream.CopyToAsync(fileStream); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
catch (GridFSFileNotFoundException) |
|||
{ |
|||
} |
|||
|
|||
var directory = FSDirectory.Open(directoryInfo); |
|||
|
|||
return directory; |
|||
} |
|||
|
|||
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) |
|||
{ |
|||
var directoryInfo = ((FSDirectory)directory).Directory; |
|||
|
|||
var commit = snapshotter.Snapshot(); |
|||
try |
|||
{ |
|||
var fileId = directoryInfo.Name; |
|||
|
|||
try |
|||
{ |
|||
await bucket.DeleteAsync(fileId); |
|||
} |
|||
catch (GridFSFileNotFoundException) |
|||
{ |
|||
} |
|||
|
|||
using (var stream = await bucket.OpenUploadStreamAsync(fileId, fileId)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(stream, ZipArchiveMode.Create, true)) |
|||
{ |
|||
foreach (var fileName in commit.FileNames) |
|||
{ |
|||
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); |
|||
|
|||
using (var fileStream = file.OpenRead()) |
|||
{ |
|||
var entry = zipArchive.CreateEntry(fileStream.Name); |
|||
|
|||
using (var entryStream = entry.Open()) |
|||
{ |
|||
await fileStream.CopyToAsync(entryStream); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
finally |
|||
{ |
|||
snapshotter.Release(commit); |
|||
} |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return bucket.DropAsync(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,168 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using MongoDB.Driver; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoTextIndex : MongoRepositoryBase<MongoTextIndexEntity>, ITextIndex |
|||
{ |
|||
private static readonly List<DomainId> EmptyResults = new List<DomainId>(); |
|||
|
|||
public MongoTextIndex(IMongoDatabase database, bool setup = false) |
|||
: base(database, setup) |
|||
{ |
|||
} |
|||
|
|||
protected override Task SetupCollectionAsync(IMongoCollection<MongoTextIndexEntity> collection, CancellationToken ct = default) |
|||
{ |
|||
return collection.Indexes.CreateManyAsync(new[] |
|||
{ |
|||
new CreateIndexModel<MongoTextIndexEntity>( |
|||
Index |
|||
.Text("t.t") |
|||
.Ascending(x => x.AppId) |
|||
.Ascending(x => x.ServeAll) |
|||
.Ascending(x => x.ServePublished) |
|||
.Ascending(x => x.SchemaId)) |
|||
}, ct); |
|||
} |
|||
|
|||
protected override string CollectionName() |
|||
{ |
|||
return "TextIndex"; |
|||
} |
|||
|
|||
public Task ExecuteAsync(params IndexCommand[] commands) |
|||
{ |
|||
var writes = new List<WriteModel<MongoTextIndexEntity>>(commands.Length); |
|||
|
|||
foreach (var command in commands) |
|||
{ |
|||
switch (command) |
|||
{ |
|||
case DeleteIndexEntry delete: |
|||
writes.Add( |
|||
new DeleteOneModel<MongoTextIndexEntity>( |
|||
Filter.Eq(x => x.DocId, command.DocId))); |
|||
break; |
|||
case UpdateIndexEntry update: |
|||
writes.Add( |
|||
new UpdateOneModel<MongoTextIndexEntity>( |
|||
Filter.Eq(x => x.DocId, command.DocId), |
|||
Update |
|||
.Set(x => x.ServeAll, update.ServeAll) |
|||
.Set(x => x.ServePublished, update.ServePublished))); |
|||
break; |
|||
case UpsertIndexEntry upsert when upsert.Texts.Count > 0: |
|||
writes.Add( |
|||
new ReplaceOneModel<MongoTextIndexEntity>( |
|||
Filter.Eq(x => x.DocId, command.DocId), |
|||
new MongoTextIndexEntity |
|||
{ |
|||
DocId = upsert.DocId, |
|||
ContentId = upsert.ContentId, |
|||
SchemaId = upsert.SchemaId.Id, |
|||
ServeAll = upsert.ServeAll, |
|||
ServePublished = upsert.ServePublished, |
|||
Texts = upsert.Texts.Select(x => new MongoTextIndexEntityText { Text = x.Value }).ToList(), |
|||
AppId = upsert.AppId.Id |
|||
}) |
|||
{ |
|||
IsUpsert = true |
|||
}); |
|||
break; |
|||
} |
|||
} |
|||
|
|||
if (writes.Count == 0) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
return Collection.BulkWriteAsync(writes); |
|||
} |
|||
|
|||
public async Task<List<DomainId>?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope) |
|||
{ |
|||
if (string.IsNullOrWhiteSpace(queryText)) |
|||
{ |
|||
return EmptyResults; |
|||
} |
|||
|
|||
if (filter == null) |
|||
{ |
|||
return await SearchByAppAsync(queryText, app, scope, 2000); |
|||
} |
|||
else if (filter.Must) |
|||
{ |
|||
return await SearchBySchemaAsync(queryText, app, filter, scope, 2000); |
|||
} |
|||
else |
|||
{ |
|||
var (bySchema, byApp) = |
|||
await AsyncHelper.WhenAll( |
|||
SearchBySchemaAsync(queryText, app, filter, scope, 1000), |
|||
SearchByAppAsync(queryText, app, scope, 1000)); |
|||
|
|||
return bySchema.Union(byApp).Distinct().ToList(); |
|||
} |
|||
} |
|||
|
|||
private async Task<List<DomainId>> SearchBySchemaAsync(string queryText, IAppEntity app, SearchFilter filter, SearchScope scope, int limit) |
|||
{ |
|||
var bySchema = |
|||
await Collection.Find( |
|||
Filter.And( |
|||
Filter.Eq(x => x.AppId, app.Id), |
|||
Filter.In(x => x.SchemaId, filter.SchemaIds), |
|||
Filter_ByScope(scope), |
|||
Filter.Text(queryText))) |
|||
.Only(x => x.ContentId).Limit(limit) |
|||
.ToListAsync(); |
|||
|
|||
return bySchema.Select(x => DomainId.Create(x["_ci"].AsString)).Distinct().ToList(); |
|||
} |
|||
|
|||
private async Task<List<DomainId>> SearchByAppAsync(string queryText, IAppEntity app, SearchScope scope, int limit) |
|||
{ |
|||
var bySchema = |
|||
await Collection.Find( |
|||
Filter.And( |
|||
Filter.Eq(x => x.AppId, app.Id), |
|||
Filter.Exists(x => x.SchemaId), |
|||
Filter_ByScope(scope), |
|||
Filter.Text(queryText))) |
|||
.Only(x => x.ContentId).Limit(limit) |
|||
.ToListAsync(); |
|||
|
|||
return bySchema.Select(x => DomainId.Create(x["_ci"].AsString)).Distinct().ToList(); |
|||
} |
|||
|
|||
private static FilterDefinition<MongoTextIndexEntity> Filter_ByScope(SearchScope scope) |
|||
{ |
|||
if (scope == SearchScope.All) |
|||
{ |
|||
return Filter.Eq(x => x.ServeAll, true); |
|||
} |
|||
else |
|||
{ |
|||
return Filter.Eq(x => x.ServePublished, true); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,49 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using MongoDB.Bson; |
|||
using MongoDB.Bson.Serialization.Attributes; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoTextIndexEntity |
|||
{ |
|||
[BsonId] |
|||
[BsonElement] |
|||
[BsonRepresentation(BsonType.String)] |
|||
public string DocId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("_ci")] |
|||
[BsonRepresentation(BsonType.String)] |
|||
public DomainId ContentId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("_ai")] |
|||
[BsonRepresentation(BsonType.String)] |
|||
public DomainId AppId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("_si")] |
|||
[BsonRepresentation(BsonType.String)] |
|||
public DomainId SchemaId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("fa")] |
|||
public bool ServeAll { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("fp")] |
|||
public bool ServePublished { get; set; } |
|||
|
|||
[BsonIgnoreIfNull] |
|||
[BsonElement("t")] |
|||
public List<MongoTextIndexEntityText> Texts { get; set; } |
|||
} |
|||
} |
|||
@ -1,61 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using MongoDB.Bson.Serialization.Attributes; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.State; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.FullText |
|||
{ |
|||
public sealed class MongoTextIndexState |
|||
{ |
|||
[BsonId] |
|||
[BsonElement] |
|||
public DomainId DocumentId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement] |
|||
public DomainId ContentId { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("c")] |
|||
public string DocIdCurrent { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("n")] |
|||
public string? DocIdNew { get; set; } |
|||
|
|||
[BsonRequired] |
|||
[BsonElement("p")] |
|||
public string? DocIdForPublished { get; set; } |
|||
|
|||
public MongoTextIndexState() |
|||
{ |
|||
} |
|||
|
|||
public MongoTextIndexState(DomainId documentId, TextContentState state) |
|||
{ |
|||
DocumentId = documentId; |
|||
|
|||
ContentId = state.ContentId; |
|||
DocIdNew = state.DocIdNew; |
|||
DocIdCurrent = state.DocIdCurrent; |
|||
DocIdForPublished = state.DocIdForPublished; |
|||
} |
|||
|
|||
public TextContentState ToState() |
|||
{ |
|||
return new TextContentState |
|||
{ |
|||
ContentId = ContentId, |
|||
DocIdNew = DocIdNew, |
|||
DocIdCurrent = DocIdCurrent, |
|||
DocIdForPublished = DocIdForPublished |
|||
}; |
|||
} |
|||
} |
|||
} |
|||
@ -1,28 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public interface IIndex |
|||
{ |
|||
Analyzer? Analyzer { get; } |
|||
|
|||
IndexReader? Reader { get; } |
|||
|
|||
IndexSearcher? Searcher { get; } |
|||
|
|||
IndexWriter Writer { get; } |
|||
|
|||
void EnsureReader(); |
|||
|
|||
void MarkStale(); |
|||
} |
|||
} |
|||
@ -1,22 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Orleans; |
|||
using Orleans.Concurrency; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public interface ILuceneTextIndexGrain : IGrainWithStringKey |
|||
{ |
|||
Task IndexAsync(NamedId<DomainId> schemaId, Immutable<IndexCommand[]> updates); |
|||
|
|||
Task<List<DomainId>> SearchAsync(string queryText, SearchFilter? filter, SearchContext context); |
|||
} |
|||
} |
|||
@ -1,150 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public sealed partial class IndexManager : DisposableObjectBase |
|||
{ |
|||
private readonly Dictionary<DomainId, IndexHolder> indices = new Dictionary<DomainId, IndexHolder>(); |
|||
private readonly SemaphoreSlim lockObject = new SemaphoreSlim(1); |
|||
private readonly IIndexStorage indexStorage; |
|||
private readonly ISemanticLog log; |
|||
|
|||
public IndexManager(IIndexStorage indexStorage, ISemanticLog log) |
|||
{ |
|||
Guard.NotNull(indexStorage, nameof(indexStorage)); |
|||
Guard.NotNull(log, nameof(log)); |
|||
|
|||
this.indexStorage = indexStorage; |
|||
|
|||
this.log = log; |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
ReleaseAllAsync().Wait(); |
|||
} |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return indexStorage.ClearAsync(); |
|||
} |
|||
|
|||
public async Task<IIndex> AcquireAsync(DomainId ownerId) |
|||
{ |
|||
IndexHolder? indexHolder; |
|||
|
|||
try |
|||
{ |
|||
await lockObject.WaitAsync(); |
|||
|
|||
if (indices.TryGetValue(ownerId, out indexHolder)) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased index found.") |
|||
.WriteProperty("ownerId", ownerId.ToString())); |
|||
|
|||
await CommitInternalAsync(indexHolder, true); |
|||
} |
|||
|
|||
var directory = await indexStorage.CreateDirectoryAsync(ownerId); |
|||
|
|||
indexHolder = new IndexHolder(ownerId, directory); |
|||
indices[ownerId] = indexHolder; |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
return indexHolder; |
|||
} |
|||
|
|||
public async Task ReleaseAsync(IIndex index) |
|||
{ |
|||
Guard.NotNull(index, nameof(index)); |
|||
|
|||
var indexHolder = (IndexHolder)index; |
|||
|
|||
try |
|||
{ |
|||
await lockObject.WaitAsync(); |
|||
|
|||
indexHolder.Dispose(); |
|||
indices.Remove(indexHolder.Id); |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
await CommitInternalAsync(indexHolder, true); |
|||
} |
|||
|
|||
public Task CommitAsync(IIndex index) |
|||
{ |
|||
Guard.NotNull(index, nameof(index)); |
|||
|
|||
return CommitInternalAsync(index, false); |
|||
} |
|||
|
|||
private async Task CommitInternalAsync(IIndex index, bool dispose) |
|||
{ |
|||
if (index is IndexHolder holder) |
|||
{ |
|||
if (dispose) |
|||
{ |
|||
holder.Dispose(); |
|||
} |
|||
else |
|||
{ |
|||
holder.Commit(); |
|||
} |
|||
|
|||
await indexStorage.WriteAsync(holder.Directory, holder.Snapshotter); |
|||
} |
|||
} |
|||
|
|||
private async Task ReleaseAllAsync() |
|||
{ |
|||
var current = indices.Values.ToList(); |
|||
|
|||
try |
|||
{ |
|||
await lockObject.WaitAsync(); |
|||
|
|||
indices.Clear(); |
|||
} |
|||
finally |
|||
{ |
|||
lockObject.Release(); |
|||
} |
|||
|
|||
if (current.Count > 0) |
|||
{ |
|||
log.LogWarning(w => w |
|||
.WriteProperty("message", "Unreleased indices found.") |
|||
.WriteProperty("count", indices.Count)); |
|||
|
|||
foreach (var index in current) |
|||
{ |
|||
await CommitInternalAsync(index, true); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,180 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public sealed partial class IndexManager |
|||
{ |
|||
private sealed class IndexHolder : IDisposable, IIndex |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private static readonly MergeScheduler MergeScheduler = new ConcurrentMergeScheduler(); |
|||
private static readonly Analyzer SharedAnalyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly SnapshotDeletionPolicy snapshotter = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy()); |
|||
private readonly Directory directory; |
|||
private IndexWriter indexWriter; |
|||
private IndexSearcher? indexSearcher; |
|||
private DirectoryReader? indexReader; |
|||
private bool isDisposed; |
|||
|
|||
public Analyzer Analyzer |
|||
{ |
|||
get { return SharedAnalyzer; } |
|||
} |
|||
|
|||
public SnapshotDeletionPolicy Snapshotter |
|||
{ |
|||
get { return snapshotter; } |
|||
} |
|||
|
|||
public Directory Directory |
|||
{ |
|||
get { return directory; } |
|||
} |
|||
|
|||
public IndexWriter Writer |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
if (indexWriter == null) |
|||
{ |
|||
throw new InvalidOperationException("Index writer has not been created yet. Call Open()"); |
|||
} |
|||
|
|||
return indexWriter; |
|||
} |
|||
} |
|||
|
|||
public IndexReader? Reader |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
return indexReader; |
|||
} |
|||
} |
|||
|
|||
public IndexSearcher? Searcher |
|||
{ |
|||
get |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
return indexSearcher; |
|||
} |
|||
} |
|||
|
|||
public DomainId Id { get; } |
|||
|
|||
public IndexHolder(DomainId id, Directory directory) |
|||
{ |
|||
Id = id; |
|||
|
|||
this.directory = directory; |
|||
|
|||
RecreateIndexWriter(); |
|||
|
|||
if (indexWriter.NumDocs > 0) |
|||
{ |
|||
EnsureReader(); |
|||
} |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
if (!isDisposed) |
|||
{ |
|||
indexReader?.Dispose(); |
|||
indexReader = null; |
|||
|
|||
indexWriter.Dispose(); |
|||
|
|||
isDisposed = true; |
|||
} |
|||
} |
|||
|
|||
private IndexWriter RecreateIndexWriter() |
|||
{ |
|||
var config = new IndexWriterConfig(Version, Analyzer) |
|||
{ |
|||
IndexDeletionPolicy = snapshotter, |
|||
MergePolicy = new TieredMergePolicy(), |
|||
MergeScheduler = MergeScheduler |
|||
}; |
|||
|
|||
indexWriter = new IndexWriter(directory, config); |
|||
|
|||
MarkStale(); |
|||
|
|||
return indexWriter; |
|||
} |
|||
|
|||
public void EnsureReader() |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
if (indexReader == null && indexWriter != null) |
|||
{ |
|||
indexReader = indexWriter.GetReader(true); |
|||
indexSearcher = new IndexSearcher(indexReader); |
|||
} |
|||
} |
|||
|
|||
public void MarkStale() |
|||
{ |
|||
ThrowIfReleased(); |
|||
|
|||
MarkStaleInternal(); |
|||
} |
|||
|
|||
private void MarkStaleInternal() |
|||
{ |
|||
if (indexReader != null) |
|||
{ |
|||
indexReader.Dispose(); |
|||
indexReader = null; |
|||
indexSearcher = null; |
|||
} |
|||
} |
|||
|
|||
internal void Commit() |
|||
{ |
|||
try |
|||
{ |
|||
MarkStaleInternal(); |
|||
|
|||
indexWriter.Commit(); |
|||
} |
|||
catch (OutOfMemoryException) |
|||
{ |
|||
RecreateIndexWriter(); |
|||
|
|||
throw; |
|||
} |
|||
} |
|||
|
|||
private void ThrowIfReleased() |
|||
{ |
|||
if (indexWriter == null) |
|||
{ |
|||
throw new InvalidOperationException("Index is already released or not open yet."); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,53 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Util; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public static class LuceneExtensions |
|||
{ |
|||
public static BytesRef GetBinaryValue(this IndexReader? reader, string field, int docId, BytesRef? result = null) |
|||
{ |
|||
if (result != null) |
|||
{ |
|||
Array.Clear(result.Bytes, 0, result.Bytes.Length); |
|||
} |
|||
else |
|||
{ |
|||
result = new BytesRef(); |
|||
} |
|||
|
|||
if (reader == null || docId < 0) |
|||
{ |
|||
return result; |
|||
} |
|||
|
|||
var leaves = reader.Leaves; |
|||
|
|||
if (leaves.Count == 1) |
|||
{ |
|||
var docValues = leaves[0].AtomicReader.GetBinaryDocValues(field); |
|||
|
|||
docValues.Get(docId, result); |
|||
} |
|||
else if (leaves.Count > 1) |
|||
{ |
|||
var subIndex = ReaderUtil.SubIndex(docId, leaves); |
|||
|
|||
var subLeave = leaves[subIndex]; |
|||
var subValues = subLeave.AtomicReader.GetBinaryDocValues(field); |
|||
|
|||
subValues.Get(docId - subLeave.DocBase, result); |
|||
} |
|||
|
|||
return result; |
|||
} |
|||
} |
|||
} |
|||
@ -1,69 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Orleans; |
|||
using Orleans.Concurrency; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public sealed class LuceneTextIndex : ITextIndex |
|||
{ |
|||
private readonly IGrainFactory grainFactory; |
|||
private readonly IndexManager indexManager; |
|||
|
|||
public LuceneTextIndex(IGrainFactory grainFactory, IndexManager indexManager) |
|||
{ |
|||
Guard.NotNull(grainFactory, nameof(grainFactory)); |
|||
Guard.NotNull(indexManager, nameof(indexManager)); |
|||
|
|||
this.grainFactory = grainFactory; |
|||
|
|||
this.indexManager = indexManager; |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return indexManager.ClearAsync(); |
|||
} |
|||
|
|||
public async Task<List<DomainId>?> SearchAsync(string? queryText, IAppEntity app, SearchFilter? filter, SearchScope scope) |
|||
{ |
|||
if (string.IsNullOrWhiteSpace(queryText)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
var index = grainFactory.GetGrain<ILuceneTextIndexGrain>(app.Id.ToString()); |
|||
|
|||
using (Profiler.TraceMethod<LuceneTextIndex>()) |
|||
{ |
|||
var context = CreateContext(app, scope); |
|||
|
|||
return await index.SearchAsync(queryText, filter, context); |
|||
} |
|||
} |
|||
|
|||
private static SearchContext CreateContext(IAppEntity app, SearchScope scope) |
|||
{ |
|||
var languages = new HashSet<string>(app.LanguagesConfig.AllKeys); |
|||
|
|||
return new SearchContext { Languages = languages, Scope = scope }; |
|||
} |
|||
|
|||
public Task ExecuteAsync(NamedId<DomainId> appId, NamedId<DomainId> schemaId, params IndexCommand[] commands) |
|||
{ |
|||
var index = grainFactory.GetGrain<ILuceneTextIndexGrain>(appId.Id.ToString()); |
|||
|
|||
return index.IndexAsync(schemaId, commands.AsImmutable()); |
|||
} |
|||
} |
|||
} |
|||
@ -1,259 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Documents; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.QueryParsers.Classic; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Util; |
|||
using Orleans.Concurrency; |
|||
using Squidex.Domain.Apps.Core; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Orleans; |
|||
using Squidex.Infrastructure.Validation; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public sealed class LuceneTextIndexGrain : GrainOfString, ILuceneTextIndexGrain |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private const int MaxResults = 2000; |
|||
private const int MaxUpdates = 400; |
|||
private const string MetaId = "_id"; |
|||
private const string MetaFor = "_fd"; |
|||
private const string MetaContentId = "_cid"; |
|||
private const string MetaSchemaId = "_si"; |
|||
private const string MetaSchemaName = "_sn"; |
|||
private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(10); |
|||
private static readonly string[] Invariant = { InvariantPartitioning.Key }; |
|||
private readonly IndexManager indexManager; |
|||
private IDisposable? timer; |
|||
private IIndex index; |
|||
private QueryParser? queryParser; |
|||
private HashSet<string>? currentLanguages; |
|||
private int updates; |
|||
|
|||
public LuceneTextIndexGrain(IndexManager indexManager) |
|||
{ |
|||
Guard.NotNull(indexManager, nameof(indexManager)); |
|||
|
|||
this.indexManager = indexManager; |
|||
} |
|||
|
|||
public override async Task OnDeactivateAsync() |
|||
{ |
|||
if (index != null) |
|||
{ |
|||
await CommitAsync(); |
|||
|
|||
await indexManager.ReleaseAsync(index); |
|||
} |
|||
} |
|||
|
|||
protected override async Task OnActivateAsync(string key) |
|||
{ |
|||
index = await indexManager.AcquireAsync(key); |
|||
} |
|||
|
|||
public Task<List<DomainId>> SearchAsync(string queryText, SearchFilter? filter, SearchContext context) |
|||
{ |
|||
var result = new List<DomainId>(); |
|||
|
|||
if (!string.IsNullOrWhiteSpace(queryText)) |
|||
{ |
|||
index.EnsureReader(); |
|||
|
|||
if (index.Searcher != null) |
|||
{ |
|||
var query = BuildQuery(queryText, filter, context); |
|||
|
|||
var hits = index.Searcher.Search(query, MaxResults).ScoreDocs; |
|||
|
|||
if (hits.Length > 0) |
|||
{ |
|||
var buffer = new BytesRef(2); |
|||
|
|||
var found = new HashSet<DomainId>(); |
|||
|
|||
foreach (var hit in hits) |
|||
{ |
|||
var forValue = index.Reader.GetBinaryValue(MetaFor, hit.Doc, buffer); |
|||
|
|||
if (context.Scope == SearchScope.All && forValue.Bytes[0] != 1) |
|||
{ |
|||
continue; |
|||
} |
|||
|
|||
if (context.Scope == SearchScope.Published && forValue.Bytes[1] != 1) |
|||
{ |
|||
continue; |
|||
} |
|||
|
|||
var document = index.Searcher.Doc(hit.Doc); |
|||
|
|||
var idString = document?.Get(MetaContentId); |
|||
|
|||
if (idString != null) |
|||
{ |
|||
if (found.Add(idString)) |
|||
{ |
|||
result.Add(idString); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return Task.FromResult(result); |
|||
} |
|||
|
|||
private Query BuildQuery(string query, SearchFilter? filter, SearchContext context) |
|||
{ |
|||
if (queryParser == null || currentLanguages == null || !currentLanguages.SetEquals(context.Languages)) |
|||
{ |
|||
var fields = context.Languages.Union(Invariant).ToArray(); |
|||
|
|||
queryParser = new MultiFieldQueryParser(Version, fields, index.Analyzer); |
|||
|
|||
currentLanguages = context.Languages; |
|||
} |
|||
|
|||
try |
|||
{ |
|||
var byQuery = queryParser.Parse(query); |
|||
|
|||
if (filter?.SchemaIds.Count > 0) |
|||
{ |
|||
var bySchemas = new BooleanQuery |
|||
{ |
|||
Boost = 2f |
|||
}; |
|||
|
|||
foreach (var schemaId in filter.SchemaIds) |
|||
{ |
|||
var term = new Term(MetaSchemaId, schemaId.ToString()); |
|||
|
|||
bySchemas.Add(new TermQuery(term), Occur.SHOULD); |
|||
} |
|||
|
|||
var occur = filter.Must ? Occur.MUST : Occur.SHOULD; |
|||
|
|||
return new BooleanQuery |
|||
{ |
|||
{ byQuery, Occur.MUST }, |
|||
{ bySchemas, occur } |
|||
}; |
|||
} |
|||
|
|||
return byQuery; |
|||
} |
|||
catch (ParseException ex) |
|||
{ |
|||
throw new ValidationException(ex.Message); |
|||
} |
|||
} |
|||
|
|||
private async Task<bool> TryCommitAsync() |
|||
{ |
|||
timer?.Dispose(); |
|||
|
|||
updates++; |
|||
|
|||
if (updates >= MaxUpdates) |
|||
{ |
|||
await CommitAsync(); |
|||
|
|||
return true; |
|||
} |
|||
else |
|||
{ |
|||
index.MarkStale(); |
|||
|
|||
try |
|||
{ |
|||
timer = RegisterTimer(_ => CommitAsync(), null, CommitDelay, CommitDelay); |
|||
} |
|||
catch (InvalidOperationException) |
|||
{ |
|||
return false; |
|||
} |
|||
} |
|||
|
|||
return false; |
|||
} |
|||
|
|||
public async Task CommitAsync() |
|||
{ |
|||
if (updates > 0) |
|||
{ |
|||
await indexManager.CommitAsync(index); |
|||
|
|||
updates = 0; |
|||
} |
|||
} |
|||
|
|||
public Task IndexAsync(NamedId<DomainId> schemaId, Immutable<IndexCommand[]> updates) |
|||
{ |
|||
foreach (var command in updates.Value) |
|||
{ |
|||
switch (command) |
|||
{ |
|||
case DeleteIndexEntry delete: |
|||
index.Writer.DeleteDocuments(new Term(MetaId, delete.DocId)); |
|||
break; |
|||
case UpdateIndexEntry update: |
|||
try |
|||
{ |
|||
var values = GetValue(update.ServeAll, update.ServePublished); |
|||
|
|||
index.Writer.UpdateBinaryDocValue(new Term(MetaId, update.DocId), MetaFor, values); |
|||
} |
|||
catch (ArgumentException) |
|||
{ |
|||
} |
|||
|
|||
break; |
|||
case UpsertIndexEntry upsert: |
|||
{ |
|||
var document = new Document(); |
|||
|
|||
document.AddStringField(MetaId, upsert.DocId, Field.Store.YES); |
|||
document.AddStringField(MetaContentId, upsert.ContentId.ToString(), Field.Store.YES); |
|||
document.AddStringField(MetaSchemaId, schemaId.Id.ToString(), Field.Store.YES); |
|||
document.AddStringField(MetaSchemaName, schemaId.Name, Field.Store.YES); |
|||
document.AddBinaryDocValuesField(MetaFor, GetValue(upsert.ServeAll, upsert.ServePublished)); |
|||
|
|||
foreach (var (key, value) in upsert.Texts) |
|||
{ |
|||
document.AddTextField(key, value, Field.Store.NO); |
|||
} |
|||
|
|||
index.Writer.UpdateDocument(new Term(MetaId, upsert.DocId), document); |
|||
|
|||
break; |
|||
} |
|||
} |
|||
} |
|||
|
|||
return TryCommitAsync(); |
|||
} |
|||
|
|||
private static BytesRef GetValue(bool forDraft, bool forPublished) |
|||
{ |
|||
return new BytesRef(new[] |
|||
{ |
|||
(byte)(forDraft ? 1 : 0), |
|||
(byte)(forPublished ? 1 : 0) |
|||
}); |
|||
} |
|||
} |
|||
} |
|||
@ -1,65 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Analysis.Standard; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public sealed class MultiLanguageAnalyzer : AnalyzerWrapper |
|||
{ |
|||
private readonly StandardAnalyzer fallbackAnalyzer; |
|||
private readonly Dictionary<string, Analyzer> analyzers = new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase); |
|||
|
|||
public MultiLanguageAnalyzer(LuceneVersion version) |
|||
: base(PER_FIELD_REUSE_STRATEGY) |
|||
{ |
|||
fallbackAnalyzer = new StandardAnalyzer(version); |
|||
|
|||
foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) |
|||
{ |
|||
if (typeof(Analyzer).IsAssignableFrom(type)) |
|||
{ |
|||
var language = type.Namespace!.Split('.').Last(); |
|||
|
|||
if (language.Length == 2) |
|||
{ |
|||
try |
|||
{ |
|||
var analyzer = Activator.CreateInstance(type, version)!; |
|||
|
|||
analyzers[language] = (Analyzer)analyzer; |
|||
} |
|||
catch (MissingMethodException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected override Analyzer GetWrappedAnalyzer(string fieldName) |
|||
{ |
|||
if (fieldName.Length >= 2) |
|||
{ |
|||
var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; |
|||
|
|||
return analyzer; |
|||
} |
|||
else |
|||
{ |
|||
return fallbackAnalyzer; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,113 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.IO; |
|||
using System.IO.Compression; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Store; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Assets; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage |
|||
{ |
|||
public sealed class AssetIndexStorage : IIndexStorage |
|||
{ |
|||
private const string ArchiveFile = "Archive.zip"; |
|||
private readonly IAssetStore assetStore; |
|||
|
|||
public AssetIndexStorage(IAssetStore assetStore) |
|||
{ |
|||
Guard.NotNull(assetStore, nameof(assetStore)); |
|||
|
|||
this.assetStore = assetStore; |
|||
} |
|||
|
|||
public async Task<LuceneDirectory> CreateDirectoryAsync(DomainId ownerId) |
|||
{ |
|||
var directoryInfo = new DirectoryInfo(Path.Combine(Path.GetTempPath(), "LocalIndices", ownerId.ToString())); |
|||
|
|||
if (directoryInfo.Exists) |
|||
{ |
|||
directoryInfo.Delete(true); |
|||
} |
|||
|
|||
directoryInfo.Create(); |
|||
|
|||
using (var fileStream = GetArchiveStream(directoryInfo)) |
|||
{ |
|||
try |
|||
{ |
|||
await assetStore.DownloadAsync(directoryInfo.Name, fileStream); |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) |
|||
{ |
|||
zipArchive.ExtractToDirectory(directoryInfo.FullName); |
|||
} |
|||
} |
|||
catch (AssetNotFoundException) |
|||
{ |
|||
} |
|||
} |
|||
|
|||
var directory = FSDirectory.Open(directoryInfo); |
|||
|
|||
return directory; |
|||
} |
|||
|
|||
public async Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) |
|||
{ |
|||
var directoryInfo = ((FSDirectory)directory).Directory; |
|||
|
|||
var commit = snapshotter.Snapshot(); |
|||
try |
|||
{ |
|||
using (var fileStream = GetArchiveStream(directoryInfo)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) |
|||
{ |
|||
foreach (var fileName in commit.FileNames) |
|||
{ |
|||
var file = new FileInfo(Path.Combine(directoryInfo.FullName, fileName)); |
|||
|
|||
zipArchive.CreateEntryFromFile(file.FullName, file.Name); |
|||
} |
|||
} |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
await assetStore.UploadAsync(directoryInfo.Name, fileStream, true); |
|||
} |
|||
} |
|||
finally |
|||
{ |
|||
snapshotter.Release(commit); |
|||
} |
|||
} |
|||
|
|||
private static FileStream GetArchiveStream(DirectoryInfo directoryInfo) |
|||
{ |
|||
var path = Path.Combine(directoryInfo.FullName, ArchiveFile); |
|||
|
|||
return new FileStream( |
|||
path, |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose); |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
} |
|||
} |
|||
@ -1,37 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.IO; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Store; |
|||
using Squidex.Infrastructure; |
|||
using LuceneDirectory = Lucene.Net.Store.Directory; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage |
|||
{ |
|||
public sealed class FileIndexStorage : IIndexStorage |
|||
{ |
|||
public Task<LuceneDirectory> CreateDirectoryAsync(DomainId ownerId) |
|||
{ |
|||
var folderName = $"Indexes/{ownerId}"; |
|||
var folderPath = Path.Combine(Path.GetTempPath(), folderName); |
|||
|
|||
return Task.FromResult<LuceneDirectory>(FSDirectory.Open(folderPath)); |
|||
} |
|||
|
|||
public Task WriteAsync(LuceneDirectory directory, SnapshotDeletionPolicy snapshotter) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
} |
|||
} |
|||
@ -1,23 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Store; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text.Lucene |
|||
{ |
|||
public interface IIndexStorage |
|||
{ |
|||
Task<Directory> CreateDirectoryAsync(DomainId ownerId); |
|||
|
|||
Task WriteAsync(Directory directory, SnapshotDeletionPolicy snapshotter); |
|||
|
|||
Task ClearAsync(); |
|||
} |
|||
} |
|||
@ -1,77 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public sealed class CompoundEventConsumer : IEventConsumer |
|||
{ |
|||
private readonly IEventConsumer[] inners; |
|||
|
|||
public string Name { get; } |
|||
|
|||
public string EventsFilter { get; } |
|||
|
|||
public CompoundEventConsumer(IEventConsumer first, params IEventConsumer[] inners) |
|||
: this(first?.Name!, first!, inners) |
|||
{ |
|||
} |
|||
|
|||
public CompoundEventConsumer(IEventConsumer[] inners) |
|||
{ |
|||
Guard.NotNull(inners, nameof(inners)); |
|||
Guard.NotEmpty(inners, nameof(inners)); |
|||
|
|||
this.inners = inners; |
|||
|
|||
Name = inners.First().Name; |
|||
|
|||
var innerFilters = |
|||
this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter)) |
|||
.Select(x => $"({x.EventsFilter})"); |
|||
|
|||
EventsFilter = string.Join("|", innerFilters); |
|||
} |
|||
|
|||
public CompoundEventConsumer(string name, IEventConsumer first, params IEventConsumer[] inners) |
|||
{ |
|||
Guard.NotNull(first, nameof(first)); |
|||
Guard.NotNull(inners, nameof(inners)); |
|||
Guard.NotNullOrEmpty(name, nameof(name)); |
|||
|
|||
this.inners = new[] { first }.Union(inners).ToArray(); |
|||
|
|||
Name = name; |
|||
|
|||
var innerFilters = |
|||
this.inners.Where(x => !string.IsNullOrWhiteSpace(x.EventsFilter)) |
|||
.Select(x => $"({x.EventsFilter})"); |
|||
|
|||
EventsFilter = string.Join("|", innerFilters); |
|||
} |
|||
|
|||
public bool Handles(StoredEvent @event) |
|||
{ |
|||
return inners.Any(x => x.Handles(@event)); |
|||
} |
|||
|
|||
public Task ClearAsync() |
|||
{ |
|||
return Task.WhenAll(inners.Select(i => i.ClearAsync())); |
|||
} |
|||
|
|||
public async Task On(Envelope<IEvent> @event) |
|||
{ |
|||
foreach (var inner in inners) |
|||
{ |
|||
await inner.On(@event); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,193 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
using System.Threading.Tasks.Dataflow; |
|||
using Squidex.Infrastructure.Reflection; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Grains |
|||
{ |
|||
internal sealed class BatchSubscriber : IEventSubscriber |
|||
{ |
|||
private readonly ITargetBlock<Job> pipelineStart; |
|||
private readonly IEventDataFormatter eventDataFormatter; |
|||
private readonly IEventSubscription eventSubscription; |
|||
private readonly IDataflowBlock pipelineEnd; |
|||
|
|||
private sealed class Job |
|||
{ |
|||
public StoredEvent? StoredEvent { get; set; } |
|||
|
|||
public Exception? Exception { get; set; } |
|||
|
|||
public Envelope<IEvent>? Event { get; set; } |
|||
|
|||
public bool ShouldHandle { get; set; } |
|||
|
|||
public object Sender { get; set; } |
|||
} |
|||
|
|||
public BatchSubscriber( |
|||
EventConsumerGrain grain, |
|||
IEventDataFormatter eventDataFormatter, |
|||
IEventConsumer eventConsumer, |
|||
Func<IEventSubscriber, IEventSubscription> factory, |
|||
TaskScheduler scheduler) |
|||
{ |
|||
this.eventDataFormatter = eventDataFormatter; |
|||
|
|||
var batchSize = Math.Max(1, eventConsumer!.BatchSize); |
|||
var batchDelay = Math.Max(100, eventConsumer.BatchDelay); |
|||
|
|||
var parse = new TransformBlock<Job, Job>(job => |
|||
{ |
|||
if (job.StoredEvent != null) |
|||
{ |
|||
job.ShouldHandle = eventConsumer.Handles(job.StoredEvent); |
|||
} |
|||
|
|||
if (job.ShouldHandle) |
|||
{ |
|||
try |
|||
{ |
|||
job.Event = ParseKnownEvent(job.StoredEvent!); |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
job.Exception = ex; |
|||
} |
|||
} |
|||
|
|||
return job; |
|||
}, new ExecutionDataflowBlockOptions |
|||
{ |
|||
BoundedCapacity = batchSize, |
|||
MaxDegreeOfParallelism = 1, |
|||
MaxMessagesPerTask = 1 |
|||
}); |
|||
|
|||
var buffer = AsyncHelper.CreateBatchBlock<Job>(batchSize, batchDelay, new GroupingDataflowBlockOptions |
|||
{ |
|||
BoundedCapacity = batchSize * 2 |
|||
}); |
|||
|
|||
var handle = new ActionBlock<IList<Job>>(async jobs => |
|||
{ |
|||
foreach (var jobsBySender in jobs.GroupBy<Job, object>(x => x.Sender)) |
|||
{ |
|||
var sender = jobsBySender.Key; |
|||
|
|||
if (ReferenceEquals(sender, eventSubscription.Sender)) |
|||
{ |
|||
var exception = jobs.FirstOrDefault(x => x.Exception != null)?.Exception; |
|||
|
|||
if (exception != null) |
|||
{ |
|||
await grain.OnErrorAsync(exception); |
|||
} |
|||
else |
|||
{ |
|||
await grain.OnEventsAsync(GetEvents(jobsBySender), GetPosition(jobsBySender)); |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
new ExecutionDataflowBlockOptions |
|||
{ |
|||
BoundedCapacity = 2, |
|||
MaxDegreeOfParallelism = 1, |
|||
MaxMessagesPerTask = 1, |
|||
TaskScheduler = scheduler |
|||
}); |
|||
|
|||
parse.LinkTo(buffer, new DataflowLinkOptions |
|||
{ |
|||
PropagateCompletion = true |
|||
}); |
|||
|
|||
buffer.LinkTo(handle, new DataflowLinkOptions |
|||
{ |
|||
PropagateCompletion = true |
|||
}); |
|||
|
|||
pipelineStart = parse; |
|||
pipelineEnd = handle; |
|||
|
|||
eventSubscription = factory(this); |
|||
} |
|||
|
|||
private static List<Envelope<IEvent>> GetEvents(IEnumerable<Job> jobsBySender) |
|||
{ |
|||
return jobsBySender.NotNull(x => x.Event).ToList(); |
|||
} |
|||
|
|||
private static string GetPosition(IEnumerable<Job> jobsBySender) |
|||
{ |
|||
return jobsBySender.Last().StoredEvent!.EventPosition; |
|||
} |
|||
|
|||
public Task CompleteAsync() |
|||
{ |
|||
pipelineStart.Complete(); |
|||
|
|||
return pipelineEnd.Completion; |
|||
} |
|||
|
|||
public void WakeUp() |
|||
{ |
|||
eventSubscription.WakeUp(); |
|||
} |
|||
|
|||
public void Unsubscribe() |
|||
{ |
|||
eventSubscription.Unsubscribe(); |
|||
} |
|||
|
|||
private Envelope<IEvent>? ParseKnownEvent(StoredEvent storedEvent) |
|||
{ |
|||
try |
|||
{ |
|||
var @event = eventDataFormatter.Parse(storedEvent.Data); |
|||
|
|||
@event.SetEventPosition(storedEvent.EventPosition); |
|||
@event.SetEventStreamNumber(storedEvent.EventStreamNumber); |
|||
|
|||
return @event; |
|||
} |
|||
catch (TypeNameNotFoundException) |
|||
{ |
|||
return null; |
|||
} |
|||
} |
|||
|
|||
public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) |
|||
{ |
|||
var job = new Job |
|||
{ |
|||
Sender = subscription, |
|||
StoredEvent = storedEvent |
|||
}; |
|||
|
|||
return pipelineStart.SendAsync(job); |
|||
} |
|||
|
|||
public Task OnErrorAsync(IEventSubscription subscription, Exception exception) |
|||
{ |
|||
var job = new Job |
|||
{ |
|||
Sender = subscription, |
|||
Exception = exception |
|||
}; |
|||
|
|||
return pipelineStart.SendAsync(job); |
|||
} |
|||
} |
|||
} |
|||
@ -1,42 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Orleans.Concurrency; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing.Grains |
|||
{ |
|||
internal sealed class WrapperSubscription : IEventSubscriber |
|||
{ |
|||
private readonly IEventConsumerGrain grain; |
|||
private readonly TaskScheduler scheduler; |
|||
|
|||
public WrapperSubscription(IEventConsumerGrain grain, TaskScheduler scheduler) |
|||
{ |
|||
this.grain = grain; |
|||
|
|||
this.scheduler = scheduler ?? TaskScheduler.Default; |
|||
} |
|||
|
|||
public Task OnEventAsync(IEventSubscription subscription, StoredEvent storedEvent) |
|||
{ |
|||
return Dispatch(() => grain.OnEventAsync(subscription.AsImmutable(), storedEvent.AsImmutable())); |
|||
} |
|||
|
|||
public Task OnErrorAsync(IEventSubscription subscription, Exception exception) |
|||
{ |
|||
return Dispatch(() => grain.OnErrorAsync(subscription.AsImmutable(), exception.AsImmutable())); |
|||
} |
|||
|
|||
private Task Dispatch(Func<Task> task) |
|||
{ |
|||
return Task<Task>.Factory.StartNew(task, CancellationToken.None, TaskCreationOptions.None, scheduler).Unwrap(); |
|||
} |
|||
} |
|||
} |
|||
@ -1,67 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading.Tasks; |
|||
using System.Threading.Tasks.Dataflow; |
|||
|
|||
namespace Squidex.Infrastructure.Tasks |
|||
{ |
|||
public sealed class SingleThreadedDispatcher |
|||
{ |
|||
private readonly ActionBlock<Func<Task>> block; |
|||
private bool isStopped; |
|||
|
|||
public SingleThreadedDispatcher(int capacity = 1) |
|||
{ |
|||
var options = new ExecutionDataflowBlockOptions |
|||
{ |
|||
BoundedCapacity = capacity, |
|||
MaxMessagesPerTask = 1, |
|||
MaxDegreeOfParallelism = 1 |
|||
}; |
|||
|
|||
block = new ActionBlock<Func<Task>>(Handle, options); |
|||
} |
|||
|
|||
public Task DispatchAsync(Func<Task> action) |
|||
{ |
|||
Guard.NotNull(action, nameof(action)); |
|||
|
|||
return block.SendAsync(action); |
|||
} |
|||
|
|||
public Task DispatchAsync(Action action) |
|||
{ |
|||
Guard.NotNull(action, nameof(action)); |
|||
|
|||
return block.SendAsync(() => { action(); return Task.CompletedTask; }); |
|||
} |
|||
|
|||
public async Task StopAndWaitAsync() |
|||
{ |
|||
await DispatchAsync(() => |
|||
{ |
|||
isStopped = true; |
|||
|
|||
block.Complete(); |
|||
}); |
|||
|
|||
await block.Completion; |
|||
} |
|||
|
|||
private Task Handle(Func<Task> action) |
|||
{ |
|||
if (isStopped) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
return action(); |
|||
} |
|||
} |
|||
} |
|||
@ -1,56 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Lucene.Net.Analysis.Standard; |
|||
using Lucene.Net.Documents; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class DocValuesTests |
|||
{ |
|||
[Fact] |
|||
public void Should_read_and_write_doc_values() |
|||
{ |
|||
var version = LuceneVersion.LUCENE_48; |
|||
|
|||
var indexWriter = |
|||
new IndexWriter(new RAMDirectory(), |
|||
new IndexWriterConfig(version, new StandardAnalyzer(version))); |
|||
|
|||
using (indexWriter) |
|||
{ |
|||
for (byte i = 0; i < 255; i++) |
|||
{ |
|||
var document = new Document(); |
|||
|
|||
document.AddBinaryDocValuesField("field", new BytesRef(new[] { i })); |
|||
|
|||
indexWriter.AddDocument(document); |
|||
} |
|||
|
|||
indexWriter.Commit(); |
|||
|
|||
using (var reader = indexWriter.GetReader(true)) |
|||
{ |
|||
var bytesRef = new BytesRef(1); |
|||
|
|||
for (byte i = 0; i < 255; i++) |
|||
{ |
|||
reader.GetBinaryValue("field", i, bytesRef); |
|||
|
|||
Assert.Equal(i, bytesRef.Bytes[0]); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,50 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Tasks; |
|||
using FakeItEasy; |
|||
using Orleans; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class LuceneIndexFactory : IIndexerFactory |
|||
{ |
|||
private readonly IGrainFactory grainFactory = A.Fake<IGrainFactory>(); |
|||
private readonly IIndexStorage storage; |
|||
private LuceneTextIndexGrain grain; |
|||
|
|||
public LuceneIndexFactory(IIndexStorage storage) |
|||
{ |
|||
this.storage = storage; |
|||
|
|||
A.CallTo(() => grainFactory.GetGrain<ILuceneTextIndexGrain>(A<string>._, null)) |
|||
.ReturnsLazily(() => grain); |
|||
} |
|||
|
|||
public async Task<ITextIndex> CreateAsync(DomainId schemaId) |
|||
{ |
|||
var indexManager = new IndexManager(storage, A.Fake<ISemanticLog>()); |
|||
|
|||
grain = new LuceneTextIndexGrain(indexManager); |
|||
|
|||
await grain.ActivateAsync(schemaId.ToString()); |
|||
|
|||
return new LuceneTextIndex(grainFactory, indexManager); |
|||
} |
|||
|
|||
public async Task CleanupAsync() |
|||
{ |
|||
if (grain != null) |
|||
{ |
|||
await grain.OnDeactivateAsync(); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,49 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using MongoDB.Driver; |
|||
using MongoDB.Driver.GridFS; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Lucene.Storage; |
|||
using Squidex.Domain.Apps.Entities.MongoDb.FullText; |
|||
using Squidex.Infrastructure.Assets; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public static class TestStorages |
|||
{ |
|||
public static IIndexStorage Assets() |
|||
{ |
|||
var storage = new AssetIndexStorage(new MemoryAssetStore()); |
|||
|
|||
return storage; |
|||
} |
|||
|
|||
public static IIndexStorage TempFolder() |
|||
{ |
|||
var storage = new FileIndexStorage(); |
|||
|
|||
return storage; |
|||
} |
|||
|
|||
public static IIndexStorage MongoDB() |
|||
{ |
|||
var mongoClient = new MongoClient("mongodb://localhost"); |
|||
var mongoDatabase = mongoClient.GetDatabase("FullText"); |
|||
|
|||
var mongoBucket = new GridFSBucket<string>(mongoDatabase, new GridFSBucketOptions |
|||
{ |
|||
BucketName = $"bucket_{DateTime.UtcNow.Ticks}" |
|||
}); |
|||
|
|||
var storage = new MongoIndexStorage(mongoBucket); |
|||
|
|||
return storage; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Entities.Contents.Text.Elastic; |
|||
using Squidex.Infrastructure; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
[Trait("Category", "Dependencies")] |
|||
public class TextIndexerTests_Elastic : TextIndexerTestsBase |
|||
{ |
|||
private sealed class TheFactory : IIndexerFactory |
|||
{ |
|||
public Task CleanupAsync() |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public Task<ITextIndex> CreateAsync(DomainId schemaId) |
|||
{ |
|||
var index = new ElasticSearchTextIndex("http://localhost:9200", "squidex", true); |
|||
|
|||
return Task.FromResult<ITextIndex>(index); |
|||
} |
|||
} |
|||
|
|||
public override IIndexerFactory Factory { get; } = new TheFactory(); |
|||
|
|||
public TextIndexerTests_Elastic() |
|||
{ |
|||
SupportsSearchSyntax = false; |
|||
SupportsMultiLanguage = false; |
|||
} |
|||
} |
|||
} |
|||
@ -1,14 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerTests_FS : TextIndexerTestsBase |
|||
{ |
|||
public override IIndexerFactory Factory { get; } = new LuceneIndexFactory(TestStorages.TempFolder()); |
|||
} |
|||
} |
|||
@ -1,131 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Threading.Tasks; |
|||
using FakeItEasy; |
|||
using Squidex.Infrastructure.TestHelpers; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public class CompoundEventConsumerTests |
|||
{ |
|||
private readonly IEventConsumer consumer1 = A.Fake<IEventConsumer>(); |
|||
private readonly IEventConsumer consumer2 = A.Fake<IEventConsumer>(); |
|||
|
|||
[Fact] |
|||
public void Should_return_given_name() |
|||
{ |
|||
var sut = new CompoundEventConsumer("consumer-name", consumer1); |
|||
|
|||
Assert.Equal("consumer-name", sut.Name); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_return_first_inner_name() |
|||
{ |
|||
A.CallTo(() => consumer1.Name).Returns("my-inner-consumer"); |
|||
|
|||
var sut = new CompoundEventConsumer(consumer1, consumer2); |
|||
|
|||
Assert.Equal("my-inner-consumer", sut.Name); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_return_compound_filter() |
|||
{ |
|||
A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); |
|||
A.CallTo(() => consumer2.EventsFilter).Returns("filter2"); |
|||
|
|||
var sut = new CompoundEventConsumer("my", consumer1, consumer2); |
|||
|
|||
Assert.Equal("(filter1)|(filter2)", sut.EventsFilter); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_return_compound_filter_from_array() |
|||
{ |
|||
A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); |
|||
A.CallTo(() => consumer2.EventsFilter).Returns("filter2"); |
|||
|
|||
var sut = new CompoundEventConsumer(new[] { consumer1, consumer2 }); |
|||
|
|||
Assert.Equal("(filter1)|(filter2)", sut.EventsFilter); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_ignore_empty_filters() |
|||
{ |
|||
A.CallTo(() => consumer1.EventsFilter).Returns("filter1"); |
|||
A.CallTo(() => consumer2.EventsFilter).Returns(string.Empty); |
|||
|
|||
var sut = new CompoundEventConsumer("my", consumer1, consumer2); |
|||
|
|||
Assert.Equal("(filter1)", sut.EventsFilter); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_clear_all_consumers() |
|||
{ |
|||
var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); |
|||
|
|||
await sut.ClearAsync(); |
|||
|
|||
A.CallTo(() => consumer1.ClearAsync()).MustHaveHappened(); |
|||
A.CallTo(() => consumer2.ClearAsync()).MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_invoke_all_consumers() |
|||
{ |
|||
var @event = Envelope.Create<IEvent>(new MyEvent()); |
|||
|
|||
var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); |
|||
|
|||
await sut.On(@event); |
|||
|
|||
A.CallTo(() => consumer1.On(@event)).MustHaveHappened(); |
|||
A.CallTo(() => consumer2.On(@event)).MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_handle_if_any_consumer_handles() |
|||
{ |
|||
var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload")); |
|||
|
|||
A.CallTo(() => consumer1.Handles(stored)) |
|||
.Returns(false); |
|||
|
|||
A.CallTo(() => consumer2.Handles(stored)) |
|||
.Returns(true); |
|||
|
|||
var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); |
|||
|
|||
var result = sut.Handles(stored); |
|||
|
|||
Assert.True(result); |
|||
} |
|||
|
|||
[Fact] |
|||
public void Should_no_handle_if_no_consumer_handles() |
|||
{ |
|||
var stored = new StoredEvent("Stream", "1", 1, new EventData("Type", new EnvelopeHeaders(), "Payload")); |
|||
|
|||
A.CallTo(() => consumer1.Handles(stored)) |
|||
.Returns(false); |
|||
|
|||
A.CallTo(() => consumer2.Handles(stored)) |
|||
.Returns(false); |
|||
|
|||
var sut = new CompoundEventConsumer("consumer-name", consumer1, consumer2); |
|||
|
|||
var result = sut.Handles(stored); |
|||
|
|||
Assert.False(result); |
|||
} |
|||
} |
|||
} |
|||
@ -1,56 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Infrastructure.Tasks |
|||
{ |
|||
public class SingleThreadedDispatcherTests |
|||
{ |
|||
private readonly SingleThreadedDispatcher sut = new SingleThreadedDispatcher(); |
|||
|
|||
[Fact] |
|||
public async Task Should_handle_with_task_messages_sequentially() |
|||
{ |
|||
var source = Enumerable.Range(1, 100); |
|||
var target = new List<int>(); |
|||
|
|||
foreach (var item in source) |
|||
{ |
|||
sut.DispatchAsync(() => |
|||
{ |
|||
target.Add(item); |
|||
|
|||
return Task.CompletedTask; |
|||
}).Forget(); |
|||
} |
|||
|
|||
await sut.StopAndWaitAsync(); |
|||
|
|||
Assert.Equal(source, target); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_handle_messages_sequentially() |
|||
{ |
|||
var source = Enumerable.Range(1, 100); |
|||
var target = new List<int>(); |
|||
|
|||
foreach (var item in source) |
|||
{ |
|||
sut.DispatchAsync(() => target.Add(item)).Forget(); |
|||
} |
|||
|
|||
await sut.StopAndWaitAsync(); |
|||
|
|||
Assert.Equal(source, target); |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue