mirror of https://github.com/Squidex/squidex.git
Sebastian
7 years ago
67 changed files with 2449 additions and 383 deletions
@ -1,147 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using MongoDB.Driver; |
|||
using NodaTime; |
|||
using Squidex.Domain.Apps.Core.ConvertContent; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Contents.State; |
|||
using Squidex.Domain.Apps.Entities.MongoDb.Contents.Visitors; |
|||
using Squidex.Domain.Apps.Entities.Schemas; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Json; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
using Squidex.Infrastructure.Queries; |
|||
using Squidex.Infrastructure.Reflection; |
|||
using Squidex.Infrastructure.States; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents |
|||
{ |
|||
internal sealed class MongoContentDraftCollection : MongoContentCollection |
|||
{ |
|||
public MongoContentDraftCollection(IMongoDatabase database, IJsonSerializer serializer) |
|||
: base(database, serializer, "State_Content_Draft") |
|||
{ |
|||
} |
|||
|
|||
protected override async Task SetupCollectionAsync(IMongoCollection<MongoContentEntity> collection, CancellationToken ct = default) |
|||
{ |
|||
await collection.Indexes.CreateManyAsync( |
|||
new[] |
|||
{ |
|||
new CreateIndexModel<MongoContentEntity>( |
|||
Index |
|||
.Ascending(x => x.IndexedSchemaId) |
|||
.Ascending(x => x.Id) |
|||
.Ascending(x => x.IsDeleted)), |
|||
new CreateIndexModel<MongoContentEntity>( |
|||
Index |
|||
.Text(x => x.DataText) |
|||
.Ascending(x => x.IndexedSchemaId) |
|||
.Ascending(x => x.IsDeleted) |
|||
.Ascending(x => x.Status)) |
|||
}, ct); |
|||
|
|||
await base.SetupCollectionAsync(collection, ct); |
|||
} |
|||
|
|||
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId, ISchemaEntity schema, FilterNode filterNode) |
|||
{ |
|||
var filter = filterNode.AdjustToModel(schema.SchemaDef, true).ToFilter(schema.Id); |
|||
|
|||
var contentEntities = |
|||
await Collection.Find(filter).Only(x => x.Id) |
|||
.ToListAsync(); |
|||
|
|||
return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); |
|||
} |
|||
|
|||
public async Task<IReadOnlyList<Guid>> QueryIdsAsync(Guid appId) |
|||
{ |
|||
var contentEntities = |
|||
await Collection.Find(x => x.IndexedAppId == appId).Only(x => x.Id) |
|||
.ToListAsync(); |
|||
|
|||
return contentEntities.Select(x => Guid.Parse(x["_id"].AsString)).ToList(); |
|||
} |
|||
|
|||
public Task QueryScheduledWithoutDataAsync(Instant now, Func<IContentEntity, Task> callback) |
|||
{ |
|||
return Collection.Find(x => x.ScheduledAt < now && x.IsDeleted != true) |
|||
.Not(x => x.DataByIds) |
|||
.Not(x => x.DataDraftByIds) |
|||
.Not(x => x.DataText) |
|||
.ForEachAsync(c => |
|||
{ |
|||
callback(c); |
|||
}); |
|||
} |
|||
|
|||
public async Task<IContentEntity> FindContentAsync(IAppEntity app, ISchemaEntity schema, Guid id) |
|||
{ |
|||
var contentEntity = |
|||
await Collection.Find(x => x.IndexedSchemaId == schema.Id && x.Id == id && x.IsDeleted != true).Not(x => x.DataText) |
|||
.FirstOrDefaultAsync(); |
|||
|
|||
contentEntity?.ParseData(schema.SchemaDef, Serializer); |
|||
|
|||
return contentEntity; |
|||
} |
|||
|
|||
public async Task<(ContentState Value, long Version)> ReadAsync(Guid key, Func<Guid, Guid, Task<ISchemaEntity>> getSchema) |
|||
{ |
|||
var contentEntity = |
|||
await Collection.Find(x => x.Id == key).Not(x => x.DataText) |
|||
.FirstOrDefaultAsync(); |
|||
|
|||
if (contentEntity != null) |
|||
{ |
|||
var schema = await getSchema(contentEntity.IndexedAppId, contentEntity.IndexedSchemaId); |
|||
|
|||
contentEntity.ParseData(schema.SchemaDef, Serializer); |
|||
|
|||
return (SimpleMapper.Map(contentEntity, new ContentState()), contentEntity.Version); |
|||
} |
|||
|
|||
return (null, EtagVersion.NotFound); |
|||
} |
|||
|
|||
public async Task UpsertAsync(MongoContentEntity content, long oldVersion) |
|||
{ |
|||
try |
|||
{ |
|||
content.DataText = content.DataDraftByIds.ToFullText(); |
|||
|
|||
await Collection.ReplaceOneAsync(x => x.Id == content.Id && x.Version == oldVersion, content, Upsert); |
|||
} |
|||
catch (MongoWriteException ex) |
|||
{ |
|||
if (ex.WriteError.Category == ServerErrorCategory.DuplicateKey) |
|||
{ |
|||
var existingVersion = |
|||
await Collection.Find(x => x.Id == content.Id).Only(x => x.Id, x => x.Version) |
|||
.FirstOrDefaultAsync(); |
|||
|
|||
if (existingVersion != null) |
|||
{ |
|||
throw new InconsistentStateException(existingVersion["vs"].AsInt64, oldVersion, ex); |
|||
} |
|||
} |
|||
else |
|||
{ |
|||
throw; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -1,61 +0,0 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using MongoDB.Driver; |
|||
using Squidex.Domain.Apps.Core.ConvertContent; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Contents; |
|||
using Squidex.Domain.Apps.Entities.Schemas; |
|||
using Squidex.Infrastructure.Json; |
|||
using Squidex.Infrastructure.MongoDb; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.MongoDb.Contents |
|||
{ |
|||
internal sealed class MongoContentPublishedCollection : MongoContentCollection |
|||
{ |
|||
public MongoContentPublishedCollection(IMongoDatabase database, IJsonSerializer serializer) |
|||
: base(database, serializer, "State_Content_Published") |
|||
{ |
|||
} |
|||
|
|||
protected override async Task SetupCollectionAsync(IMongoCollection<MongoContentEntity> collection, CancellationToken ct = default) |
|||
{ |
|||
await collection.Indexes.CreateManyAsync( |
|||
new[] |
|||
{ |
|||
new CreateIndexModel<MongoContentEntity>(Index.Text(x => x.DataText).Ascending(x => x.IndexedSchemaId)), |
|||
new CreateIndexModel<MongoContentEntity>(Index.Ascending(x => x.IndexedSchemaId).Ascending(x => x.Id)) |
|||
}, ct); |
|||
|
|||
await base.SetupCollectionAsync(collection, ct); |
|||
} |
|||
|
|||
public async Task<IContentEntity> FindContentAsync(IAppEntity app, ISchemaEntity schema, Guid id) |
|||
{ |
|||
var contentEntity = |
|||
await Collection.Find(x => x.IndexedSchemaId == schema.Id && x.Id == id).Not(x => x.DataText) |
|||
.FirstOrDefaultAsync(); |
|||
|
|||
contentEntity?.ParseData(schema.SchemaDef, Serializer); |
|||
|
|||
return contentEntity; |
|||
} |
|||
|
|||
public Task UpsertAsync(MongoContentEntity content) |
|||
{ |
|||
content.DataText = content.DataByIds.ToFullText(); |
|||
content.DataDraftByIds = null; |
|||
content.ScheduleJob = null; |
|||
content.ScheduledAt = null; |
|||
|
|||
return Collection.ReplaceOneAsync(x => x.Id == content.Id, content, new UpdateOptions { IsUpsert = true }); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,106 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using System.Threading.Tasks; |
|||
using Orleans; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Domain.Apps.Entities.Schemas; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class GrainTextIndexer : ITextIndexer |
|||
{ |
|||
private readonly IGrainFactory grainFactory; |
|||
private readonly ISemanticLog log; |
|||
|
|||
public GrainTextIndexer(IGrainFactory grainFactory, ISemanticLog log) |
|||
{ |
|||
Guard.NotNull(grainFactory, nameof(grainFactory)); |
|||
Guard.NotNull(log, nameof(log)); |
|||
|
|||
this.grainFactory = grainFactory; |
|||
|
|||
this.log = log; |
|||
} |
|||
|
|||
public async Task DeleteAsync(Guid schemaId, Guid id) |
|||
{ |
|||
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
|||
|
|||
using (Profiler.TraceMethod<GrainTextIndexer>()) |
|||
{ |
|||
try |
|||
{ |
|||
await index.DeleteAsync(id); |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
log.LogError(ex, w => w |
|||
.WriteProperty("action", "DeleteTextEntry") |
|||
.WriteProperty("status", "Failed")); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public async Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft) |
|||
{ |
|||
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
|||
|
|||
using (Profiler.TraceMethod<GrainTextIndexer>()) |
|||
{ |
|||
try |
|||
{ |
|||
if (data != null) |
|||
{ |
|||
await index.IndexAsync(id, new IndexData { Data = data }); |
|||
} |
|||
|
|||
if (dataDraft != null) |
|||
{ |
|||
await index.IndexAsync(id, new IndexData { Data = dataDraft, IsDraft = true }); |
|||
} |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
log.LogError(ex, w => w |
|||
.WriteProperty("action", "UpdateTextEntry") |
|||
.WriteProperty("status", "Failed")); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public async Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false) |
|||
{ |
|||
if (string.IsNullOrWhiteSpace(queryText)) |
|||
{ |
|||
return null; |
|||
} |
|||
|
|||
var index = grainFactory.GetGrain<ITextIndexerGrain>(schemaId); |
|||
|
|||
using (Profiler.TraceMethod<GrainTextIndexer>()) |
|||
{ |
|||
var context = CreateContext(app, useDraft); |
|||
|
|||
return await index.SearchAsync(queryText, context); |
|||
} |
|||
} |
|||
|
|||
private static SearchContext CreateContext(IAppEntity app, bool useDraft) |
|||
{ |
|||
var languages = new HashSet<string>(app.LanguagesConfig.Select(x => x.Key)); |
|||
|
|||
return new SearchContext { Languages = languages, IsDraft = useDraft }; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public interface ITextIndexer |
|||
{ |
|||
Task DeleteAsync(Guid schemaId, Guid id); |
|||
|
|||
Task IndexAsync(Guid schemaId, Guid id, NamedContentData data, NamedContentData dataDraft); |
|||
|
|||
Task<List<Guid>> SearchAsync(string queryText, IAppEntity app, Guid schemaId, bool useDraft = false); |
|||
} |
|||
} |
|||
@ -0,0 +1,24 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Orleans; |
|||
using Squidex.Infrastructure.Orleans; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public interface ITextIndexerGrain : IGrainWithGuidKey |
|||
{ |
|||
Task DeleteAsync(Guid id); |
|||
|
|||
Task IndexAsync(Guid id, J<IndexData> data); |
|||
|
|||
Task<List<Guid>> SearchAsync(string queryText, SearchContext context); |
|||
} |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class IndexData |
|||
{ |
|||
public NamedContentData Data { get; set; } |
|||
|
|||
public bool IsDraft { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,65 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Linq; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Analysis.Standard; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class MultiLanguageAnalyzer : AnalyzerWrapper |
|||
{ |
|||
private readonly StandardAnalyzer fallbackAnalyzer; |
|||
private readonly Dictionary<string, Analyzer> analyzers = new Dictionary<string, Analyzer>(StringComparer.OrdinalIgnoreCase); |
|||
|
|||
public MultiLanguageAnalyzer(LuceneVersion version) |
|||
: base(PER_FIELD_REUSE_STRATEGY) |
|||
{ |
|||
fallbackAnalyzer = new StandardAnalyzer(version); |
|||
|
|||
foreach (var type in typeof(StandardAnalyzer).Assembly.GetTypes()) |
|||
{ |
|||
if (typeof(Analyzer).IsAssignableFrom(type)) |
|||
{ |
|||
var language = type.Namespace.Split('.').Last(); |
|||
|
|||
if (language.Length == 2) |
|||
{ |
|||
try |
|||
{ |
|||
var analyzer = Activator.CreateInstance(type, version); |
|||
|
|||
analyzers[language] = (Analyzer)analyzer; |
|||
} |
|||
catch (MissingMethodException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
protected override Analyzer GetWrappedAnalyzer(string fieldName) |
|||
{ |
|||
if (fieldName.Length > 0) |
|||
{ |
|||
var analyzer = analyzers.GetOrDefault(fieldName.Substring(0, 2)) ?? fallbackAnalyzer; |
|||
|
|||
return analyzer; |
|||
} |
|||
else |
|||
{ |
|||
return fallbackAnalyzer; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,91 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.IO; |
|||
using System.IO.Compression; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Infrastructure.Assets; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public static class PersistenceHelper |
|||
{ |
|||
private const string ArchiveFile = "Archive.zip"; |
|||
private const string LockFile = "write.lock"; |
|||
|
|||
public static async Task UploadDirectoryAsync(this IAssetStore assetStore, DirectoryInfo directory) |
|||
{ |
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Create, true)) |
|||
{ |
|||
foreach (var file in directory.GetFiles()) |
|||
{ |
|||
try |
|||
{ |
|||
if (!file.Name.Equals(ArchiveFile, StringComparison.OrdinalIgnoreCase) && |
|||
!file.Name.Equals(LockFile, StringComparison.OrdinalIgnoreCase)) |
|||
{ |
|||
zipArchive.CreateEntryFromFile(file.FullName, file.Name); |
|||
} |
|||
} |
|||
catch (IOException) |
|||
{ |
|||
continue; |
|||
} |
|||
} |
|||
} |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
await assetStore.UploadAsync(directory.Name, 0, string.Empty, fileStream, true); |
|||
} |
|||
} |
|||
|
|||
public static async Task DownloadAsync(this IAssetStore assetStore, DirectoryInfo directory) |
|||
{ |
|||
if (directory.Exists) |
|||
{ |
|||
directory.Delete(true); |
|||
} |
|||
|
|||
directory.Create(); |
|||
|
|||
using (var fileStream = new FileStream( |
|||
Path.Combine(directory.FullName, ArchiveFile), |
|||
FileMode.Create, |
|||
FileAccess.ReadWrite, |
|||
FileShare.None, |
|||
4096, |
|||
FileOptions.DeleteOnClose)) |
|||
{ |
|||
try |
|||
{ |
|||
await assetStore.DownloadAsync(directory.Name, 0, string.Empty, fileStream); |
|||
|
|||
fileStream.Position = 0; |
|||
|
|||
using (var zipArchive = new ZipArchive(fileStream, ZipArchiveMode.Read, true)) |
|||
{ |
|||
zipArchive.ExtractToDirectory(directory.FullName); |
|||
} |
|||
} |
|||
catch (AssetNotFoundException) |
|||
{ |
|||
return; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,18 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class SearchContext |
|||
{ |
|||
public bool IsDraft { get; set; } |
|||
|
|||
public HashSet<string> Languages { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,272 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.IO; |
|||
using System.Linq; |
|||
using System.Text; |
|||
using System.Threading.Tasks; |
|||
using Lucene.Net.Analysis; |
|||
using Lucene.Net.Documents; |
|||
using Lucene.Net.Index; |
|||
using Lucene.Net.Queries; |
|||
using Lucene.Net.QueryParsers.Classic; |
|||
using Lucene.Net.Search; |
|||
using Lucene.Net.Store; |
|||
using Lucene.Net.Util; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Assets; |
|||
using Squidex.Infrastructure.Json.Objects; |
|||
using Squidex.Infrastructure.Orleans; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public sealed class TextIndexerGrain : GrainOfGuid, ITextIndexerGrain |
|||
{ |
|||
private const LuceneVersion Version = LuceneVersion.LUCENE_48; |
|||
private const int MaxResults = 2000; |
|||
private const int MaxUpdates = 100; |
|||
private static readonly TimeSpan CommitDelay = TimeSpan.FromSeconds(30); |
|||
private static readonly Analyzer Analyzer = new MultiLanguageAnalyzer(Version); |
|||
private readonly IAssetStore assetStore; |
|||
private IDisposable timer; |
|||
private DirectoryInfo directory; |
|||
private IndexWriter indexWriter; |
|||
private IndexReader indexReader; |
|||
private QueryParser queryParser; |
|||
private HashSet<string> currentLanguages; |
|||
private long updates; |
|||
|
|||
public TextIndexerGrain(IAssetStore assetStore) |
|||
{ |
|||
Guard.NotNull(assetStore, nameof(assetStore)); |
|||
|
|||
this.assetStore = assetStore; |
|||
} |
|||
|
|||
public override async Task OnDeactivateAsync() |
|||
{ |
|||
await DeactivateAsync(true); |
|||
} |
|||
|
|||
protected override async Task OnActivateAsync(Guid key) |
|||
{ |
|||
directory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), $"Index_{key}")); |
|||
|
|||
await assetStore.DownloadAsync(directory); |
|||
|
|||
indexWriter = new IndexWriter(FSDirectory.Open(directory), new IndexWriterConfig(Version, Analyzer)); |
|||
indexReader = indexWriter.GetReader(true); |
|||
} |
|||
|
|||
public Task DeleteAsync(Guid id) |
|||
{ |
|||
indexWriter.DeleteDocuments(new Term("id", id.ToString())); |
|||
|
|||
return TryFlushAsync(); |
|||
} |
|||
|
|||
public Task IndexAsync(Guid id, J<IndexData> data) |
|||
{ |
|||
var docId = id.ToString(); |
|||
var docDraft = data.Value.IsDraft.ToString(); |
|||
var docKey = $"{docId}_{docDraft}"; |
|||
|
|||
var query = new BooleanQuery(); |
|||
|
|||
indexWriter.DeleteDocuments(new Term("key", docKey)); |
|||
|
|||
var languages = new Dictionary<string, StringBuilder>(); |
|||
|
|||
void AppendText(string language, string text) |
|||
{ |
|||
if (!string.IsNullOrWhiteSpace(text)) |
|||
{ |
|||
var sb = languages.GetOrAddNew(language); |
|||
|
|||
if (sb.Length > 0) |
|||
{ |
|||
sb.Append(" "); |
|||
} |
|||
|
|||
sb.Append(text); |
|||
} |
|||
} |
|||
|
|||
foreach (var field in data.Value.Data) |
|||
{ |
|||
foreach (var fieldValue in field.Value) |
|||
{ |
|||
var appendText = new Action<string>(text => AppendText(fieldValue.Key, text)); |
|||
|
|||
AppendJsonText(fieldValue.Value, appendText); |
|||
} |
|||
} |
|||
|
|||
if (languages.Count > 0) |
|||
{ |
|||
var document = new Document(); |
|||
|
|||
document.AddStringField("id", docId, Field.Store.YES); |
|||
document.AddStringField("key", docKey, Field.Store.YES); |
|||
document.AddStringField("draft", docDraft, Field.Store.YES); |
|||
|
|||
foreach (var field in languages) |
|||
{ |
|||
var fieldName = BuildFieldName(field.Key); |
|||
|
|||
document.AddTextField(fieldName, field.Value.ToString(), Field.Store.NO); |
|||
} |
|||
|
|||
indexWriter.AddDocument(document); |
|||
} |
|||
|
|||
return TryFlushAsync(); |
|||
} |
|||
|
|||
private static void AppendJsonText(IJsonValue value, Action<string> appendText) |
|||
{ |
|||
if (value.Type == JsonValueType.String) |
|||
{ |
|||
appendText(value.ToString()); |
|||
} |
|||
else if (value is JsonArray array) |
|||
{ |
|||
foreach (var item in array) |
|||
{ |
|||
AppendJsonText(item, appendText); |
|||
} |
|||
} |
|||
else if (value is JsonObject obj) |
|||
{ |
|||
foreach (var item in obj.Values) |
|||
{ |
|||
AppendJsonText(item, appendText); |
|||
} |
|||
} |
|||
} |
|||
|
|||
public Task<List<Guid>> SearchAsync(string queryText, SearchContext context) |
|||
{ |
|||
var result = new HashSet<Guid>(); |
|||
|
|||
if (!string.IsNullOrWhiteSpace(queryText)) |
|||
{ |
|||
var query = BuildQuery(queryText, context); |
|||
|
|||
if (indexReader != null) |
|||
{ |
|||
var filter = new TermsFilter(new Term("draft", context.IsDraft.ToString())); |
|||
|
|||
var hits = new IndexSearcher(indexReader).Search(query, filter, MaxResults).ScoreDocs; |
|||
|
|||
foreach (var hit in hits) |
|||
{ |
|||
var document = indexReader.Document(hit.Doc); |
|||
|
|||
var idField = document.GetField("id")?.GetStringValue(); |
|||
|
|||
if (idField != null && Guid.TryParse(idField, out var guid)) |
|||
{ |
|||
result.Add(guid); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
return Task.FromResult(result.ToList()); |
|||
} |
|||
|
|||
private Query BuildQuery(string query, SearchContext context) |
|||
{ |
|||
if (queryParser == null || !currentLanguages.SetEquals(context.Languages)) |
|||
{ |
|||
var fields = |
|||
context.Languages.Select(BuildFieldName) |
|||
.Union(Enumerable.Repeat(BuildFieldName("iv"), 1)).ToArray(); |
|||
|
|||
queryParser = new MultiFieldQueryParser(Version, fields, Analyzer); |
|||
|
|||
currentLanguages = context.Languages; |
|||
} |
|||
|
|||
try |
|||
{ |
|||
return queryParser.Parse(query); |
|||
} |
|||
catch (ParseException ex) |
|||
{ |
|||
throw new ValidationException(ex.Message); |
|||
} |
|||
} |
|||
|
|||
private async Task TryFlushAsync() |
|||
{ |
|||
updates++; |
|||
|
|||
if (updates >= MaxUpdates) |
|||
{ |
|||
await FlushAsync(); |
|||
} |
|||
else |
|||
{ |
|||
timer?.Dispose(); |
|||
|
|||
try |
|||
{ |
|||
timer = RegisterTimer(_ => FlushAsync(), null, CommitDelay, CommitDelay); |
|||
} |
|||
catch (InvalidOperationException) |
|||
{ |
|||
return; |
|||
} |
|||
} |
|||
} |
|||
|
|||
public async Task FlushAsync() |
|||
{ |
|||
if (updates > 0 && indexWriter != null) |
|||
{ |
|||
indexWriter.Flush(true, true); |
|||
indexWriter.Commit(); |
|||
|
|||
indexReader?.Dispose(); |
|||
indexReader = indexWriter.GetReader(true); |
|||
|
|||
await assetStore.UploadDirectoryAsync(directory); |
|||
|
|||
updates = 0; |
|||
} |
|||
else |
|||
{ |
|||
timer?.Dispose(); |
|||
} |
|||
} |
|||
|
|||
public async Task DeactivateAsync(bool deleteFolder = false) |
|||
{ |
|||
await TryFlushAsync(); |
|||
|
|||
indexWriter?.Dispose(); |
|||
indexWriter = null; |
|||
|
|||
indexReader?.Dispose(); |
|||
indexReader = null; |
|||
|
|||
if (deleteFolder && directory.Exists) |
|||
{ |
|||
directory.Delete(true); |
|||
} |
|||
} |
|||
|
|||
private static string BuildFieldName(string language) |
|||
{ |
|||
return $"{language}_field"; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,32 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents.Client; |
|||
using Microsoft.Extensions.Diagnostics.HealthChecks; |
|||
|
|||
namespace Squidex.Infrastructure.Diagnostics |
|||
{ |
|||
public sealed class CosmosDbHealthCheck : IHealthCheck |
|||
{ |
|||
private readonly DocumentClient documentClient; |
|||
|
|||
public CosmosDbHealthCheck(Uri uri, string masterKey) |
|||
{ |
|||
documentClient = new DocumentClient(uri, masterKey); |
|||
} |
|||
|
|||
public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) |
|||
{ |
|||
await documentClient.ReadDatabaseFeedAsync(); |
|||
|
|||
return HealthCheckResult.Healthy("Application must query data from CosmosDB."); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,16 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal static class Constants |
|||
{ |
|||
public const string Collection = "Events"; |
|||
|
|||
public const string LeaseCollection = "Leases"; |
|||
} |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Newtonsoft.Json; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal sealed class CosmosDbEvent |
|||
{ |
|||
[JsonProperty("type")] |
|||
public string Type { get; set; } |
|||
|
|||
[JsonProperty("payload")] |
|||
public string Payload { get; set; } |
|||
|
|||
[JsonProperty("header")] |
|||
public EnvelopeHeaders Headers { get; set; } |
|||
|
|||
public static CosmosDbEvent FromEventData(EventData data) |
|||
{ |
|||
return new CosmosDbEvent { Type = data.Type, Headers = data.Headers, Payload = data.Payload }; |
|||
} |
|||
|
|||
public EventData ToEventData() |
|||
{ |
|||
return new EventData(Type, Headers, Payload); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,33 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Newtonsoft.Json; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal sealed class CosmosDbEventCommit |
|||
{ |
|||
[JsonProperty("id")] |
|||
public Guid Id { get; set; } |
|||
|
|||
[JsonProperty("events")] |
|||
public CosmosDbEvent[] Events { get; set; } |
|||
|
|||
[JsonProperty("eventStreamOffset")] |
|||
public long EventStreamOffset { get; set; } |
|||
|
|||
[JsonProperty("eventsCount")] |
|||
public long EventsCount { get; set; } |
|||
|
|||
[JsonProperty("eventStream")] |
|||
public string EventStream { get; set; } |
|||
|
|||
[JsonProperty("timestamp")] |
|||
public long Timestamp { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,124 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.ObjectModel; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents; |
|||
using Microsoft.Azure.Documents.Client; |
|||
using Newtonsoft.Json; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public sealed partial class CosmosDbEventStore : DisposableObjectBase, IEventStore, IInitializable |
|||
{ |
|||
private readonly DocumentClient documentClient; |
|||
private readonly Uri collectionUri; |
|||
private readonly Uri databaseUri; |
|||
private readonly string masterKey; |
|||
private readonly string databaseId; |
|||
private readonly JsonSerializerSettings serializerSettings; |
|||
|
|||
public JsonSerializerSettings SerializerSettings |
|||
{ |
|||
get { return serializerSettings; } |
|||
} |
|||
|
|||
public string DatabaseId |
|||
{ |
|||
get { return databaseId; } |
|||
} |
|||
|
|||
public string MasterKey |
|||
{ |
|||
get { return masterKey; } |
|||
} |
|||
|
|||
public Uri ServiceUri |
|||
{ |
|||
get { return documentClient.ServiceEndpoint; } |
|||
} |
|||
|
|||
public CosmosDbEventStore(DocumentClient documentClient, string masterKey, string database, JsonSerializerSettings serializerSettings) |
|||
{ |
|||
Guard.NotNull(documentClient, nameof(documentClient)); |
|||
Guard.NotNull(serializerSettings, nameof(serializerSettings)); |
|||
Guard.NotNullOrEmpty(masterKey, nameof(masterKey)); |
|||
Guard.NotNullOrEmpty(database, nameof(database)); |
|||
|
|||
this.documentClient = documentClient; |
|||
|
|||
databaseUri = UriFactory.CreateDatabaseUri(database); |
|||
databaseId = database; |
|||
|
|||
collectionUri = UriFactory.CreateDocumentCollectionUri(database, Constants.Collection); |
|||
|
|||
this.masterKey = masterKey; |
|||
|
|||
this.serializerSettings = serializerSettings; |
|||
} |
|||
|
|||
protected override void DisposeObject(bool disposing) |
|||
{ |
|||
if (disposing) |
|||
{ |
|||
documentClient.Dispose(); |
|||
} |
|||
} |
|||
|
|||
public async Task InitializeAsync(CancellationToken ct = default) |
|||
{ |
|||
await documentClient.CreateDatabaseIfNotExistsAsync(new Database { Id = databaseId }); |
|||
|
|||
await documentClient.CreateDocumentCollectionIfNotExistsAsync(databaseUri, |
|||
new DocumentCollection |
|||
{ |
|||
Id = Constants.LeaseCollection, |
|||
}); |
|||
|
|||
await documentClient.CreateDocumentCollectionIfNotExistsAsync(databaseUri, |
|||
new DocumentCollection |
|||
{ |
|||
IndexingPolicy = new IndexingPolicy |
|||
{ |
|||
IncludedPaths = new Collection<IncludedPath> |
|||
{ |
|||
new IncludedPath |
|||
{ |
|||
Path = "/*", |
|||
Indexes = new Collection<Index> |
|||
{ |
|||
Index.Range(DataType.Number), |
|||
Index.Range(DataType.String), |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
UniqueKeyPolicy = new UniqueKeyPolicy |
|||
{ |
|||
UniqueKeys = new Collection<UniqueKey> |
|||
{ |
|||
new UniqueKey |
|||
{ |
|||
Paths = new Collection<string> |
|||
{ |
|||
$"/eventStream", |
|||
$"/eventStreamOffset" |
|||
} |
|||
} |
|||
} |
|||
}, |
|||
Id = Constants.Collection, |
|||
}, |
|||
new RequestOptions |
|||
{ |
|||
PartitionKey = new PartitionKey($"/eventStream") |
|||
}); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,142 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents; |
|||
using Squidex.Infrastructure.Log; |
|||
using Squidex.Infrastructure.Tasks; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public delegate bool EventPredicate(EventData data); |
|||
|
|||
public partial class CosmosDbEventStore : IEventStore, IInitializable |
|||
{ |
|||
public IEventSubscription CreateSubscription(IEventSubscriber subscriber, string streamFilter = null, string position = null) |
|||
{ |
|||
Guard.NotNull(subscriber, nameof(subscriber)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
return new CosmosDbSubscription(this, subscriber, streamFilter, position); |
|||
} |
|||
|
|||
public Task CreateIndexAsync(string property) |
|||
{ |
|||
Guard.NotNullOrEmpty(property, nameof(property)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public async Task<IReadOnlyList<StoredEvent>> QueryAsync(string streamName, long streamPosition = 0) |
|||
{ |
|||
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
|||
{ |
|||
var query = FilterBuilder.ByStreamName(streamName, streamPosition - MaxCommitSize); |
|||
|
|||
var result = new List<StoredEvent>(); |
|||
|
|||
await documentClient.QueryAsync(collectionUri, query, commit => |
|||
{ |
|||
var eventStreamOffset = (int)commit.EventStreamOffset; |
|||
|
|||
var commitTimestamp = commit.Timestamp; |
|||
var commitOffset = 0; |
|||
|
|||
foreach (var @event in commit.Events) |
|||
{ |
|||
eventStreamOffset++; |
|||
|
|||
if (eventStreamOffset >= streamPosition) |
|||
{ |
|||
var eventData = @event.ToEventData(); |
|||
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); |
|||
|
|||
result.Add(new StoredEvent(streamName, eventToken, eventStreamOffset, eventData)); |
|||
} |
|||
} |
|||
|
|||
return TaskHelper.Done; |
|||
}); |
|||
|
|||
return result; |
|||
} |
|||
} |
|||
|
|||
public Task QueryAsync(Func<StoredEvent, Task> callback, string property, object value, string position = null, CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(callback, nameof(callback)); |
|||
Guard.NotNullOrEmpty(property, nameof(property)); |
|||
Guard.NotNull(value, nameof(value)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
StreamPosition lastPosition = position; |
|||
|
|||
var filterDefinition = FilterBuilder.CreateByProperty(property, value, lastPosition); |
|||
var filterExpression = FilterBuilder.CreateExpression(property, value); |
|||
|
|||
return QueryAsync(callback, lastPosition, filterDefinition, filterExpression, ct); |
|||
} |
|||
|
|||
public Task QueryAsync(Func<StoredEvent, Task> callback, string streamFilter = null, string position = null, CancellationToken ct = default) |
|||
{ |
|||
Guard.NotNull(callback, nameof(callback)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
StreamPosition lastPosition = position; |
|||
|
|||
var filterDefinition = FilterBuilder.CreateByFilter(streamFilter, lastPosition); |
|||
var filterExpression = FilterBuilder.CreateExpression(null, null); |
|||
|
|||
return QueryAsync(callback, lastPosition, filterDefinition, filterExpression, ct); |
|||
} |
|||
|
|||
private async Task QueryAsync(Func<StoredEvent, Task> callback, StreamPosition lastPosition, SqlQuerySpec query, EventPredicate filterExpression, CancellationToken ct = default) |
|||
{ |
|||
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
|||
{ |
|||
await documentClient.QueryAsync(collectionUri, query, async commit => |
|||
{ |
|||
var eventStreamOffset = (int)commit.EventStreamOffset; |
|||
|
|||
var commitTimestamp = commit.Timestamp; |
|||
var commitOffset = 0; |
|||
|
|||
foreach (var @event in commit.Events) |
|||
{ |
|||
eventStreamOffset++; |
|||
|
|||
if (commitOffset > lastPosition.CommitOffset || commitTimestamp > lastPosition.Timestamp) |
|||
{ |
|||
var eventData = @event.ToEventData(); |
|||
|
|||
if (filterExpression(eventData)) |
|||
{ |
|||
var eventToken = new StreamPosition(commitTimestamp, commitOffset, commit.Events.Length); |
|||
|
|||
await callback(new StoredEvent(commit.EventStream, eventToken, eventStreamOffset, eventData)); |
|||
} |
|||
} |
|||
|
|||
commitOffset++; |
|||
} |
|||
}, ct); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,149 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Net; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents; |
|||
using Microsoft.Azure.Documents.Client; |
|||
using NodaTime; |
|||
using Squidex.Infrastructure.Log; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public partial class CosmosDbEventStore |
|||
{ |
|||
private const int MaxWriteAttempts = 20; |
|||
private const int MaxCommitSize = 10; |
|||
|
|||
public Task DeleteStreamAsync(string streamName) |
|||
{ |
|||
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
var query = FilterBuilder.AllIds(streamName); |
|||
|
|||
return documentClient.QueryAsync(collectionUri, query, commit => |
|||
{ |
|||
var documentUri = UriFactory.CreateDocumentUri(databaseId, Constants.Collection, commit.Id.ToString()); |
|||
|
|||
return documentClient.DeleteDocumentAsync(documentUri); |
|||
}); |
|||
} |
|||
|
|||
public Task AppendAsync(Guid commitId, string streamName, ICollection<EventData> events) |
|||
{ |
|||
return AppendAsync(commitId, streamName, EtagVersion.Any, events); |
|||
} |
|||
|
|||
public async Task AppendAsync(Guid commitId, string streamName, long expectedVersion, ICollection<EventData> events) |
|||
{ |
|||
Guard.NotEmpty(commitId, nameof(commitId)); |
|||
Guard.NotNullOrEmpty(streamName, nameof(streamName)); |
|||
Guard.NotNull(events, nameof(events)); |
|||
Guard.LessThan(events.Count, MaxCommitSize, "events.Count"); |
|||
|
|||
ThrowIfDisposed(); |
|||
|
|||
using (Profiler.TraceMethod<CosmosDbEventStore>()) |
|||
{ |
|||
if (events.Count == 0) |
|||
{ |
|||
return; |
|||
} |
|||
|
|||
var currentVersion = await GetEventStreamOffsetAsync(streamName); |
|||
|
|||
if (expectedVersion != EtagVersion.Any && expectedVersion != currentVersion) |
|||
{ |
|||
throw new WrongEventVersionException(currentVersion, expectedVersion); |
|||
} |
|||
|
|||
var commit = BuildCommit(commitId, streamName, expectedVersion >= -1 ? expectedVersion : currentVersion, events); |
|||
|
|||
for (var attempt = 0; attempt < MaxWriteAttempts; attempt++) |
|||
{ |
|||
try |
|||
{ |
|||
await documentClient.CreateDocumentAsync(collectionUri, commit); |
|||
|
|||
return; |
|||
} |
|||
catch (DocumentClientException ex) |
|||
{ |
|||
if (ex.StatusCode == HttpStatusCode.Conflict) |
|||
{ |
|||
currentVersion = await GetEventStreamOffsetAsync(streamName); |
|||
|
|||
if (expectedVersion != EtagVersion.Any) |
|||
{ |
|||
throw new WrongEventVersionException(currentVersion, expectedVersion); |
|||
} |
|||
|
|||
if (attempt < MaxWriteAttempts) |
|||
{ |
|||
expectedVersion = currentVersion; |
|||
} |
|||
else |
|||
{ |
|||
throw new TimeoutException("Could not acquire a free slot for the commit within the provided time."); |
|||
} |
|||
} |
|||
else |
|||
{ |
|||
throw; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
private async Task<long> GetEventStreamOffsetAsync(string streamName) |
|||
{ |
|||
var query = |
|||
documentClient.CreateDocumentQuery<CosmosDbEventCommit>(collectionUri, |
|||
FilterBuilder.LastPosition(streamName)); |
|||
|
|||
var document = await query.FirstOrDefaultAsync(); |
|||
|
|||
if (document != null) |
|||
{ |
|||
return document.EventStreamOffset + document.EventsCount; |
|||
} |
|||
|
|||
return EtagVersion.Empty; |
|||
} |
|||
|
|||
private static CosmosDbEventCommit BuildCommit(Guid commitId, string streamName, long expectedVersion, ICollection<EventData> events) |
|||
{ |
|||
var commitEvents = new CosmosDbEvent[events.Count]; |
|||
|
|||
var i = 0; |
|||
|
|||
foreach (var e in events) |
|||
{ |
|||
var mongoEvent = CosmosDbEvent.FromEventData(e); |
|||
|
|||
commitEvents[i++] = mongoEvent; |
|||
} |
|||
|
|||
var mongoCommit = new CosmosDbEventCommit |
|||
{ |
|||
Id = commitId, |
|||
Events = commitEvents, |
|||
EventsCount = events.Count, |
|||
EventStream = streamName, |
|||
EventStreamOffset = expectedVersion, |
|||
Timestamp = SystemClock.Instance.GetCurrentInstant().ToUnixTimeTicks() |
|||
}; |
|||
|
|||
return mongoCommit; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,150 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Text.RegularExpressions; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents; |
|||
using Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing; |
|||
using Newtonsoft.Json; |
|||
using Builder = Microsoft.Azure.Documents.ChangeFeedProcessor.ChangeFeedProcessorBuilder; |
|||
using Collection = Microsoft.Azure.Documents.ChangeFeedProcessor.DocumentCollectionInfo; |
|||
using Options = Microsoft.Azure.Documents.ChangeFeedProcessor.ChangeFeedProcessorOptions; |
|||
|
|||
#pragma warning disable IDE0017 // Simplify object initialization
|
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal sealed class CosmosDbSubscription : IEventSubscription, IChangeFeedObserverFactory, IChangeFeedObserver |
|||
{ |
|||
private readonly TaskCompletionSource<bool> processorStopRequested = new TaskCompletionSource<bool>(); |
|||
private readonly Task processorTask; |
|||
private readonly CosmosDbEventStore store; |
|||
private readonly Regex regex; |
|||
private readonly string hostName; |
|||
private readonly IEventSubscriber subscriber; |
|||
|
|||
public CosmosDbSubscription(CosmosDbEventStore store, IEventSubscriber subscriber, string streamFilter, string position = null) |
|||
{ |
|||
this.store = store; |
|||
|
|||
var fromBeginning = string.IsNullOrWhiteSpace(position); |
|||
|
|||
if (fromBeginning) |
|||
{ |
|||
hostName = $"squidex.{DateTime.UtcNow.Ticks.ToString()}"; |
|||
} |
|||
else |
|||
{ |
|||
hostName = position; |
|||
} |
|||
|
|||
if (!StreamFilter.IsAll(streamFilter)) |
|||
{ |
|||
regex = new Regex(streamFilter); |
|||
} |
|||
|
|||
this.subscriber = subscriber; |
|||
|
|||
processorTask = Task.Run(async () => |
|||
{ |
|||
try |
|||
{ |
|||
Collection CreateCollection(string name) |
|||
{ |
|||
var collection = new Collection(); |
|||
|
|||
collection.CollectionName = name; |
|||
collection.DatabaseName = store.DatabaseId; |
|||
collection.MasterKey = store.MasterKey; |
|||
collection.Uri = store.ServiceUri; |
|||
|
|||
return collection; |
|||
} |
|||
|
|||
var processor = |
|||
await new Builder() |
|||
.WithFeedCollection(CreateCollection(Constants.Collection)) |
|||
.WithLeaseCollection(CreateCollection(Constants.LeaseCollection)) |
|||
.WithHostName(hostName) |
|||
.WithProcessorOptions(new Options { StartFromBeginning = fromBeginning, LeasePrefix = hostName }) |
|||
.WithObserverFactory(this) |
|||
.BuildAsync(); |
|||
|
|||
await processor.StartAsync(); |
|||
await processorStopRequested.Task; |
|||
await processor.StopAsync(); |
|||
} |
|||
catch (Exception ex) |
|||
{ |
|||
await subscriber.OnErrorAsync(this, ex); |
|||
} |
|||
}); |
|||
} |
|||
|
|||
public IChangeFeedObserver CreateObserver() |
|||
{ |
|||
return this; |
|||
} |
|||
|
|||
public async Task CloseAsync(IChangeFeedObserverContext context, ChangeFeedObserverCloseReason reason) |
|||
{ |
|||
if (reason == ChangeFeedObserverCloseReason.ObserverError) |
|||
{ |
|||
await subscriber.OnErrorAsync(this, new InvalidOperationException("Change feed observer failed.")); |
|||
} |
|||
} |
|||
|
|||
public Task OpenAsync(IChangeFeedObserverContext context) |
|||
{ |
|||
return Task.CompletedTask; |
|||
} |
|||
|
|||
public async Task ProcessChangesAsync(IChangeFeedObserverContext context, IReadOnlyList<Document> docs, CancellationToken cancellationToken) |
|||
{ |
|||
if (!processorStopRequested.Task.IsCompleted) |
|||
{ |
|||
foreach (var document in docs) |
|||
{ |
|||
if (!processorStopRequested.Task.IsCompleted) |
|||
{ |
|||
var streamName = document.GetPropertyValue<string>("eventStream"); |
|||
|
|||
if (regex == null || regex.IsMatch(streamName)) |
|||
{ |
|||
var commit = JsonConvert.DeserializeObject<CosmosDbEventCommit>(document.ToString(), store.SerializerSettings); |
|||
|
|||
var eventStreamOffset = (int)commit.EventStreamOffset; |
|||
|
|||
foreach (var @event in commit.Events) |
|||
{ |
|||
eventStreamOffset++; |
|||
|
|||
var eventData = @event.ToEventData(); |
|||
|
|||
await subscriber.OnEventAsync(this, new StoredEvent(commit.EventStream, hostName, eventStreamOffset, eventData)); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
|
|||
public void WakeUp() |
|||
{ |
|||
} |
|||
|
|||
public Task StopAsync() |
|||
{ |
|||
processorStopRequested.SetResult(true); |
|||
|
|||
return processorTask; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,156 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System.Collections.Generic; |
|||
using Microsoft.Azure.Documents; |
|||
using Squidex.Infrastructure.Json.Objects; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal static class FilterBuilder |
|||
{ |
|||
public static SqlQuerySpec AllIds(string streamName) |
|||
{ |
|||
var query = |
|||
$"SELECT TOP 1 " + |
|||
$" e.id," + |
|||
$" e.eventsCount " + |
|||
$"FROM {Constants.Collection} e " + |
|||
$"WHERE " + |
|||
$" e.eventStream = @name " + |
|||
$"ORDER BY e.eventStreamOffset DESC"; |
|||
|
|||
var parameters = new SqlParameterCollection |
|||
{ |
|||
new SqlParameter("@name", streamName) |
|||
}; |
|||
|
|||
return new SqlQuerySpec(query, parameters); |
|||
} |
|||
|
|||
public static SqlQuerySpec LastPosition(string streamName) |
|||
{ |
|||
var query = |
|||
$"SELECT TOP 1 " + |
|||
$" e.eventStreamOffset," + |
|||
$" e.eventsCount " + |
|||
$"FROM {Constants.Collection} e " + |
|||
$"WHERE " + |
|||
$" e.eventStream = @name " + |
|||
$"ORDER BY e.eventStreamOffset DESC"; |
|||
|
|||
var parameters = new SqlParameterCollection |
|||
{ |
|||
new SqlParameter("@name", streamName) |
|||
}; |
|||
|
|||
return new SqlQuerySpec(query, parameters); |
|||
} |
|||
|
|||
public static SqlQuerySpec ByStreamName(string streamName, long streamPosition = 0) |
|||
{ |
|||
var query = |
|||
$"SELECT * " + |
|||
$"FROM {Constants.Collection} e " + |
|||
$"WHERE " + |
|||
$" e.eventStream = @name " + |
|||
$"AND e.eventStreamOffset >= @position " + |
|||
$"ORDER BY e.eventStreamOffset ASC"; |
|||
|
|||
var parameters = new SqlParameterCollection |
|||
{ |
|||
new SqlParameter("@name", streamName), |
|||
new SqlParameter("@position", streamPosition) |
|||
}; |
|||
|
|||
return new SqlQuerySpec(query, parameters); |
|||
} |
|||
|
|||
public static SqlQuerySpec CreateByProperty(string property, object value, StreamPosition streamPosition) |
|||
{ |
|||
var filters = new List<string>(); |
|||
|
|||
var parameters = new SqlParameterCollection(); |
|||
|
|||
filters.ForPosition(parameters, streamPosition); |
|||
filters.ForProperty(parameters, property, value); |
|||
|
|||
return BuildQuery(filters, parameters); |
|||
} |
|||
|
|||
public static SqlQuerySpec CreateByFilter(string streamFilter, StreamPosition streamPosition) |
|||
{ |
|||
var filters = new List<string>(); |
|||
|
|||
var parameters = new SqlParameterCollection(); |
|||
|
|||
filters.ForPosition(parameters, streamPosition); |
|||
filters.ForRegex(parameters, streamFilter); |
|||
|
|||
return BuildQuery(filters, parameters); |
|||
} |
|||
|
|||
private static SqlQuerySpec BuildQuery(List<string> filters, SqlParameterCollection parameters) |
|||
{ |
|||
var query = $"SELECT * FROM {Constants.Collection} e WHERE {string.Join(" AND ", filters)} ORDER BY e.timestamp"; |
|||
|
|||
return new SqlQuerySpec(query, parameters); |
|||
} |
|||
|
|||
private static void ForProperty(this List<string> filters, SqlParameterCollection parameters, string property, object value) |
|||
{ |
|||
filters.Add($"ARRAY_CONTAINS(e.events, {{ \"header\": {{ \"{property}\": @value }} }}, true)"); |
|||
|
|||
parameters.Add(new SqlParameter("@value", value)); |
|||
} |
|||
|
|||
private static void ForRegex(this List<string> filters, SqlParameterCollection parameters, string streamFilter) |
|||
{ |
|||
if (!StreamFilter.IsAll(streamFilter)) |
|||
{ |
|||
if (streamFilter.Contains("^")) |
|||
{ |
|||
filters.Add($"STARTSWITH(e.eventStream, @filter)"); |
|||
} |
|||
else |
|||
{ |
|||
filters.Add($"e.eventStream = @filter"); |
|||
} |
|||
|
|||
parameters.Add(new SqlParameter("@filter", streamFilter)); |
|||
} |
|||
} |
|||
|
|||
private static void ForPosition(this List<string> filters, SqlParameterCollection parameters, StreamPosition streamPosition) |
|||
{ |
|||
if (streamPosition.IsEndOfCommit) |
|||
{ |
|||
filters.Add($"e.timestamp > @time"); |
|||
} |
|||
else |
|||
{ |
|||
filters.Add($"e.timestamp >= @time"); |
|||
} |
|||
|
|||
parameters.Add(new SqlParameter("@time", streamPosition.Timestamp)); |
|||
} |
|||
|
|||
public static EventPredicate CreateExpression(string property, object value) |
|||
{ |
|||
if (!string.IsNullOrWhiteSpace(property)) |
|||
{ |
|||
var jsonValue = JsonValue.Create(value); |
|||
|
|||
return x => x.Headers.TryGetValue(property, out var p) && p.Equals(jsonValue); |
|||
} |
|||
else |
|||
{ |
|||
return x => true; |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,62 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Linq; |
|||
using System.Threading; |
|||
using System.Threading.Tasks; |
|||
using Microsoft.Azure.Documents; |
|||
using Microsoft.Azure.Documents.Client; |
|||
using Microsoft.Azure.Documents.Linq; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal static class FilterExtensions |
|||
{ |
|||
public static async Task<T> FirstOrDefaultAsync<T>(this IQueryable<T> queryable, CancellationToken ct = default) |
|||
{ |
|||
var documentQuery = queryable.AsDocumentQuery(); |
|||
|
|||
using (documentQuery) |
|||
{ |
|||
if (documentQuery.HasMoreResults) |
|||
{ |
|||
var results = await documentQuery.ExecuteNextAsync<T>(ct); |
|||
|
|||
return results.FirstOrDefault(); |
|||
} |
|||
} |
|||
|
|||
return default; |
|||
} |
|||
|
|||
public static Task QueryAsync(this DocumentClient documentClient, Uri collectionUri, SqlQuerySpec querySpec, Func<CosmosDbEventCommit, Task> handler, CancellationToken ct = default) |
|||
{ |
|||
var query = documentClient.CreateDocumentQuery<CosmosDbEventCommit>(collectionUri, querySpec); |
|||
|
|||
return query.QueryAsync(handler, ct); |
|||
} |
|||
|
|||
public static async Task QueryAsync<T>(this IQueryable<T> queryable, Func<T, Task> handler, CancellationToken ct = default) |
|||
{ |
|||
var documentQuery = queryable.AsDocumentQuery(); |
|||
|
|||
using (documentQuery) |
|||
{ |
|||
while (documentQuery.HasMoreResults && !ct.IsCancellationRequested) |
|||
{ |
|||
var items = await documentQuery.ExecuteNextAsync<T>(ct); |
|||
|
|||
foreach (var item in items) |
|||
{ |
|||
await handler(item); |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,55 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschränkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
internal sealed class StreamPosition |
|||
{ |
|||
public long Timestamp { get; } |
|||
|
|||
public long CommitOffset { get; } |
|||
|
|||
public long CommitSize { get; } |
|||
|
|||
public bool IsEndOfCommit |
|||
{ |
|||
get { return CommitOffset == CommitSize - 1; } |
|||
} |
|||
|
|||
public StreamPosition(long timestamp, long commitOffset, long commitSize) |
|||
{ |
|||
Timestamp = timestamp; |
|||
|
|||
CommitOffset = commitOffset; |
|||
CommitSize = commitSize; |
|||
} |
|||
|
|||
public static implicit operator string(StreamPosition position) |
|||
{ |
|||
var parts = new object[] |
|||
{ |
|||
position.Timestamp, |
|||
position.CommitOffset, |
|||
position.CommitSize |
|||
}; |
|||
|
|||
return string.Join("-", parts); |
|||
} |
|||
|
|||
public static implicit operator StreamPosition(string position) |
|||
{ |
|||
if (!string.IsNullOrWhiteSpace(position)) |
|||
{ |
|||
var parts = position.Split('-'); |
|||
|
|||
return new StreamPosition(long.Parse(parts[0]), long.Parse(parts[1]), long.Parse(parts[2])); |
|||
} |
|||
|
|||
return new StreamPosition(0, -1, -1); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,14 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
namespace Squidex.Infrastructure.MongoDb |
|||
{ |
|||
public sealed class MongoDbOptions |
|||
{ |
|||
public bool IsCosmosDb { get; set; } |
|||
} |
|||
} |
|||
@ -0,0 +1,22 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public static class StreamFilter |
|||
{ |
|||
public static bool IsAll(string filter) |
|||
{ |
|||
return string.IsNullOrWhiteSpace(filter) |
|||
|| string.Equals(filter, ".*", StringComparison.OrdinalIgnoreCase) |
|||
|| string.Equals(filter, "(.*)", StringComparison.OrdinalIgnoreCase) |
|||
|| string.Equals(filter, "(.*?)", StringComparison.OrdinalIgnoreCase); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,144 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using FakeItEasy; |
|||
using Orleans; |
|||
using Squidex.Domain.Apps.Core.Apps; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Domain.Apps.Entities.Apps; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Log; |
|||
using Squidex.Infrastructure.Orleans; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class GrainTextIndexerTests |
|||
{ |
|||
private readonly IGrainFactory grainFactory = A.Fake<IGrainFactory>(); |
|||
private readonly ITextIndexerGrain grain = A.Fake<ITextIndexerGrain>(); |
|||
private readonly Guid schemaId = Guid.NewGuid(); |
|||
private readonly Guid contentId = Guid.NewGuid(); |
|||
private readonly GrainTextIndexer sut; |
|||
|
|||
public GrainTextIndexerTests() |
|||
{ |
|||
A.CallTo(() => grainFactory.GetGrain<ITextIndexerGrain>(schemaId, null)) |
|||
.Returns(grain); |
|||
|
|||
sut = new GrainTextIndexer(grainFactory, A.Fake<ISemanticLog>()); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_call_grain_when_deleting_entry() |
|||
{ |
|||
await sut.DeleteAsync(schemaId, contentId); |
|||
|
|||
A.CallTo(() => grain.DeleteAsync(contentId)) |
|||
.MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_catch_exception_when_deleting_failed() |
|||
{ |
|||
A.CallTo(() => grain.DeleteAsync(contentId)) |
|||
.Throws(new InvalidOperationException()); |
|||
|
|||
await sut.DeleteAsync(schemaId, contentId); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_call_grain_when_indexing_data() |
|||
{ |
|||
var data = new NamedContentData(); |
|||
var dataDraft = new NamedContentData(); |
|||
|
|||
await sut.IndexAsync(schemaId, contentId, data, dataDraft); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) |
|||
.MustHaveHappened(); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) |
|||
.MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_not_call_grain_when_data_is_null() |
|||
{ |
|||
var dataDraft = new NamedContentData(); |
|||
|
|||
await sut.IndexAsync(schemaId, contentId, null, dataDraft); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => !x.Value.IsDraft))) |
|||
.MustNotHaveHappened(); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == dataDraft && x.Value.IsDraft))) |
|||
.MustHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_not_call_grain_when_data_draft_is_null() |
|||
{ |
|||
var data = new NamedContentData(); |
|||
|
|||
await sut.IndexAsync(schemaId, contentId, data, null); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.Data == data && !x.Value.IsDraft))) |
|||
.MustHaveHappened(); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.That.Matches(x => x.Value.IsDraft))) |
|||
.MustNotHaveHappened(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_catch_exception_when_indexing_failed() |
|||
{ |
|||
var data = new NamedContentData(); |
|||
|
|||
A.CallTo(() => grain.IndexAsync(contentId, A<J<IndexData>>.Ignored)) |
|||
.Throws(new InvalidOperationException()); |
|||
|
|||
await sut.IndexAsync(schemaId, contentId, data, null); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_call_grain_when_searching() |
|||
{ |
|||
var foundIds = new List<Guid> { Guid.NewGuid() }; |
|||
|
|||
A.CallTo(() => grain.SearchAsync("Search", A<SearchContext>.Ignored)) |
|||
.Returns(foundIds); |
|||
|
|||
var ids = await sut.SearchAsync("Search", GetApp(), schemaId, true); |
|||
|
|||
Assert.Equal(foundIds, ids); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_not_call_grain_when_input_is_empty() |
|||
{ |
|||
var ids = await sut.SearchAsync(string.Empty, GetApp(), schemaId, false); |
|||
|
|||
Assert.Null(ids); |
|||
|
|||
A.CallTo(() => grain.SearchAsync(A<string>.Ignored, A<SearchContext>.Ignored)) |
|||
.MustNotHaveHappened(); |
|||
} |
|||
|
|||
private static IAppEntity GetApp() |
|||
{ |
|||
var app = A.Fake<IAppEntity>(); |
|||
|
|||
A.CallTo(() => app.LanguagesConfig).Returns(LanguagesConfig.Build(Language.EN, Language.DE)); |
|||
|
|||
return app; |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,178 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using System.Collections.Generic; |
|||
using System.Threading.Tasks; |
|||
using Squidex.Domain.Apps.Core.Contents; |
|||
using Squidex.Infrastructure; |
|||
using Squidex.Infrastructure.Assets; |
|||
using Xunit; |
|||
|
|||
namespace Squidex.Domain.Apps.Entities.Contents.Text |
|||
{ |
|||
public class TextIndexerGrainTests : IDisposable |
|||
{ |
|||
private readonly Guid schemaId = Guid.NewGuid(); |
|||
private readonly List<Guid> ids1 = new List<Guid> { Guid.NewGuid() }; |
|||
private readonly List<Guid> ids2 = new List<Guid> { Guid.NewGuid() }; |
|||
private readonly SearchContext context; |
|||
private readonly IAssetStore assetStore = new MemoryAssetStore(); |
|||
private readonly TextIndexerGrain sut; |
|||
|
|||
public TextIndexerGrainTests() |
|||
{ |
|||
context = new SearchContext |
|||
{ |
|||
Languages = new HashSet<string> { "de", "en" } |
|||
}; |
|||
|
|||
sut = new TextIndexerGrain(assetStore); |
|||
sut.ActivateAsync(schemaId).Wait(); |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
sut.OnDeactivateAsync().Wait(); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_read_index_and_retrieve() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
await sut.DeactivateAsync(); |
|||
|
|||
var other = new TextIndexerGrain(assetStore); |
|||
try |
|||
{ |
|||
await other.ActivateAsync(schemaId); |
|||
|
|||
var foundHello = await other.SearchAsync("Hello", context); |
|||
var foundWorld = await other.SearchAsync("World", context); |
|||
|
|||
Assert.Equal(ids1, foundHello); |
|||
Assert.Equal(ids2, foundWorld); |
|||
} |
|||
finally |
|||
{ |
|||
await other.OnDeactivateAsync(); |
|||
} |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_index_invariant_content_and_retrieve() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
var foundHello = await sut.SearchAsync("Hello", context); |
|||
var foundWorld = await sut.SearchAsync("World", context); |
|||
|
|||
Assert.Equal(ids1, foundHello); |
|||
Assert.Equal(ids2, foundWorld); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_index_invariant_content_and_retrieve_with_fuzzy() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
var foundHello = await sut.SearchAsync("helo~", context); |
|||
var foundWorld = await sut.SearchAsync("wold~", context); |
|||
|
|||
Assert.Equal(ids1, foundHello); |
|||
Assert.Equal(ids2, foundWorld); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_delete_documents_from_index() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
await sut.DeleteAsync(ids1[0]); |
|||
await sut.FlushAsync(); |
|||
|
|||
var helloIds = await sut.SearchAsync("Hello", context); |
|||
|
|||
var worldIds = await sut.SearchAsync("World", context); |
|||
|
|||
Assert.Empty(helloIds); |
|||
Assert.Equal(ids2, worldIds); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_index_localized_content_and_retrieve() |
|||
{ |
|||
await AddLocalizedContent(); |
|||
|
|||
var german1 = await sut.SearchAsync("Stadt", context); |
|||
var german2 = await sut.SearchAsync("and", context); |
|||
|
|||
var germanStopwordsIds = await sut.SearchAsync("und", context); |
|||
|
|||
Assert.Equal(ids1, german1); |
|||
Assert.Equal(ids1, german2); |
|||
Assert.Equal(ids2, germanStopwordsIds); |
|||
|
|||
var english1 = await sut.SearchAsync("City", context); |
|||
var english2 = await sut.SearchAsync("und", context); |
|||
|
|||
var englishStopwordsIds = await sut.SearchAsync("and", context); |
|||
|
|||
Assert.Equal(ids2, english1); |
|||
Assert.Equal(ids2, english2); |
|||
Assert.Equal(ids1, englishStopwordsIds); |
|||
} |
|||
|
|||
[Fact] |
|||
public async Task Should_throw_exception_for_invalid_query() |
|||
{ |
|||
await AddInvariantContent(); |
|||
|
|||
await Assert.ThrowsAsync<ValidationException>(() => sut.SearchAsync("~hello", context)); |
|||
} |
|||
|
|||
private async Task AddLocalizedContent() |
|||
{ |
|||
var germanData = |
|||
new NamedContentData() |
|||
.AddField("localized", |
|||
new ContentFieldData() |
|||
.AddValue("de", "Stadt und Umgebung and whatever")); |
|||
|
|||
var englishData = |
|||
new NamedContentData() |
|||
.AddField("localized", |
|||
new ContentFieldData() |
|||
.AddValue("en", "City and Surroundings und sonstiges")); |
|||
|
|||
await sut.IndexAsync(ids1[0], new IndexData { Data = germanData }); |
|||
await sut.IndexAsync(ids2[0], new IndexData { Data = englishData }); |
|||
await sut.FlushAsync(); |
|||
} |
|||
|
|||
private async Task AddInvariantContent() |
|||
{ |
|||
var data1 = |
|||
new NamedContentData() |
|||
.AddField("test", |
|||
new ContentFieldData() |
|||
.AddValue("iv", "Hello")); |
|||
|
|||
var data2 = |
|||
new NamedContentData() |
|||
.AddField("test", |
|||
new ContentFieldData() |
|||
.AddValue("iv", "World")); |
|||
|
|||
await sut.IndexAsync(ids1[0], new IndexData { Data = data1 }); |
|||
await sut.IndexAsync(ids2[0], new IndexData { Data = data2 }); |
|||
|
|||
await sut.FlushAsync(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,35 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using System; |
|||
using Microsoft.Azure.Documents.Client; |
|||
using Squidex.Infrastructure.TestHelpers; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
public sealed class CosmosDbEventStoreFixture : IDisposable |
|||
{ |
|||
private const string EmulatorKey = "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw=="; |
|||
private const string EmulatorUri = "https://localhost:8081"; |
|||
private readonly DocumentClient client; |
|||
|
|||
public CosmosDbEventStore EventStore { get; } |
|||
|
|||
public CosmosDbEventStoreFixture() |
|||
{ |
|||
client = new DocumentClient(new Uri(EmulatorUri), EmulatorKey, JsonHelper.DefaultSettings()); |
|||
|
|||
EventStore = new CosmosDbEventStore(client, EmulatorKey, "Test", JsonHelper.DefaultSettings()); |
|||
EventStore.InitializeAsync().Wait(); |
|||
} |
|||
|
|||
public void Dispose() |
|||
{ |
|||
client.DeleteDatabaseAsync(UriFactory.CreateDatabaseUri("Test")).Wait(); |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,29 @@ |
|||
// ==========================================================================
|
|||
// Squidex Headless CMS
|
|||
// ==========================================================================
|
|||
// Copyright (c) Squidex UG (haftungsbeschraenkt)
|
|||
// All rights reserved. Licensed under the MIT license.
|
|||
// ==========================================================================
|
|||
|
|||
using Xunit; |
|||
|
|||
namespace Squidex.Infrastructure.EventSourcing |
|||
{ |
|||
[Trait("Category", "Dependencies")] |
|||
public class CosmosDbEventStoreTests : EventStoreTests<CosmosDbEventStore>, IClassFixture<CosmosDbEventStoreFixture> |
|||
{ |
|||
private readonly CosmosDbEventStoreFixture fixture; |
|||
|
|||
protected override int SubscriptionDelayInMs { get; } = 1000; |
|||
|
|||
public CosmosDbEventStoreTests(CosmosDbEventStoreFixture fixture) |
|||
{ |
|||
this.fixture = fixture; |
|||
} |
|||
|
|||
public override CosmosDbEventStore CreateStore() |
|||
{ |
|||
return fixture.EventStore; |
|||
} |
|||
} |
|||
} |
|||
Loading…
Reference in new issue